Compare commits

..

3 Commits
nodb ... main

Author SHA1 Message Date
Andrew Lalis a7561c32c3 Moved stuff. 2023-05-17 12:44:23 +02:00
Andrew Lalis 00cfcda9b8 Added improved CLI structure. 2023-05-17 12:40:01 +02:00
Andrew Lalis ba4a0c16d6 Improved main project readme. 2023-05-17 11:38:26 +02:00
88 changed files with 1296 additions and 1949 deletions

1
.gitignore vendored
View File

@ -1 +1,2 @@
build/
cli*

View File

@ -1,7 +1,53 @@
# Gymboard
Leaderboards for your local community gym.
Leaderboards and social lifting for your local community gym.
Gymboard is a platform for sharing videos of your gym lifts with the world,
from your local gym to the world's stage.
## Architecture Overview
Gymboard is designed as a sort of hybrid architecture combining the best of
microservices and monoliths. Here's a short list of each project that makes
up the Gymboard ecosystem, and what they do:
| Project | Type | Description |
| --- | --- | --- |
| 💻 **gymboard-app** | TS, VueJS, Quasar | A front-end web application for users accessing Gymboard. |
| 🧬 **gymboard-api** | Java, Spring | The *main* backend service that the app talks to. Includes auth and most business logic. |
| 🔍 **gymboard-search** | Java, Lucene | An indexing and searching service that scrapes data from *gymboard-api* to build Lucene search indexes. |
| 🗂 **gymboard-cdn** | Java, Spring | A minimal content-delivery service that manages general file storage, including video processing. |
| 🛠 **gymboard-cli** | D | **WIP** command-line-interface for managing all services for development and deployment. |
| 📸 **gymboard-uploads** | D, Handy-Httpd | **WIP** dedicated service for video upload processing, to extract functionality from *gymboard-cdn*. |
## Development
Gymboard is comprised of a variety of components, each in its own directory, and with its own project format. Follow the instructions in the README of the respective project to set that one up.
A `docker-compose.yml` file is defined in this directory, and it defines a set of services that may be used by one or more services. Install docker on your system if you haven't already, and run `docker-compose up -d` to start the services.
Gymboard is comprised of a variety of components, each in its own directory,
and with its own project format. Follow the instructions in the README of the
respective project to set that one up.
A `docker-compose.yml` file is defined in this directory, and it defines a set
of services that may be used by one or more services. Install docker on your
system if you haven't already, and run `docker-compose up -d` to start the
services.
Run `./build-cli.d` to build and prepare a `cli` executable that you can use to
run the Gymboard CLI.
**WIP:**
A `build-apps.d` script is available to try and build all projects and collect
their artifacts in a `build/` directory for deployment.
> Eventually, this functionality will be merged into *gymboard-cli*.
### Local Environment
The requirements to develop each project depend of course on the type of
project. But in general, the following software recommendations should hold:
| Type | Requirements |
| --- | --- |
| **Java** | [Latest LTS version](https://adoptium.net/en-GB/temurin/releases/), latest Maven version |
| **VueJS** | Vue 3, with a recent version of NodeJS and NPM or similar. |
| **D** | [D toolchain](https://dlang.org/download.html) (compiler + dub) for D version >= 2.103, DMD recommended compiler |
[Docker](https://www.docker.com/) is recommended for running local dependencies
like DB, mail server, message queues, etc.

View File

@ -6,6 +6,8 @@
* thread.
*
* Supply the "clean" command to remove all build artifacts instead of building.
*
* Eventually, this will be migrated to gymboard-cli.
*/
module build_apps;
@ -20,7 +22,8 @@ enum BUILDS_DIR = "build";
int main(string[] args) {
if (args.length > 1 && args[1] == "clean") {
rmdirRecurse(BUILDS_DIR);
writeln("Cleaning builds");
if (exists(BUILDS_DIR)) rmdirRecurse(BUILDS_DIR);
return 0;
}
Thread[] buildThreads = [
@ -91,8 +94,10 @@ int runBuild(const BuildSpec spec) {
string buildDir = buildPath(BUILDS_DIR, spec.name);
if (exists(buildDir)) rmdirRecurse(buildDir);
mkdirRecurse(buildDir);
File buildLogFile = File(buildPath(buildDir, "build.log"), "w");
File buildErrorLogFile = File(buildPath(buildDir, "build-error.log"), "w");
const logFilePath = buildPath(buildDir, "build.log");
const errorLogFilePath = buildPath(buildDir, "build-error.log");
File buildLogFile = File(logFilePath, "w");
File buildErrorLogFile = File(errorLogFilePath, "w");
Pid pid = spawnShell(
spec.buildCommand,
std.stdio.stdin,
@ -104,7 +109,14 @@ int runBuild(const BuildSpec spec) {
nativeShell()
);
int result = wait(pid);
if (result != 0) return result;
if (result != 0) {
writefln!"Build command failed for build \"%s\". Check %s for more info."(spec.name, errorLogFilePath);
return result;
}
// Clean up unused log files.
if (getSize(logFilePath) == 0) std.file.remove(logFilePath);
if (getSize(errorLogFilePath) == 0) std.file.remove(errorLogFilePath);
// Find and extract artifacts.
bool artifactFound = false;

42
build-cli.d Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env rdmd
/**
* Run this script with `./build.d` to prepare the latest version of the CLI
* for use. It compiles the CLI application, and copies a "cli" executable to
* the project's root directory for use.
*/
module build_cli;
import std.stdio;
import std.process;
import std.file;
import std.path;
int main() {
writeln("Building...");
const mainDir = getcwd();
chdir("gymboard-cli");
auto result = executeShell("dub build --build=release");
if (result.status != 0) {
stderr.writefln!"Build failed: %d"(result.status);
stderr.writeln(result.output);
return result.status;
}
string finalPath = buildPath("..", "cli");
if (exists(finalPath)) std.file.remove(finalPath);
version (Posix) {
string sourceExecutable = "gymboard-cli";
}
version (Windows) {
string sourceExecutable = "gymboard-cli.exe";
}
std.file.copy(sourceExecutable, finalPath);
version (Posix) {
result = executeShell("chmod +x " ~ finalPath);
if (result.status != 0) {
stderr.writefln!"Failed to enable executable permission: %d"(result.status);
return result.status;
}
}
writeln("Done! Run ./cli to start using Gymboard CLI.");
return 0;
}

View File

@ -62,8 +62,7 @@ public class SecurityConfig {
"/auth/token",
"/auth/register",
"/auth/activate",
"/auth/reset-password",
"/submissions/video-processing-complete"
"/auth/reset-password"
).permitAll()
// Everything else must be authenticated, just to be safe.
.anyRequest().authenticated();

View File

@ -1,39 +0,0 @@
package nl.andrewlalis.gymboard_api.config;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.method.HandlerMethod;
import org.springframework.web.servlet.HandlerInterceptor;
import java.lang.reflect.Method;
/**
* An interceptor that checks that requests to endpoints annotated with
* {@link ServiceOnly} have a valid service secret header value.
*/
@Component
public class ServiceAccessInterceptor implements HandlerInterceptor {
public static final String HEADER_NAME = "X-Gymboard-Service-Secret";
@Value("${app.service-secret}")
private String serviceSecret;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Method handlerMethod = ((HandlerMethod) handler).getMethod();
Class<?> handlerClass = handlerMethod.getDeclaringClass();
ServiceOnly methodAnnotation = handlerMethod.getAnnotation(ServiceOnly.class);
ServiceOnly classAnnotation = handlerClass.getAnnotation(ServiceOnly.class);
if (methodAnnotation != null || classAnnotation != null) {
String secret = request.getHeader(HEADER_NAME);
if (secret == null || !secret.trim().equals(serviceSecret)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
return false;
}
}
return true;
}
}

View File

@ -1,16 +0,0 @@
package nl.andrewlalis.gymboard_api.config;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation that can be applied to a controller or controller method to
* restrict access to only requests from another service that provide a
* legitimate service secret.
* @see ServiceAccessInterceptor
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ServiceOnly {}

View File

@ -22,11 +22,9 @@ public class WebComponents {
@Value("${app.cdn-origin}")
private String cdnOrigin;
@Value("${app.cdn-secret}")
private String cdnSecret;
@Bean
public CdnClient cdnClient() {
return new CdnClient(cdnOrigin, cdnSecret);
return new CdnClient(cdnOrigin);
}
}

View File

@ -1,8 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.GymService;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;

View File

@ -1,6 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.LeaderboardService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;

View File

@ -1,8 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.submission.controller;
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.config.ServiceOnly;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.VideoProcessingCompletePayload;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.springframework.http.ResponseEntity;
@ -28,10 +26,4 @@ public class SubmissionController {
submissionService.deleteSubmission(submissionId, user);
return ResponseEntity.noContent().build();
}
@PostMapping(path = "/video-processing-complete") @ServiceOnly
public ResponseEntity<Void> handleVideoProcessingComplete(@RequestBody VideoProcessingCompletePayload taskStatus) {
submissionService.handleVideoProcessingComplete(taskStatus);
return ResponseEntity.noContent().build();
}
}

View File

@ -1,6 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.UserSubmissionService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;

View File

@ -1,24 +1,14 @@
package nl.andrewlalis.gymboard_api.domains.submission.dao;
package nl.andrewlalis.gymboard_api.domains.api.dao.submission;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface SubmissionRepository extends JpaRepository<Submission, String>, JpaSpecificationExecutor<Submission> {
@Modifying
void deleteAllByUser(User user);
@Query("SELECT s FROM Submission s " +
"WHERE s.videoProcessingTaskId = :taskId AND " +
"s.processing = TRUE")
List<Submission> findUnprocessedByTaskId(long taskId);
List<Submission> findAllByProcessingTrue();
}

View File

@ -0,0 +1,12 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
import java.time.LocalDateTime;
public record SubmissionPayload(
String exerciseShortName,
LocalDateTime performedAt,
float weight,
String weightUnit,
int reps,
String videoFileId
) {}

View File

@ -0,0 +1,37 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.dto.UserResponse;
import nl.andrewlalis.gymboard_api.util.StandardDateFormatter;
public record SubmissionResponse(
String id,
String createdAt,
GymSimpleResponse gym,
ExerciseResponse exercise,
UserResponse user,
String performedAt,
String videoFileId,
double rawWeight,
String weightUnit,
double metricWeight,
int reps,
boolean verified
) {
public SubmissionResponse(Submission submission) {
this(
submission.getId(),
StandardDateFormatter.format(submission.getCreatedAt()),
new GymSimpleResponse(submission.getGym()),
new ExerciseResponse(submission.getExercise()),
new UserResponse(submission.getUser()),
StandardDateFormatter.format(submission.getPerformedAt()),
submission.getVideoFileId(),
submission.getRawWeight().doubleValue(),
submission.getWeightUnit().name(),
submission.getMetricWeight().doubleValue(),
submission.getReps(),
submission.isVerified()
);
}
}

View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
public record VideoProcessingCompletePayload(
long taskId,
String status,
String videoFileId,
String thumbnailFileId
) {}

View File

@ -0,0 +1,137 @@
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.hibernate.annotations.CreationTimestamp;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Entity
@Table(name = "submission")
public class Submission {
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Gym gym;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Exercise exercise;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private User user;
@Column(nullable = false)
private LocalDateTime performedAt;
/**
* The id of the video file that was submitted for this submission. It lives
* on the <em>gymboard-cdn</em> service as a stored file, which can be
* accessed via <code>GET https://CDN-HOST/files/{videoFileId}</code>.
*/
@Column(nullable = false, updatable = false, length = 26)
private String videoFileId;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal rawWeight;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private WeightUnit weightUnit;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal metricWeight;
@Column(nullable = false)
private int reps;
@Column(nullable = false)
private boolean verified;
public Submission() {}
public Submission(
String id,
Gym gym,
Exercise exercise,
User user,
LocalDateTime performedAt,
String videoFileId,
BigDecimal rawWeight,
WeightUnit unit,
BigDecimal metricWeight,
int reps
) {
this.id = id;
this.gym = gym;
this.exercise = exercise;
this.videoFileId = videoFileId;
this.user = user;
this.performedAt = performedAt;
this.rawWeight = rawWeight;
this.weightUnit = unit;
this.metricWeight = metricWeight;
this.reps = reps;
this.verified = false;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Gym getGym() {
return gym;
}
public Exercise getExercise() {
return exercise;
}
public String getVideoFileId() {
return videoFileId;
}
public User getUser() {
return user;
}
public LocalDateTime getPerformedAt() {
return performedAt;
}
public BigDecimal getRawWeight() {
return rawWeight;
}
public WeightUnit getWeightUnit() {
return weightUnit;
}
public BigDecimal getMetricWeight() {
return metricWeight;
}
public int getReps() {
return reps;
}
public boolean isVerified() {
return verified;
}
public void setVerified(boolean verified) {
this.verified = verified;
}
}

View File

@ -1,8 +1,7 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;

View File

@ -1,8 +1,7 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
@Entity
@Table(

View File

@ -1,10 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.api.service;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymResponse;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.util.PredicateBuilder;
import org.slf4j.Logger;
@ -42,14 +42,13 @@ public class GymService {
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return submissionRepository.findAll((root, query, criteriaBuilder) -> {
query.orderBy(
criteriaBuilder.desc(root.get("properties").get("performedAt")),
criteriaBuilder.desc(root.get("performedAt")),
criteriaBuilder.desc(root.get("createdAt"))
);
query.distinct(true);
return PredicateBuilder.and(criteriaBuilder)
.with(criteriaBuilder.equal(root.get("gym"), gym))
.with(criteriaBuilder.isTrue(root.get("verified")))
.with(criteriaBuilder.isFalse(root.get("processing")))
.build();
}, PageRequest.of(0, 5))
.map(SubmissionResponse::new)

View File

@ -1,10 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.api.service;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.LeaderboardTimeframe;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
@ -55,10 +55,8 @@ public class LeaderboardService {
query.distinct(true);
query.orderBy(criteriaBuilder.desc(root.get("metricWeight")));
// Basic predicates that should always hold.
PredicateBuilder pb = PredicateBuilder.and(criteriaBuilder)
.with(criteriaBuilder.isTrue(root.get("verified")))
.with(criteriaBuilder.isFalse(root.get("processing")));
.with(criteriaBuilder.isTrue(root.get("verified")));
cutoffTime.ifPresent(time -> pb.with(criteriaBuilder.greaterThan(root.get("performedAt"), time)));
optionalExercise.ifPresent(exercise -> pb.with(criteriaBuilder.equal(root.get("exercise"), exercise)));

View File

@ -1,7 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboard_api.config.ServiceAccessInterceptor;
import java.io.IOException;
import java.net.URI;
@ -14,28 +13,23 @@ import java.time.Duration;
public class CdnClient {
private final HttpClient httpClient;
private final String baseUrl;
private final String cdnSecret;
private final ObjectMapper objectMapper;
public final UploadsClient uploads;
public final FilesClient files;
public CdnClient(String baseUrl, String cdnSecret) {
public CdnClient(String baseUrl) {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(3))
.followRedirects(HttpClient.Redirect.NORMAL)
.build();
this.baseUrl = baseUrl;
this.cdnSecret = cdnSecret;
this.objectMapper = new ObjectMapper();
this.uploads = new UploadsClient(this);
this.files = new FilesClient(this);
}
public <T> T get(String urlPath, Class<T> responseType) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.GET()
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() == 200) {
@ -52,31 +46,8 @@ public class CdnClient {
.POST(HttpRequest.BodyPublishers.ofFile(filePath))
.header("Content-Type", contentType)
.header("X-Gymboard-Filename", filePath.getFileName().toString())
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
return objectMapper.readValue(response.body(), responseType);
}
public void post(String urlPath) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.POST(HttpRequest.BodyPublishers.noBody())
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<Void> response = httpClient.send(req, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() != 200) {
throw new IOException("Request failed with code " + response.statusCode());
}
}
public void delete(String urlPath) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.DELETE()
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<Void> response = httpClient.send(req, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() >= 400) {
throw new IOException("Request failed with code " + response.statusCode());
}
}
}

View File

@ -1,18 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
public record FilesClient(CdnClient client) {
public record FileMetadataResponse(
String filename,
String mimeType,
long size,
String createdAt
) {}
public FileMetadataResponse getFileMetadata(String id) throws Exception {
return client.get("/files/" + id + "/metadata", FileMetadataResponse.class);
}
public void deleteFile(String id) throws Exception {
client.delete("/files/" + id);
}
}

View File

@ -3,22 +3,26 @@ package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
import java.nio.file.Path;
public record UploadsClient(CdnClient client) {
public record FileUploadResponse(long taskId) {}
public record VideoProcessingTaskStatusResponse(
String status,
String videoFileId,
String thumbnailFileId
public record FileUploadResponse(String id) {}
public record VideoProcessingTaskStatusResponse(String status) {}
public record FileMetadataResponse(
String filename,
String mimeType,
long size,
String uploadedAt,
boolean availableForDownload
) {}
public long uploadVideo(Path filePath, String contentType) throws Exception {
return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class).taskId();
public FileUploadResponse uploadVideo(Path filePath, String contentType) throws Exception {
return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class);
}
public VideoProcessingTaskStatusResponse getVideoProcessingTaskStatus(long id) throws Exception {
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) throws Exception {
return client.get("/uploads/video/" + id + "/status", VideoProcessingTaskStatusResponse.class);
}
public void startTask(long taskId) throws Exception {
client.post("/uploads/video/" + taskId + "/start");
public FileMetadataResponse getFileMetadata(String id) throws Exception {
return client.get("/files/" + id + "/metadata", FileMetadataResponse.class);
}
}

View File

@ -2,33 +2,26 @@ package nl.andrewlalis.gymboard_api.domains.api.service.submission;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dto.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.CdnClient;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.UploadsClient;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionProperties;
import nl.andrewlalis.gymboard_api.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.concurrent.TimeUnit;
/**
* Service which handles the rather mundane tasks associated with exercise
@ -87,52 +80,31 @@ public class ExerciseSubmissionService {
}
// Create the submission.
LocalDateTime performedAt = LocalDateTime.now();
if (payload.performedAt() != null) {
performedAt = LocalDate.parse(payload.performedAt()).atTime(performedAt.toLocalTime());
}
LocalDateTime performedAt = payload.performedAt();
if (performedAt == null) performedAt = LocalDateTime.now();
BigDecimal rawWeight = BigDecimal.valueOf(payload.weight());
WeightUnit weightUnit = WeightUnit.parse(payload.weightUnit());
BigDecimal metricWeight = BigDecimal.valueOf(payload.weight());
if (weightUnit == WeightUnit.POUNDS) {
metricWeight = WeightUnit.toKilograms(rawWeight);
}
SubmissionProperties properties = new SubmissionProperties(
exercise,
Submission submission = submissionRepository.saveAndFlush(new Submission(
ulid.nextULID(), gym, exercise, user,
performedAt,
rawWeight,
weightUnit,
payload.reps()
);
Submission submission = new Submission(ulid.nextULID(), gym, user, payload.taskId(), properties);
try {
cdnClient.uploads.startTask(submission.getVideoProcessingTaskId());
submission.setProcessing(true);
} catch (Exception e) {
log.error("Failed to start video processing task for submission.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to start video processing.");
}
submission = submissionRepository.saveAndFlush(submission);
payload.videoFileId(),
rawWeight, weightUnit, metricWeight, payload.reps()
));
return new SubmissionResponse(submission);
}
private ValidationResponse validateSubmissionData(Gym gym, User user, Exercise exercise, SubmissionPayload data) {
ValidationResponse response = new ValidationResponse();
LocalDateTime now = LocalDateTime.now();
LocalDateTime cutoff = LocalDateTime.now().minusDays(3);
if (data.performedAt() != null) {
try {
LocalDateTime performedAt = LocalDate.parse(data.performedAt()).atTime(now.toLocalTime());
if (performedAt.isAfter(now)) {
response.addMessage("Cannot submit an exercise from the future.");
}
if (performedAt.isBefore(cutoff)) {
response.addMessage("Cannot submit an exercise too far in the past.");
}
} catch (DateTimeParseException e) {
response.addMessage("Invalid performedAt format.");
}
if (data.performedAt() != null && data.performedAt().isAfter(LocalDateTime.now())) {
response.addMessage("Cannot submit an exercise from the future.");
}
if (data.performedAt() != null && data.performedAt().isBefore(cutoff)) {
response.addMessage("Cannot submit an exercise too far in the past.");
}
if (data.reps() < 1 || data.reps() > 500) {
response.addMessage("Invalid rep count.");
@ -146,13 +118,17 @@ public class ExerciseSubmissionService {
}
try {
var status = cdnClient.uploads.getVideoProcessingTaskStatus(data.taskId());
if (status == null || !status.status().equalsIgnoreCase("NOT_STARTED")) {
response.addMessage("Invalid video processing task.");
UploadsClient.FileMetadataResponse metadata = cdnClient.uploads.getFileMetadata(data.videoFileId());
if (metadata == null) {
response.addMessage("Missing video file.");
} else if (!metadata.availableForDownload()) {
response.addMessage("File not yet available for download.");
} else if (!"video/mp4".equals(metadata.mimeType())) {
response.addMessage("Invalid video file format.");
}
} catch (Exception e) {
log.error("Error fetching task status.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error fetching uploaded video task status.");
log.error("Error fetching file metadata.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error fetching uploaded video file metadata.");
}
return response;
}
@ -164,133 +140,7 @@ public class ExerciseSubmissionService {
if (!submission.getUser().getId().equals(user.getId())) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Cannot delete other user's submission.");
}
try {
if (submission.getVideoFileId() != null) {
cdnClient.files.deleteFile(submission.getVideoFileId());
}
if (submission.getThumbnailFileId() != null) {
cdnClient.files.deleteFile(submission.getThumbnailFileId());
}
} catch (Exception e) {
log.error("Couldn't delete CDN content for submission " + submissionId, e);
}
// TODO: Find a secure way to delete the associated video.
submissionRepository.delete(submission);
}
/**
* This method is invoked when the CDN calls this API's endpoint to notify
* us that a video processing task has completed. If the task completed
* successfully, we can set any related submissions' video and thumbnail
* file ids and remove its "processing" flag. Otherwise, we should delete
* the failed submission.
* @param payload The information about the task.
*/
@Transactional
public void handleVideoProcessingComplete(VideoProcessingCompletePayload payload) {
var submissionsToUpdate = submissionRepository.findUnprocessedByTaskId(payload.taskId());
log.info("Received video processing complete message from CDN: {}, affecting {} submissions.", payload, submissionsToUpdate.size());
for (var submission : submissionsToUpdate) {
if (payload.status().equalsIgnoreCase("COMPLETED")) {
submission.setVideoFileId(payload.videoFileId());
submission.setThumbnailFileId(payload.thumbnailFileId());
submission.setProcessing(false);
submissionRepository.save(submission);
// TODO: Send notification of successful processing to the user!
} else if (payload.status().equalsIgnoreCase("FAILED")) {
submissionRepository.delete(submission);
// TODO: Send notification of failed video processing to the user!
}
}
}
/**
* A scheduled task that checks and resolves issues with any submission that
* stays in the "processing" state for too long.
* TODO: Find some way to clean up this mess of logic!
*/
@Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
public void checkProcessingSubmissions() {
var processingSubmissions = submissionRepository.findAllByProcessingTrue();
LocalDateTime actionCutoff = LocalDateTime.now().minus(Duration.ofMinutes(3));
LocalDateTime deleteCutoff = LocalDateTime.now().minus(Duration.ofMinutes(30));
for (var submission : processingSubmissions) {
if (submission.getCreatedAt().isBefore(actionCutoff)) {
// Sanity check to remove any inconsistent submission that doesn't have a task id for whatever reason.
if (submission.getVideoProcessingTaskId() == null) {
log.warn(
"Removing long-processing submission {} for user {} because it doesn't have a task id.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
continue;
}
try {
var status = cdnClient.uploads.getVideoProcessingTaskStatus(submission.getVideoProcessingTaskId());
if (status == null) {
// The task no longer exists on the CDN, so remove the submission.
log.warn(
"Removing long-processing submission {} for user {} because its task no longer exists on the CDN.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("FAILED")) {
// The task failed, so we should remove the submission.
log.warn(
"Removing long-processing submission {} for user {} because its task failed.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("COMPLETED")) {
// The submission should be marked as complete.
submission.setVideoFileId(status.videoFileId());
submission.setThumbnailFileId(status.thumbnailFileId());
submission.setProcessing(false);
submissionRepository.save(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("NOT_STARTED")) {
// If for whatever reason the submission's video processing never started, start now.
try {
cdnClient.uploads.startTask(submission.getVideoProcessingTaskId());
} catch (Exception e) {
log.error("Failed to start processing task " + submission.getVideoProcessingTaskId(), e);
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
} else {
// The task is waiting or processing, so delete the submission if it's been in that state for an unreasonably long time.
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
} catch (Exception e) {
log.error("Couldn't fetch status of long-processing submission " + submission.getId() + " for user " + submission.getUser().getEmail(), e);
// We can't reliably remove this submission yet, so we'll try again on the next pass.
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
}
}
}
}

View File

@ -1,7 +1,7 @@
package nl.andrewlalis.gymboard_api.domains.api.service.submission;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.auth.service.UserAccessService;
@ -9,8 +9,10 @@ import nl.andrewlalis.gymboard_api.util.PredicateBuilder;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.util.List;

View File

@ -1,15 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.auth.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.auth.model.UserAccountDataRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface UserAccountDataRequestRepository extends JpaRepository<UserAccountDataRequest, Long> {
boolean existsByUserIdAndFulfilledFalse(String userId);
@Modifying
void deleteAllByUser(User user);
}

View File

@ -116,15 +116,10 @@ public class TokenService {
public Jws<Claims> getToken(String token) {
if (token == null) return null;
try {
var builder = Jwts.parserBuilder()
.setSigningKey(this.getPrivateKey())
.requireIssuer(ISSUER);
return builder.build().parseClaimsJws(token);
} catch (Exception e) {
log.warn("Error parsing JWT.", e);
return null;
}
var builder = Jwts.parserBuilder()
.setSigningKey(this.getPrivateKey())
.requireIssuer(ISSUER);
return builder.build().parseClaimsJws(token);
}
private PrivateKey getPrivateKey() {

View File

@ -1,10 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.auth.service;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionReportRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.auth.dao.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionVoteRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@ -21,9 +19,6 @@ public class UserAccountDeletionService {
private final EmailResetCodeRepository emailResetCodeRepository;
private final PasswordResetCodeRepository passwordResetCodeRepository;
private final SubmissionRepository submissionRepository;
private final SubmissionReportRepository submissionReportRepository;
private final SubmissionVoteRepository submissionVoteRepository;
private final UserAccountDataRequestRepository accountDataRequestRepository;
public UserAccountDeletionService(UserRepository userRepository,
UserReportRepository userReportRepository,
@ -31,10 +26,7 @@ public class UserAccountDeletionService {
UserActivationCodeRepository userActivationCodeRepository,
EmailResetCodeRepository emailResetCodeRepository,
PasswordResetCodeRepository passwordResetCodeRepository,
SubmissionRepository submissionRepository,
SubmissionReportRepository submissionReportRepository,
SubmissionVoteRepository submissionVoteRepository,
UserAccountDataRequestRepository accountDataRequestRepository) {
SubmissionRepository submissionRepository) {
this.userRepository = userRepository;
this.userReportRepository = userReportRepository;
this.userFollowingRepository = userFollowingRepository;
@ -42,9 +34,6 @@ public class UserAccountDeletionService {
this.emailResetCodeRepository = emailResetCodeRepository;
this.passwordResetCodeRepository = passwordResetCodeRepository;
this.submissionRepository = submissionRepository;
this.submissionReportRepository = submissionReportRepository;
this.submissionVoteRepository = submissionVoteRepository;
this.accountDataRequestRepository = accountDataRequestRepository;
}
@Transactional
@ -57,9 +46,6 @@ public class UserAccountDeletionService {
userReportRepository.deleteAllByUserOrReportedBy(user, user);
userFollowingRepository.deleteAllByFollowedUserOrFollowingUser(user, user);
submissionRepository.deleteAllByUser(user);
submissionReportRepository.deleteAllByUser(user);
submissionVoteRepository.deleteAllByUser(user);
accountDataRequestRepository.deleteAllByUser(user);
userRepository.deleteById(user.getId());
}
}

View File

@ -1,13 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionReport;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface SubmissionReportRepository extends JpaRepository<SubmissionReport, Long> {
@Modifying
void deleteAllByUser(User user);
}

View File

@ -1,13 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionVote;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface SubmissionVoteRepository extends JpaRepository<SubmissionVote, Long> {
@Modifying
void deleteAllByUser(User user);
}

View File

@ -1,10 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.dto;
public record SubmissionPayload(
String exerciseShortName,
String performedAt,
float weight,
String weightUnit,
int reps,
long taskId
) {}

View File

@ -1,48 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.dto;
import nl.andrewlalis.gymboard_api.domains.api.dto.ExerciseResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymSimpleResponse;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.dto.UserResponse;
import nl.andrewlalis.gymboard_api.util.StandardDateFormatter;
public record SubmissionResponse(
String id,
String createdAt,
GymSimpleResponse gym,
UserResponse user,
long videoProcessingTaskId,
String videoFileId,
String thumbnailFileId,
boolean processing,
boolean verified,
// From SubmissionProperties
ExerciseResponse exercise,
String performedAt,
double rawWeight,
String weightUnit,
double metricWeight,
int reps
) {
public SubmissionResponse(Submission submission) {
this(
submission.getId(),
StandardDateFormatter.format(submission.getCreatedAt()),
new GymSimpleResponse(submission.getGym()),
new UserResponse(submission.getUser()),
submission.getVideoProcessingTaskId(),
submission.getVideoFileId(),
submission.getThumbnailFileId(),
submission.isProcessing(),
submission.isVerified(),
new ExerciseResponse(submission.getProperties().getExercise()),
StandardDateFormatter.format(submission.getCreatedAt()),
submission.getProperties().getRawWeight().doubleValue(),
submission.getProperties().getWeightUnit().name(),
submission.getProperties().getMetricWeight().doubleValue(),
submission.getProperties().getReps()
);
}
}

View File

@ -1,159 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* The Submission entity represents a user's posted video of a lift they did at
* a gym.
* <p>
* A submission is created in the front-end using the following flow:
* </p>
* <ol>
* <li>User uploads a raw video of their lift.</li>
* <li>User enters some basic information about the lift.</li>
* <li>User submits the lift.</li>
* <li>API validates the information.</li>
* <li>API creates a new Submission, and tells the CDN service to process
* the uploaded video.</li>
* <li>Once processing completes successfully, the CDN sends the final video
* and thumbnail file ids to the API and the Submission's "processing" flag
* is removed.</li>
* <li>If for whatever reason the CDN's video processing fails or never
* completes, the Submission is deleted and the user is notified of the
* issue.</li>
* </ol>
*/
@Entity
@Table(name = "submission")
public class Submission {
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Gym gym;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private User user;
/**
* The id of the video processing task that a user gives to us when they
* create the submission, so that when the task finishes processing, we can
* route its data to the right submission.
*/
@Column
private Long videoProcessingTaskId;
/**
* The id of the video file that was submitted for this submission. It lives
* on the <em>gymboard-cdn</em> service as a stored file, which can be
* accessed via <code>GET https://CDN-HOST/files/{videoFileId}</code>.
*/
@Column(length = 26)
private String videoFileId = null;
/**
* The id of the thumbnail file that was generated for this submission.
* Similarly to the video file id, it refers to a file managed by the CDN.
*/
@Column(length = 26)
private String thumbnailFileId = null;
/**
* The user-specified properties of the submission.
*/
@Embedded
private SubmissionProperties properties;
/**
* A flag that indicates whether this submission is currently processing.
* A submission is processing until its associated processing task completes
* either successfully or unsuccessfully.
*/
@Column(nullable = false)
private boolean processing;
@Column(nullable = false)
private boolean verified;
public Submission() {}
public Submission(
String id,
Gym gym,
User user,
long videoProcessingTaskId,
SubmissionProperties properties
) {
this.id = id;
this.gym = gym;
this.user = user;
this.videoProcessingTaskId = videoProcessingTaskId;
this.properties = properties;
this.verified = false;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Gym getGym() {
return gym;
}
public Long getVideoProcessingTaskId() {
return videoProcessingTaskId;
}
public String getVideoFileId() {
return videoFileId;
}
public String getThumbnailFileId() {
return thumbnailFileId;
}
public void setVideoFileId(String videoFileId) {
this.videoFileId = videoFileId;
}
public void setThumbnailFileId(String thumbnailFileId) {
this.thumbnailFileId = thumbnailFileId;
}
public User getUser() {
return user;
}
public SubmissionProperties getProperties() {
return properties;
}
public boolean isProcessing() {
return processing;
}
public void setProcessing(boolean processing) {
this.processing = processing;
}
public boolean isVerified() {
return verified;
}
public void setVerified(boolean verified) {
this.verified = verified;
}
}

View File

@ -1,74 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import java.math.BigDecimal;
import java.time.LocalDateTime;
/**
* Basic user-specified properties about a Submission.
*/
@Embeddable
public class SubmissionProperties {
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Exercise exercise;
@Column(nullable = false)
private LocalDateTime performedAt;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal rawWeight;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private WeightUnit weightUnit;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal metricWeight;
@Column(nullable = false)
private int reps;
public SubmissionProperties() {}
public SubmissionProperties(
Exercise exercise,
LocalDateTime performedAt,
BigDecimal rawWeight,
WeightUnit weightUnit,
int reps
) {
this.exercise = exercise;
this.performedAt = performedAt;
this.rawWeight = rawWeight;
this.weightUnit = weightUnit;
this.metricWeight = WeightUnit.toKilograms(rawWeight, weightUnit);
this.reps = reps;
}
public Exercise getExercise() {
return exercise;
}
public LocalDateTime getPerformedAt() {
return performedAt;
}
public BigDecimal getRawWeight() {
return rawWeight;
}
public WeightUnit getWeightUnit() {
return weightUnit;
}
public BigDecimal getMetricWeight() {
return metricWeight;
}
public int getReps() {
return reps;
}
}

View File

@ -56,7 +56,7 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
gen.generate();
completed.add(gen);
} catch (Exception e) {
throw new RuntimeException("Generator failed: " + gen.getClass().getSimpleName(), e);
throw new RuntimeException("Generator failed: " + gen.getClass().getSimpleName());
}
}
}

View File

@ -1,25 +1,21 @@
package nl.andrewlalis.gymboard_api.util.sample_data;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.CdnClient;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.UploadsClient;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionProperties;
import nl.andrewlalis.gymboard_api.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Path;
import java.time.Duration;
@ -29,31 +25,35 @@ import java.util.*;
@Component
@Profile("development")
public class SampleSubmissionGenerator implements SampleDataGenerator {
private static final Logger log = LoggerFactory.getLogger(SampleSubmissionGenerator.class);
private final GymRepository gymRepository;
private final UserRepository userRepository;
private final ExerciseRepository exerciseRepository;
private final ExerciseSubmissionService submissionService;
private final SubmissionRepository submissionRepository;
private final ULID ulid;
private final CdnClient cdnClient;
public SampleSubmissionGenerator(GymRepository gymRepository, UserRepository userRepository, ExerciseRepository exerciseRepository, SubmissionRepository submissionRepository, ULID ulid, CdnClient cdnClient) {
@Value("${app.cdn-origin}")
private String cdnOrigin;
public SampleSubmissionGenerator(GymRepository gymRepository, UserRepository userRepository, ExerciseRepository exerciseRepository, ExerciseSubmissionService submissionService, SubmissionRepository submissionRepository, ULID ulid) {
this.gymRepository = gymRepository;
this.userRepository = userRepository;
this.exerciseRepository = exerciseRepository;
this.submissionService = submissionService;
this.submissionRepository = submissionRepository;
this.ulid = ulid;
this.cdnClient = cdnClient;
}
@Override
public void generate() throws Exception {
// First we generate a small set of uploaded files that all the
// submissions can link to, instead of having them all upload new content.
var uploads = generateUploads();
final CdnClient cdnClient = new CdnClient(cdnOrigin);
List<String> videoIds = new ArrayList<>();
var video1 = cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_curl.mp4"), "video/mp4");
var video2 = cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_ohp.mp4"), "video/mp4");
videoIds.add(video1.id());
videoIds.add(video2.id());
// Now that uploads are complete, we can proceed with generating the submissions.
List<Gym> gyms = gymRepository.findAll();
List<User> users = userRepository.findAll();
List<Exercise> exercises = exerciseRepository.findAll();
@ -65,16 +65,15 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
Random random = new Random(1);
List<Submission> submissions = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
Submission submission = generateRandomSubmission(
submissions.add(generateRandomSubmission(
gyms,
users,
exercises,
uploads,
videoIds,
earliestSubmission,
latestSubmission,
random
);
submissions.add(submission);
));
}
submissionRepository.saveAll(submissions);
}
@ -83,7 +82,7 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
List<Gym> gyms,
List<User> users,
List<Exercise> exercises,
Map<Long, Pair<String, String>> uploads,
List<String> videoIds,
LocalDateTime earliestSubmission,
LocalDateTime latestSubmission,
Random random
@ -96,26 +95,20 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
weightUnit = WeightUnit.POUNDS;
rawWeight = metricWeight.multiply(new BigDecimal("2.2046226218"));
}
SubmissionProperties properties = new SubmissionProperties(
randomChoice(exercises, random),
time,
rawWeight,
weightUnit,
random.nextInt(13) + 1
);
var submission = new Submission(
ulid.nextULID(),
randomChoice(gyms, random),
randomChoice(exercises, random),
randomChoice(users, random),
randomChoice(new ArrayList<>(uploads.keySet()), random),
properties
time,
randomChoice(videoIds, random),
rawWeight,
weightUnit,
metricWeight,
random.nextInt(13) + 1
);
submission.setVerified(true);
submission.setProcessing(false);
var uploadData = uploads.get(submission.getVideoProcessingTaskId());
submission.setVideoFileId(uploadData.getFirst());
submission.setThumbnailFileId(uploadData.getSecond());
return submission;
}
@ -132,46 +125,4 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
Duration dur = Duration.between(start, end);
return start.plusSeconds(rand.nextLong(dur.toSeconds() + 1));
}
/**
* Generates a set of sample video uploads to use for all the sample
* submissions.
* @return A map containing keys representing video processing task ids, and
* values being a pair of video and thumbnail file ids.
* @throws Exception If an error occurs.
*/
private Map<Long, Pair<String, String>> generateUploads() throws Exception {
List<Long> taskIds = new ArrayList<>();
taskIds.add(cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_curl.mp4"), "video/mp4"));
taskIds.add(cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_ohp.mp4"), "video/mp4"));
Map<Long, UploadsClient.VideoProcessingTaskStatusResponse> taskStatus = new HashMap<>();
for (long taskId : taskIds) {
cdnClient.uploads.startTask(taskId);
taskStatus.put(taskId, cdnClient.uploads.getVideoProcessingTaskStatus(taskId));
}
// Wait for all video uploads to complete.
while (
taskStatus.values().stream()
.map(UploadsClient.VideoProcessingTaskStatusResponse::status)
.anyMatch(status -> !List.of("COMPLETED", "FAILED").contains(status.toUpperCase()))
) {
log.info("Waiting for sample video upload tasks to finish...");
Thread.sleep(1000);
for (long taskId : taskIds) taskStatus.put(taskId, cdnClient.uploads.getVideoProcessingTaskStatus(taskId));
}
// If any upload failed, throw an exception and cancel this generator.
if (taskStatus.values().stream().anyMatch(r -> r.status().equalsIgnoreCase("FAILED"))) {
throw new IOException("Video upload task processing failed.");
}
// Prepare the final data structure.
Map<Long, Pair<String, String>> finalResults = new HashMap<>();
for (var entry : taskStatus.entrySet()) {
finalResults.put(entry.getKey(), Pair.of(entry.getValue().videoFileId(), entry.getValue().thumbnailFileId()));
}
return finalResults;
}
}

View File

@ -15,9 +15,7 @@ spring.mail.protocol=smtp
spring.mail.properties.mail.smtp.timeout=10000
app.auth.private-key-location=./private_key.der
app.service-secret=testing
app.web-origin=http://localhost:9000
app.cdn-origin=http://localhost:8082
app.cdn-secret=testing
#logging.level.root=DEBUG

View File

@ -8,7 +8,6 @@ const api = axios.create({
});
export enum VideoProcessingStatus {
NOT_STARTED = 'NOT_STARTED',
WAITING = 'WAITING',
IN_PROGRESS = 'IN_PROGRESS',
COMPLETED = 'COMPLETED',
@ -19,21 +18,24 @@ export interface FileMetadata {
filename: string;
mimeType: string;
size: number;
createdAt: string;
uploadedAt: string;
availableForDownload: boolean;
}
export async function uploadVideoToCDN(file: File): Promise<number> {
export async function uploadVideoToCDN(file: File): Promise<string> {
const response = await api.post('/uploads/video', file, {
headers: {
'Content-Type': file.type,
},
});
return response.data.taskId;
return response.data.id;
}
export async function getVideoProcessingStatus(taskId: number): Promise<VideoProcessingStatus | null> {
export async function getVideoProcessingStatus(
id: string
): Promise<VideoProcessingStatus | null> {
try {
const response = await api.get(`/uploads/video/${taskId}/status`);
const response = await api.get(`/uploads/video/${id}/status`);
return response.data.status;
} catch (error: any) {
if (error.response && error.response.status === 404) {
@ -43,14 +45,16 @@ export async function getVideoProcessingStatus(taskId: number): Promise<VideoPro
}
}
export async function waitUntilVideoProcessingComplete(taskId: number): Promise<VideoProcessingStatus> {
export async function waitUntilVideoProcessingComplete(
id: string
): Promise<VideoProcessingStatus> {
let failureCount = 0;
let attemptCount = 0;
while (failureCount < 5 && attemptCount < 60) {
await sleep(1000);
attemptCount++;
try {
const status = await getVideoProcessingStatus(taskId);
const status = await getVideoProcessingStatus(id);
failureCount = 0;
if (
status === VideoProcessingStatus.COMPLETED ||

View File

@ -1,5 +1,5 @@
import { GeoPoint } from 'src/api/main/models';
import SubmissionsModule, { Submission, parseSubmission } from 'src/api/main/submission';
import SubmissionsModule, { ExerciseSubmission, parseSubmission } from 'src/api/main/submission';
import { api } from 'src/api/main/index';
import { GymRoutable } from 'src/router/gym-routing';
@ -51,7 +51,7 @@ class GymsModule {
public async getRecentSubmissions(
gym: GymRoutable
): Promise<Array<Submission>> {
): Promise<Array<ExerciseSubmission>> {
const response = await api.get(
`/gyms/${gym.countryCode}_${gym.cityShortName}_${gym.shortName}/recent-submissions`
);

View File

@ -1,4 +1,4 @@
import { Submission, parseSubmission } from 'src/api/main/submission';
import { ExerciseSubmission, parseSubmission } from 'src/api/main/submission';
import { getGymCompoundId, GymRoutable } from 'src/router/gym-routing';
import { api } from 'src/api/main/index';
@ -29,7 +29,7 @@ interface RequestParams {
class LeaderboardsModule {
public async getLeaderboard(
params: LeaderboardParams
): Promise<Array<Submission>> {
): Promise<Array<ExerciseSubmission>> {
const requestParams: RequestParams = {};
if (params.exerciseShortName) {
requestParams.exercise = params.exerciseShortName;

View File

@ -14,7 +14,7 @@ export interface ExerciseSubmissionPayload {
weight: number;
weightUnit: string;
reps: number;
taskId: number;
videoFileId: string;
}
export enum WeightUnit {
@ -29,35 +29,30 @@ export class WeightUnitUtil {
}
}
export interface Submission {
export interface ExerciseSubmission {
id: string;
createdAt: DateTime;
gym: SimpleGym;
user: User;
videoFileId: string | null;
thumbnailFileId: string | null;
processing: boolean;
verified: boolean;
exercise: Exercise;
user: User;
performedAt: DateTime;
videoFileId: string;
rawWeight: number;
weightUnit: WeightUnit;
metricWeight: number;
reps: number;
}
export function parseSubmission(data: any): Submission {
export function parseSubmission(data: any): ExerciseSubmission {
data.createdAt = DateTime.fromISO(data.createdAt);
data.performedAt = DateTime.fromISO(data.performedAt);
return data as Submission;
return data as ExerciseSubmission;
}
class SubmissionsModule {
public async getSubmission(
submissionId: string
): Promise<Submission> {
): Promise<ExerciseSubmission> {
const response = await api.get(`/submissions/${submissionId}`);
return parseSubmission(response.data);
}
@ -66,7 +61,7 @@ class SubmissionsModule {
gym: GymRoutable,
payload: ExerciseSubmissionPayload,
authStore: AuthStoreType
): Promise<Submission> {
): Promise<ExerciseSubmission> {
const gymId = getGymCompoundId(gym);
const response = await api.post(`/gyms/${gymId}/submissions`, payload, authStore.axiosConfig);
return parseSubmission(response.data);

View File

@ -1,15 +1,15 @@
import {api} from 'src/api/main';
import {AuthStoreType} from 'stores/auth-store';
import {Submission, parseSubmission} from 'src/api/main/submission';
import {ExerciseSubmission, parseSubmission} from 'src/api/main/submission';
import {defaultPaginationOptions, Page, PaginationOptions, toQueryParams} from 'src/api/main/models';
class UsersModule {
public async getRecentSubmissions(userId: string, authStore: AuthStoreType): Promise<Array<Submission>> {
public async getRecentSubmissions(userId: string, authStore: AuthStoreType): Promise<Array<ExerciseSubmission>> {
const response = await api.get(`/users/${userId}/recent-submissions`, authStore.axiosConfig);
return response.data.map(parseSubmission);
}
public async getSubmissions(userId: string, authStore: AuthStoreType, paginationOptions: PaginationOptions = defaultPaginationOptions()): Promise<Page<Submission>> {
public async getSubmissions(userId: string, authStore: AuthStoreType, paginationOptions: PaginationOptions = defaultPaginationOptions()): Promise<Page<ExerciseSubmission>> {
const config = structuredClone(authStore.axiosConfig);
config.params = toQueryParams(paginationOptions);
const response = await api.get(`/users/${userId}/submissions`, config);

View File

@ -19,11 +19,11 @@
</template>
<script setup lang="ts">
import { Submission, WeightUnitUtil } from 'src/api/main/submission';
import { ExerciseSubmission, WeightUnitUtil } from 'src/api/main/submission';
import { DateTime } from 'luxon';
interface Props {
submission: Submission;
submission: ExerciseSubmission;
showName?: boolean;
showGym?: boolean;
}

View File

@ -51,8 +51,8 @@ export default {
upload: 'Video File to Upload',
submit: 'Submit',
submitUploading: 'Uploading video...',
submitUploadFailed: 'Video upload failed.',
submitCreatingSubmission: 'Creating submission...',
submitVideoProcessing: 'Processing...',
submitComplete: 'Submission complete!',
submitFailed: 'Submission processing failed. Please try again later.',
},

View File

@ -2,7 +2,6 @@
<q-page>
<StandardCenteredPage v-if="submission">
<video
v-if="!submission.processing"
class="submission-video"
:src="getFileUrl(submission.videoFileId)"
loop
@ -11,9 +10,6 @@
preload="metadata"
autoplay
/>
<div v-if="submission.processing">
<p>This submission is still processing.</p>
</div>
<h3>
{{ submission.rawWeight }}&nbsp;{{ WeightUnitUtil.toAbbreviation(submission.weightUnit) }}
{{ submission.exercise.displayName }}
@ -27,7 +23,7 @@
<!-- Deletion button is only visible if the user who submitted it is viewing it. -->
<q-btn
v-if="authStore.user && authStore.user.id === submission.user.id && !submission.processing"
v-if="authStore.user && authStore.user.id === submission.user.id"
label="Delete"
@click="deleteSubmission"
/>
@ -38,18 +34,18 @@
<script setup lang="ts">
import api from 'src/api/main';
import StandardCenteredPage from 'src/components/StandardCenteredPage.vue';
import { Submission, WeightUnitUtil } from 'src/api/main/submission';
import { ExerciseSubmission, WeightUnitUtil } from 'src/api/main/submission';
import { onMounted, ref, Ref } from 'vue';
import { useRoute, useRouter } from 'vue-router';
import { DateTime } from 'luxon';
import { getFileUrl } from 'src/api/cdn';
import { getGymRoute } from 'src/router/gym-routing';
import {useAuthStore} from 'stores/auth-store';
import {confirm, showApiErrorToast, showInfoToast} from 'src/utils';
import {confirm, showApiErrorToast} from 'src/utils';
import {useI18n} from 'vue-i18n';
import {useQuasar} from 'quasar';
const submission: Ref<Submission | undefined> = ref();
const submission: Ref<ExerciseSubmission | undefined> = ref();
const route = useRoute();
const router = useRouter();
@ -58,24 +54,14 @@ const i18n = useI18n();
const quasar = useQuasar();
onMounted(async () => {
await loadSubmission();
if (submission.value?.processing) {
showInfoToast('This submission is still processing.');
}
});
async function loadSubmission() {
const submissionId = route.params.submissionId as string;
try {
submission.value = await api.gyms.submissions.getSubmission(submissionId);
if (submission.value.processing) {
setTimeout(loadSubmission, 3000);
const submissionId = route.params.submissionId as string;
try {
submission.value = await api.gyms.submissions.getSubmission(submissionId);
} catch (error) {
console.error(error);
await router.push('/');
}
} catch (error) {
showApiErrorToast(error);
await router.push('/');
}
}
});
/**
* Shows a confirmation dialog asking the user if they really want to delete

View File

@ -43,7 +43,7 @@
<script setup lang="ts">
import { nextTick, onMounted, ref, Ref } from 'vue';
import { Submission } from 'src/api/main/submission';
import { ExerciseSubmission } from 'src/api/main/submission';
import api from 'src/api/main';
import { getGymFromRoute } from 'src/router/gym-routing';
import ExerciseSubmissionListItem from 'components/ExerciseSubmissionListItem.vue';
@ -51,7 +51,7 @@ import { Gym } from 'src/api/main/gyms';
import 'leaflet/dist/leaflet.css';
import { Map, Marker, TileLayer } from 'leaflet';
const recentSubmissions: Ref<Array<Submission>> = ref([]);
const recentSubmissions: Ref<Array<ExerciseSubmission>> = ref([]);
const gym: Ref<Gym | undefined> = ref();
const TILE_URL = 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png';

View File

@ -33,13 +33,13 @@ import api from 'src/api/main';
import { Exercise } from 'src/api/main/exercises';
import { Gym } from 'src/api/main/gyms';
import { LeaderboardTimeframe } from 'src/api/main/leaderboards';
import { Submission } from 'src/api/main/submission';
import { ExerciseSubmission } from 'src/api/main/submission';
import ExerciseSubmissionListItem from 'src/components/ExerciseSubmissionListItem.vue';
import { getGymFromRoute } from 'src/router/gym-routing';
import { sleep } from 'src/utils';
import { onMounted, ref, Ref, watch, computed } from 'vue';
const submissions: Ref<Array<Submission>> = ref([]);
const submissions: Ref<Array<ExerciseSubmission>> = ref([]);
const gym: Ref<Gym | undefined> = ref();
const exercises: Ref<Array<Exercise>> = ref([]);

View File

@ -48,7 +48,7 @@ A high-level overview of the submission process is as follows:
</div>
<div class="row">
<q-input
v-model="submissionModel.performedAt"
v-model="submissionModel.date"
type="date"
:label="$t('gymPage.submitPage.date')"
class="col-12"
@ -91,22 +91,28 @@ A high-level overview of the submission process is as follows:
</template>
<script setup lang="ts">
import {onMounted, ref, Ref} from 'vue';
import {getGymFromRoute} from 'src/router/gym-routing';
import { onMounted, ref, Ref } from 'vue';
import { getGymFromRoute } from 'src/router/gym-routing';
import SlimForm from 'components/SlimForm.vue';
import api from 'src/api/main';
import {Gym} from 'src/api/main/gyms';
import {Exercise} from 'src/api/main/exercises';
import {useRoute, useRouter} from 'vue-router';
import {showApiErrorToast, showWarningToast, sleep} from 'src/utils';
import {uploadVideoToCDN,} from 'src/api/cdn';
import {useAuthStore} from 'stores/auth-store';
import {useI18n} from 'vue-i18n';
import { Gym } from 'src/api/main/gyms';
import { Exercise } from 'src/api/main/exercises';
import { useRoute, useRouter } from 'vue-router';
import { showApiErrorToast, sleep } from 'src/utils';
import {
uploadVideoToCDN,
VideoProcessingStatus,
waitUntilVideoProcessingComplete,
} from 'src/api/cdn';
import { useAuthStore } from 'stores/auth-store';
import { useI18n } from 'vue-i18n';
import { useQuasar } from 'quasar';
const authStore = useAuthStore();
const router = useRouter();
const route = useRoute();
const i18n = useI18n();
const quasar = useQuasar();
interface Option {
value: string;
@ -121,8 +127,8 @@ let submissionModel = ref({
weight: 100,
weightUnit: 'Kg',
reps: 1,
performedAt: new Date().toLocaleDateString('en-CA'),
taskId: -1
videoFileId: '',
date: new Date().toLocaleDateString('en-CA'),
});
const selectedVideoFile: Ref<File | undefined> = ref<File>();
const weightUnits = ['KG', 'LBS'];
@ -163,53 +169,59 @@ async function onSubmitted() {
if (!selectedVideoFile.value || !gym.value) return;
submitting.value = true;
if (await uploadVideo()) {
await createSubmission();
}
submitting.value = false;
}
/**
* Uploads the selected video and returns true if successful.
*/
async function uploadVideo(): Promise<boolean> {
if (!selectedVideoFile.value) return false;
try {
// 1. Upload the video to the CDN.
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitUploading');
await sleep(1000);
submissionModel.value.taskId = await uploadVideoToCDN(selectedVideoFile.value);
return true;
} catch (error) {
showApiErrorToast(error);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitUploadFailed');
await sleep(1000);
selectedVideoFile.value = undefined;
submitButtonLabel.value = i18n.t('gymPage.submitPage.submit');
return false;
}
}
submissionModel.value.videoFileId = await uploadVideoToCDN(
selectedVideoFile.value
);
/**
* Tries to create a new submission, and if successful, redirects the user to it.
*/
async function createSubmission() {
if (!gym.value) return;
try {
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitCreatingSubmission');
await sleep(1000);
const submission = await api.gyms.submissions.createSubmission(gym.value, submissionModel.value, authStore);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitComplete');
await sleep(2000);
await router.push(`/submissions/${submission.id}`);
} catch (error: any) {
if (error.response && error.response.status === 400) {
showWarningToast(error.response.data.message);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitFailed');
// 2. Wait for the video to be processed.
submitButtonLabel.value = i18n.t(
'gymPage.submitPage.submitVideoProcessing'
);
const processingStatus = await waitUntilVideoProcessingComplete(
submissionModel.value.videoFileId
);
// 3. If successful upload, create the submission.
if (processingStatus === VideoProcessingStatus.COMPLETED) {
try {
submitButtonLabel.value = i18n.t(
'gymPage.submitPage.submitCreatingSubmission'
);
await sleep(1000);
const submission = await api.gyms.submissions.createSubmission(
gym.value,
submissionModel.value,
authStore
);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitComplete');
await sleep(2000);
await router.push(`/submissions/${submission.id}`);
} catch (error: any) {
if (error.response && error.response.status === 400) {
quasar.notify({
message: error.response.data.message,
type: 'warning',
position: 'top',
});
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitFailed');
await sleep(3000);
} else {
showApiErrorToast(error);
}
}
// Otherwise, report the failed submission and give up.
} else {
showApiErrorToast(error);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitFailed');
await sleep(3000);
}
await sleep(3000);
} catch (error: any) {
showApiErrorToast(error);
} finally {
submitting.value = false;
submitButtonLabel.value = i18n.t('gymPage.submitPage.submit');
}
}

View File

@ -21,7 +21,7 @@ import ExerciseSubmissionListItem from 'components/ExerciseSubmissionListItem.vu
import {showApiErrorToast} from 'src/utils';
import {PaginationHelpers} from 'src/api/main/models';
import InfinitePageLoader from 'src/api/infinite-page-loader';
import {Submission} from 'src/api/main/submission';
import {ExerciseSubmission} from 'src/api/main/submission';
interface Props {
userId: string;
@ -30,7 +30,7 @@ const props = defineProps<Props>();
const authStore = useAuthStore();
const submissions: Ref<Submission[]> = ref([]);
const submissions: Ref<ExerciseSubmission[]> = ref([]);
const loader = new InfinitePageLoader(submissions, async paginationOptions => {
try {
return await api.users.getSubmissions(props.userId, authStore, paginationOptions);
@ -41,7 +41,7 @@ const loader = new InfinitePageLoader(submissions, async paginationOptions => {
onMounted(async () => {
loader.registerWindowScrollListener();
await loader.setPagination(PaginationHelpers.sortedDescBy('properties.performedAt'));
await loader.setPagination(PaginationHelpers.sortedDescBy('performedAt'));
});
</script>

View File

@ -1,12 +1,6 @@
package nl.andrewlalis.gymboardcdn;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.files.util.ULID;
import nl.andrewlalis.gymboardcdn.uploads.service.process.FfmpegThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.FfmpegVideoProcessor;
import nl.andrewlalis.gymboardcdn.uploads.service.process.ThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.VideoProcessor;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -14,32 +8,17 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import java.util.Arrays;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
@Configuration
@EnableScheduling
public class Config implements WebMvcConfigurer {
public class Config {
@Value("${app.web-origin}")
private String webOrigin;
@Value("${app.api-origin}")
private String apiOrigin;
private final ServiceAccessInterceptor serviceAccessInterceptor;
public Config(ServiceAccessInterceptor serviceAccessInterceptor) {
this.serviceAccessInterceptor = serviceAccessInterceptor;
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(serviceAccessInterceptor);
}
@Bean
public CorsFilter corsFilter() {
final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
@ -57,29 +36,4 @@ public class Config implements WebMvcConfigurer {
public ULID ulid() {
return new ULID();
}
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
}
@Bean
public FileStorageService fileStorageService() {
return new FileStorageService(ulid(), objectMapper(), "cdn-files");
}
@Bean
public VideoProcessor videoProcessor() {
return new FfmpegVideoProcessor();
}
@Bean
public ThumbnailGenerator thumbnailGenerator() {
return new FfmpegThumbnailGenerator();
}
@Bean
public Executor videoProcessingExecutor() {
return Executors.newFixedThreadPool(1);
}
}

View File

@ -1,39 +0,0 @@
package nl.andrewlalis.gymboardcdn;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.method.HandlerMethod;
import org.springframework.web.servlet.HandlerInterceptor;
import java.lang.reflect.Method;
/**
* An interceptor that checks that requests to endpoints annotated with
* {@link ServiceOnly} have a valid service secret header value.
*/
@Component
public class ServiceAccessInterceptor implements HandlerInterceptor {
private static final String HEADER_NAME = "X-Gymboard-Service-Secret";
@Value("${app.service-secret}")
private String serviceSecret;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Method handlerMethod = ((HandlerMethod) handler).getMethod();
Class<?> handlerClass = handlerMethod.getDeclaringClass();
ServiceOnly methodAnnotation = handlerMethod.getAnnotation(ServiceOnly.class);
ServiceOnly classAnnotation = handlerClass.getAnnotation(ServiceOnly.class);
if (methodAnnotation != null || classAnnotation != null) {
String secret = request.getHeader(HEADER_NAME);
if (secret == null || !secret.trim().equals(serviceSecret)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
return false;
}
}
return true;
}
}

View File

@ -1,16 +0,0 @@
package nl.andrewlalis.gymboardcdn;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation that can be applied to a controller or controller method to
* restrict access to only requests from another service that provide a
* legitimate service secret.
* @see ServiceAccessInterceptor
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ServiceOnly {}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.files;
package nl.andrewlalis.gymboardcdn.api;
public record FileMetadataResponse(
String filename,

View File

@ -0,0 +1,5 @@
package nl.andrewlalis.gymboardcdn.api;
public record FileUploadResponse(
String id
) {}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn;
package nl.andrewlalis.gymboardcdn.api;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;

View File

@ -1,8 +1,8 @@
package nl.andrewlalis.gymboardcdn.uploads.api;
package nl.andrewlalis.gymboardcdn.api;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.ServiceOnly;
import nl.andrewlalis.gymboardcdn.uploads.service.UploadService;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.UploadService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@ -17,17 +17,22 @@ public class UploadController {
}
@PostMapping(path = "/uploads/video", consumes = {"video/mp4"})
public VideoUploadResponse uploadVideo(HttpServletRequest request) {
public FileUploadResponse uploadVideo(HttpServletRequest request) {
return uploadService.processableVideoUpload(request);
}
@PostMapping(path = "/uploads/video/{taskId}/start") @ServiceOnly
public void startVideoProcessing(@PathVariable long taskId) {
uploadService.startVideoProcessing(taskId);
@GetMapping(path = "/uploads/video/{id}/status")
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) {
return uploadService.getVideoProcessingStatus(id);
}
@GetMapping(path = "/uploads/video/{taskId}/status")
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable long taskId) {
return uploadService.getVideoProcessingStatus(taskId);
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
uploadService.streamFile(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
return uploadService.getFileMetadata(id);
}
}

View File

@ -0,0 +1,5 @@
package nl.andrewlalis.gymboardcdn.api;
public record VideoProcessingTaskStatusResponse(
String status
) {}

View File

@ -1,49 +0,0 @@
package nl.andrewlalis.gymboardcdn.files;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.ServiceOnly;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
/**
* Controller for general-purpose file access.
*/
@RestController
public class FileController {
private final FileStorageService fileStorageService;
public FileController(FileStorageService fileStorageService) {
this.fileStorageService = fileStorageService;
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
fileStorageService.streamToHttpResponse(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FullFileMetadata getFileMetadata(@PathVariable String id) {
try {
var data = fileStorageService.getMetadata(id);
if (data == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND);
return data;
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Couldn't read file metadata.", e);
}
}
@DeleteMapping(path = "/files/{id}") @ServiceOnly
public void deleteFile(@PathVariable String id) {
try {
fileStorageService.delete(id);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete file.", e);
}
}
}

View File

@ -1,7 +0,0 @@
package nl.andrewlalis.gymboardcdn.files;
public record FileMetadata (
String filename,
String mimeType,
boolean accessible
) {}

View File

@ -1,223 +0,0 @@
package nl.andrewlalis.gymboardcdn.files;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.files.util.ULID;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* This service acts as a low-level driver for interacting with the storage
* system. This includes reading and writing files and their metadata.
* <p>
* Files are stored in a top-level directory, then in 3 sub-directories
* according to their creation date. So if a file is created on 2023-04-02,
* then it will be stored in BASE_DIR/2023/04/02/. All files are uniquely
* identified by a ULID; a monotonic, time-sorted id.
* </p>
* <p>
* Each file has a 1Kb (1024 bytes) metadata block baked into it when it's
* first saved. This metadata block stores a JSON-serialized set of metadata
* properties about the file.
* </p>
*/
public class FileStorageService {
private static final int HEADER_SIZE = 1024;
private final ULID ulid;
private final ObjectMapper objectMapper;
private final String baseStorageDir;
public FileStorageService(ULID ulid, ObjectMapper objectMapper, String baseStorageDir) {
this.ulid = ulid;
this.objectMapper = objectMapper;
this.baseStorageDir = baseStorageDir;
}
public String generateFileId() {
return ulid.nextULID();
}
public String save(Path inputFile, FileMetadata metadata) throws IOException {
try (var in = Files.newInputStream(inputFile)) {
return save(in, metadata, -1);
}
}
/**
* Saves a new file to the storage.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(InputStream in, FileMetadata metadata, long maxSize) throws IOException {
ULID.Value id = ulid.nextValue();
return save(id, in, metadata, maxSize);
}
/**
* Saves a new file to the storage using a specific file id.
* @param id The file id to save to.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(ULID.Value id, InputStream in, FileMetadata metadata, long maxSize) throws IOException {
Path filePath = getStoragePathForFile(id.toString());
Files.createDirectories(filePath.getParent());
try (var out = Files.newOutputStream(filePath)) {
writeMetadata(out, metadata);
byte[] buffer = new byte[8192];
int bytesRead;
long totalBytesWritten = 0;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
totalBytesWritten += bytesRead;
if (maxSize > 0 && totalBytesWritten > maxSize) {
out.close();
Files.delete(filePath);
throw new IOException("File too large.");
}
}
}
return id.toString();
}
/**
* Gets metadata for a file identified by the given id.
* @param rawId The file's id.
* @return The metadata for the file, or null if no file is found.
* @throws IOException If an error occurs while reading metadata.
*/
public FullFileMetadata getMetadata(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) return null;
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
LocalDateTime date = dateFromULID(ULID.parseULID(rawId));
return new FullFileMetadata(
metadata.filename(),
metadata.mimeType(),
Files.size(filePath) - HEADER_SIZE,
date.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
);
}
}
/**
* Streams a stored file to an HTTP response. A NOT_FOUND response is sent
* if the file doesn't exist. Responses include a cache-control header to
* allow clients to cache the response for a long time, as stored files are
* considered to be immutable, unless rarely deleted.
* @param rawId The file's id.
* @param response The HTTP response to write to.
*/
public void streamToHttpResponse(String rawId, HttpServletResponse response) {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return;
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
response.setContentType(metadata.mimeType());
response.setContentLengthLong(Files.size(filePath) - HEADER_SIZE);
response.addHeader("Cache-Control", "max-age=604800, immutable");
var out = response.getOutputStream();
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
}
}
/**
* Gets the path to the physical location of the file with the given id.
* Note that this method makes no guarantee as to whether the file exists.
* @param rawId The id of the file.
* @return The path to the location where the file is stored.
*/
public Path getStoragePathForFile(String rawId) {
ULID.Value id = ULID.parseULID(rawId);
LocalDateTime time = dateFromULID(id);
Path dir = Path.of(baseStorageDir)
.resolve(String.format("%04d", time.getYear()))
.resolve(String.format("%02d", time.getMonthValue()))
.resolve(String.format("%02d", time.getDayOfMonth()));
return dir.resolve(rawId);
}
/**
* Deletes the file with a given id, if it exists.
* @param rawId The file's id.
* @throws IOException If an error occurs.
*/
public void delete(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
Files.deleteIfExists(filePath);
}
public void copyTo(String fileId, Path filePath) throws IOException {
Path inputFilePath = getStoragePathForFile(fileId);
if (Files.notExists(inputFilePath)) {
throw new IOException("File " + fileId + " not found.");
}
try (
var in = Files.newInputStream(inputFilePath);
var out = Files.newOutputStream(filePath)
) {
readMetadata(in);
in.transferTo(out);
}
}
private static LocalDateTime dateFromULID(ULID.Value value) {
return Instant.ofEpochMilli(value.timestamp())
.atOffset(ZoneOffset.UTC)
.toLocalDateTime();
}
private FileMetadata readMetadata(InputStream in) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
int readCount = in.read(buffer.array(), 0, HEADER_SIZE);
if (readCount != HEADER_SIZE) throw new IOException("Invalid header. Read " + readCount + " bytes instead of " + HEADER_SIZE);
short metadataBytesLength = buffer.getShort();
byte[] metadataBytes = new byte[metadataBytesLength];
buffer.get(metadataBytes);
return objectMapper.readValue(metadataBytes, FileMetadata.class);
}
private void writeMetadata(OutputStream out, FileMetadata metadata) throws IOException {
ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE);
byte[] metadataBytes = objectMapper.writeValueAsBytes(metadata);
if (metadataBytes.length > HEADER_SIZE - 2) {
throw new IOException("Metadata is too large.");
}
headerBuffer.putShort((short) metadataBytes.length);
headerBuffer.put(metadataBytes);
out.write(headerBuffer.array());
}
}

View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboardcdn.files;
public record FullFileMetadata(
String filename,
String mimeType,
long size,
String createdAt
) {}

View File

@ -0,0 +1,81 @@
package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
@Entity
@Table(name = "stored_file")
public class StoredFile {
/**
* ULID-based unique file identifier.
*/
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
/**
* The timestamp at which the file was originally uploaded.
*/
@Column(nullable = false, updatable = false)
private LocalDateTime uploadedAt;
/**
* The original filename.
*/
@Column(nullable = false, updatable = false)
private String name;
/**
* The type of the file.
*/
@Column(updatable = false)
private String mimeType;
/**
* The file's size on the disk.
*/
@Column(nullable = false, updatable = false)
private long size;
public StoredFile() {}
public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) {
this.id = id;
this.name = name;
this.mimeType = mimeType;
this.size = size;
this.uploadedAt = uploadedAt;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getName() {
return name;
}
public String getMimeType() {
return mimeType;
}
public long getSize() {
return size;
}
public LocalDateTime getUploadedAt() {
return uploadedAt;
}
}

View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboardcdn.model;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface StoredFileRepository extends JpaRepository<StoredFile, String> {
}

View File

@ -0,0 +1,88 @@
package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* An entity to keep track of a task for processing a raw video into a better
* format for Gymboard to serve.
*/
@Entity
@Table(name = "task_video_processing")
public class VideoProcessingTask {
public enum Status {
WAITING,
IN_PROGRESS,
COMPLETED,
FAILED
}
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private Status status;
/**
* The original filename.
*/
@Column(nullable = false)
private String filename;
/**
* The path to the temporary file that we'll use as input.
*/
@Column(nullable = false)
private String tempFilePath;
/**
* The identifier that will be used to identify the final video, if it
* is processed successfully.
*/
@Column(nullable = false, updatable = false, length = 26)
private String videoIdentifier;
public VideoProcessingTask() {}
public VideoProcessingTask(Status status, String filename, String tempFilePath, String videoIdentifier) {
this.status = status;
this.filename = filename;
this.tempFilePath = tempFilePath;
this.videoIdentifier = videoIdentifier;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getFilename() {
return filename;
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getTempFilePath() {
return tempFilePath;
}
public String getVideoIdentifier() {
return videoIdentifier;
}
}

View File

@ -1,13 +1,16 @@
package nl.andrewlalis.gymboardcdn.uploads.model;
package nl.andrewlalis.gymboardcdn.model;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@Repository
public interface VideoProcessingTaskRepository extends JpaRepository<VideoProcessingTask, Long> {
Optional<VideoProcessingTask> findByVideoIdentifier(String identifier);
List<VideoProcessingTask> findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status status);
List<VideoProcessingTask> findAllByCreatedAtBefore(LocalDateTime cutoff);

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.uploads.service;
package nl.andrewlalis.gymboardcdn.service;
import java.io.IOException;
import java.nio.file.Path;

View File

@ -0,0 +1,116 @@
package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.concurrent.TimeUnit;
/**
* The service that manages storing and retrieving files from a base filesystem.
*/
@Service
public class FileService {
private static final Logger log = LoggerFactory.getLogger(FileService.class);
@Value("${app.files.storage-dir}")
private String storageDir;
@Value("${app.files.temp-dir}")
private String tempDir;
private final StoredFileRepository storedFileRepository;
private final ULID ulid;
public FileService(StoredFileRepository storedFileRepository, ULID ulid) {
this.storedFileRepository = storedFileRepository;
this.ulid = ulid;
}
public Path getStoragePathForFile(StoredFile file) throws IOException {
LocalDateTime time = file.getUploadedAt();
Path dir = Path.of(storageDir)
.resolve(Integer.toString(time.getYear()))
.resolve(Integer.toString(time.getMonthValue()))
.resolve(Integer.toString(time.getDayOfMonth()));
if (Files.notExists(dir)) Files.createDirectories(dir);
return dir.resolve(file.getId());
}
public String createNewFileIdentifier() {
return ulid.nextULID();
}
public Path saveToTempFile(InputStream in, String filename) throws IOException {
Path tempDir = getTempDir();
String suffix = null;
if (filename != null) {
int idx = filename.lastIndexOf('.');
if (idx >= 0) {
suffix = filename.substring(idx);
}
}
Path tempFile = Files.createTempFile(tempDir, null, suffix);
try (var out = Files.newOutputStream(tempFile)) {
in.transferTo(out);
}
return tempFile;
}
private Path getStorageDir() throws IOException {
Path dir = Path.of(storageDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
private Path getTempDir() throws IOException {
Path dir = Path.of(tempDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
/**
* Scheduled task that removes any StoredFile entities for which no more
* physical file exists.
*/
@Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
@Transactional
public void removeOrphanedFiles() {
Pageable pageable = PageRequest.of(0, 100, Sort.by(Sort.Direction.DESC, "createdAt"));
Page<StoredFile> page = storedFileRepository.findAll(pageable);
while (!page.isEmpty()) {
for (var storedFile : page) {
try {
Path filePath = getStoragePathForFile(storedFile);
if (Files.notExists(filePath)) {
log.warn("Removing stored file {} because it no longer exists on the disk.", storedFile.getId());
storedFileRepository.delete(storedFile);
}
} catch (IOException e) {
log.error("Couldn't get storage path for stored file.", e);
}
}
pageable = pageable.next();
page = storedFileRepository.findAll(pageable);
}
}
}

View File

@ -0,0 +1,131 @@
package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.format.DateTimeFormatter;
@Service
public class UploadService {
private static final Logger log = LoggerFactory.getLogger(UploadService.class);
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final StoredFileRepository storedFileRepository;
private final VideoProcessingTaskRepository videoTaskRepository;
private final FileService fileService;
public UploadService(StoredFileRepository storedFileRepository,
VideoProcessingTaskRepository videoTaskRepository,
FileService fileService) {
this.storedFileRepository = storedFileRepository;
this.videoTaskRepository = videoTaskRepository;
this.fileService = fileService;
}
/**
* Handles uploading of a processable video file that will be processed
* before being stored in the system.
* @param request The request from which we can read the file.
* @return A response that contains an identifier that can be used to check
* the status of the video processing, and eventually fetch the video.
*/
@Transactional
public FileUploadResponse processableVideoUpload(HttpServletRequest request) {
String contentLengthStr = request.getHeader("Content-Length");
if (contentLengthStr == null || !contentLengthStr.matches("\\d+")) {
throw new ResponseStatusException(HttpStatus.LENGTH_REQUIRED);
}
long contentLength = Long.parseUnsignedLong(contentLengthStr);
if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
}
Path tempFile;
String filename = request.getHeader("X-Gymboard-Filename");
if (filename == null) filename = "unnamed.mp4";
try {
tempFile = fileService.saveToTempFile(request.getInputStream(), filename);
} catch (IOException e) {
log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
String identifier = fileService.createNewFileIdentifier();
videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.WAITING,
filename,
tempFile.toString(),
identifier
));
return new FileUploadResponse(identifier);
}
/**
* Gets the status of a video processing task.
* @param id The video identifier.
* @return The status of the video processing task.
*/
@Transactional(readOnly = true)
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) {
VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(task.getStatus().name());
}
/**
* Streams the contents of a stored file to a client via the Http response.
* @param id The file's unique identifier.
* @param response The response to stream the content to.
*/
@Transactional(readOnly = true)
public void streamFile(String id, HttpServletResponse response) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = fileService.getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
@Transactional(readOnly = true)
public FileMetadataResponse getFileMetadata(String id) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
try {
Path filePath = fileService.getStoragePathForFile(file);
boolean exists = Files.exists(filePath);
return new FileMetadataResponse(
file.getName(),
file.getMimeType(),
file.getSize(),
file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
exists
);
} catch (IOException e) {
log.error("Couldn't get path to stored file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}

View File

@ -0,0 +1,173 @@
package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@Service
public class VideoProcessingService {
private static final Logger log = LoggerFactory.getLogger(VideoProcessingService.class);
private final Executor taskExecutor;
private final VideoProcessingTaskRepository taskRepo;
private final StoredFileRepository storedFileRepository;
private final FileService fileService;
public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, StoredFileRepository storedFileRepository, FileService fileService) {
this.taskExecutor = taskExecutor;
this.taskRepo = taskRepo;
this.storedFileRepository = storedFileRepository;
this.fileService = fileService;
}
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void startWaitingTasks() {
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
for (var task : waitingTasks) {
log.info("Queueing processing of video {}.", task.getVideoIdentifier());
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
taskExecutor.execute(() -> processVideo(task));
}
}
@Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS)
public void removeOldTasks() {
LocalDateTime cutoff = LocalDateTime.now().minusHours(12);
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
for (var task : oldTasks) {
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
log.info("Deleting completed task for video {}.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.info("Deleting failed task for video {}.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
log.info("Task for video {} was in progress for too long; deleting.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
log.info("Task for video {} was waiting for too long; deleting.", task.getVideoIdentifier());
taskRepo.delete(task);
}
}
}
private void processVideo(VideoProcessingTask task) {
log.info("Started processing video {}.", task.getVideoIdentifier());
Path tempFile = Path.of(task.getTempFilePath());
if (Files.notExists(tempFile) || !Files.isReadable(tempFile)) {
log.error("Temp file {} doesn't exist or isn't readable.", tempFile);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// Then begin running the actual FFMPEG processing.
Path tempDir = tempFile.getParent();
Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier());
try {
processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile);
} catch (Exception e) {
e.printStackTrace();
log.error("""
Video processing failed for video {}:
Input file: {}
Output file: {}
Exception message: {}""",
task.getVideoIdentifier(),
tempFile,
ffmpegOutputFile,
e.getMessage()
);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// And finally, copy the output to the final location.
try {
StoredFile storedFile = new StoredFile(
task.getVideoIdentifier(),
task.getFilename(),
"video/mp4",
Files.size(ffmpegOutputFile),
task.getCreatedAt()
);
Path finalFilePath = fileService.getStoragePathForFile(storedFile);
Files.move(ffmpegOutputFile, finalFilePath);
Files.deleteIfExists(tempFile);
Files.deleteIfExists(ffmpegOutputFile);
storedFileRepository.saveAndFlush(storedFile);
updateTask(task, VideoProcessingTask.Status.COMPLETED);
log.info("Finished processing video {}.", task.getVideoIdentifier());
} catch (IOException e) {
log.error("Failed to copy processed video to final storage location.", e);
updateTask(task, VideoProcessingTask.Status.FAILED);
}
}
/**
* Uses the `ffmpeg` system command to process a raw input video and produce
* a compressed, reduced-size output video that's ready for usage in the
* application.
* @param dir The working directory.
* @param inFile The input file to read from.
* @param outFile The output file to write to. MUST have a ".mp4" extension.
* @throws IOException If a filesystem error occurs.
* @throws CommandFailedException If the ffmpeg command fails.
* @throws InterruptedException If the ffmpeg command is interrupted.
*/
private void processVideoWithFFMPEG(Path dir, Path inFile, Path outFile) throws IOException, InterruptedException {
Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log");
Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log");
final String[] command = {
"ffmpeg",
"-i", inFile.getFileName().toString(),
"-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264",
"-crf", "28",
"-f", "mp4",
outFile.getFileName().toString()
};
long startSize = Files.size(inFile);
Instant startTime = Instant.now();
Process ffmpegProcess = new ProcessBuilder()
.command(command)
.redirectOutput(tmpStdout.toFile())
.redirectError(tmpStderr.toFile())
.directory(dir.toFile())
.start();
int result = ffmpegProcess.waitFor();
if (result != 0) throw new CommandFailedException(command, result, tmpStdout, tmpStderr);
long endSize = Files.size(outFile);
Duration dur = Duration.between(startTime, Instant.now());
double reductionFactor = startSize / (double) endSize;
String reductionFactorStr = String.format("%.3f%%", reductionFactor * 100);
log.info("Processed video from {} bytes to {} bytes in {} seconds, {} reduction.", startSize, endSize, dur.getSeconds(), reductionFactorStr);
// Delete the logs if everything was successful.
Files.deleteIfExists(tmpStdout);
Files.deleteIfExists(tmpStderr);
}
private void updateTask(VideoProcessingTask task, VideoProcessingTask.Status status) {
task.setStatus(status);
taskRepo.saveAndFlush(task);
}
}

View File

@ -1,7 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.api;
public record VideoProcessingTaskStatusResponse(
String status,
String videoFileId,
String thumbnailFileId
) {}

View File

@ -1,5 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.api;
public record VideoUploadResponse(
long taskId
) {}

View File

@ -1,96 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.model;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* An entity to keep track of a task for processing a raw video into a better
* format for Gymboard to serve. Generally, tasks are processed like so:
* <ol>
* <li>A video is uploaded, and a new task is created with the NOT_STARTED status.</li>
* <li>Once the Gymboard API verifies the associated submission, it'll
* request to start the task, bringing it to the WAITING status.</li>
* <li>When a task executor picks up the waiting task, its status changes to IN_PROGRESS.</li>
* <li>If the video is processed successfully, then the task is COMPLETED, otherwise FAILED.</li>
* </ol>
*/
@Entity
@Table(name = "task_video_processing")
public class VideoProcessingTask {
public enum Status {
NOT_STARTED,
WAITING,
IN_PROGRESS,
COMPLETED,
FAILED
}
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private Status status;
/**
* The file id for the original, raw user-uploaded video file that needs to
* be processed.
*/
@Column(nullable = false, updatable = false, length = 26)
private String uploadFileId;
@Column(length = 26)
private String videoFileId;
@Column(length = 26)
private String thumbnailFileId;
public VideoProcessingTask() {}
public VideoProcessingTask(Status status, String uploadFileId) {
this.status = status;
this.uploadFileId = uploadFileId;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getUploadFileId() {
return uploadFileId;
}
public String getVideoFileId() {
return videoFileId;
}
public void setVideoFileId(String videoFileId) {
this.videoFileId = videoFileId;
}
public String getThumbnailFileId() {
return thumbnailFileId;
}
public void setThumbnailFileId(String thumbnailFileId) {
this.thumbnailFileId = thumbnailFileId;
}
}

View File

@ -1,12 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.model;
public record VideoProcessingTaskStatusUpdate(
long taskId,
String status,
String videoFileId,
String thumbnailFileId
) {
public VideoProcessingTaskStatusUpdate(VideoProcessingTask task) {
this(task.getId(), task.getStatus().name(), task.getVideoFileId(), task.getThumbnailFileId());
}
}

View File

@ -1,103 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.uploads.api.VideoUploadResponse;
import nl.andrewlalis.gymboardcdn.uploads.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.files.FileMetadata;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
@Service
public class UploadService {
private static final Logger log = LoggerFactory.getLogger(UploadService.class);
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final VideoProcessingTaskRepository videoTaskRepository;
private final FileStorageService fileStorageService;
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileStorageService fileStorageService) {
this.videoTaskRepository = videoTaskRepository;
this.fileStorageService = fileStorageService;
}
/**
* Handles uploading of a processable video file that will be processed
* before being stored in the system.
* @param request The request from which we can read the file.
* @return A response containing the id of the video processing task, to be
* given to the Gymboard API so that it can further manage processing after
* a submission is completed.
*/
@Transactional
public VideoUploadResponse processableVideoUpload(HttpServletRequest request) {
String contentLengthStr = request.getHeader("Content-Length");
if (contentLengthStr == null || !contentLengthStr.matches("\\d+")) {
throw new ResponseStatusException(HttpStatus.LENGTH_REQUIRED);
}
long contentLength = Long.parseUnsignedLong(contentLengthStr);
if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
}
String filename = request.getHeader("X-Gymboard-Filename");
if (filename == null) filename = "unnamed.mp4";
FileMetadata metadata = new FileMetadata(
filename,
request.getContentType(),
false
);
String uploadFileId;
try {
uploadFileId = fileStorageService.save(request.getInputStream(), metadata, contentLength);
} catch (IOException e) {
log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
var task = videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.NOT_STARTED,
uploadFileId
));
return new VideoUploadResponse(task.getId());
}
/**
* Gets the status of a video processing task.
* @param id The task id.
* @return The status of the video processing task.
*/
@Transactional(readOnly = true)
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(long id) {
VideoProcessingTask task = videoTaskRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(
task.getStatus().name(),
task.getVideoFileId(),
task.getThumbnailFileId()
);
}
/**
* Marks this task as waiting to be picked up for processing. The Gymboard
* API should send a message itself to start processing of an uploaded video
* once it validates a submission.
* @param taskId The task id.
*/
@Transactional
public void startVideoProcessing(long taskId) {
VideoProcessingTask task = videoTaskRepository.findById(taskId)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
if (task.getStatus() == VideoProcessingTask.Status.NOT_STARTED) {
task.setStatus(VideoProcessingTask.Status.WAITING);
videoTaskRepository.save(task);
}
}
}

View File

@ -1,229 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboardcdn.files.FileMetadata;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskRepository;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskStatusUpdate;
import nl.andrewlalis.gymboardcdn.uploads.service.process.ThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.VideoProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@Service
public class VideoProcessingService {
private static final Logger log = LoggerFactory.getLogger(VideoProcessingService.class);
private final Executor videoProcessingExecutor;
private final VideoProcessingTaskRepository taskRepo;
private final FileStorageService fileStorageService;
private final VideoProcessor videoProcessor;
private final ThumbnailGenerator thumbnailGenerator;
private final ObjectMapper objectMapper;
@Value("${app.api-origin}")
private String apiOrigin;
@Value("${app.api-secret}")
private String apiSecret;
public VideoProcessingService(Executor videoProcessingExecutor,
VideoProcessingTaskRepository taskRepo,
FileStorageService fileStorageService,
VideoProcessor videoProcessor,
ThumbnailGenerator thumbnailGenerator, ObjectMapper objectMapper) {
this.videoProcessingExecutor = videoProcessingExecutor;
this.taskRepo = taskRepo;
this.fileStorageService = fileStorageService;
this.videoProcessor = videoProcessor;
this.thumbnailGenerator = thumbnailGenerator;
this.objectMapper = objectMapper;
}
private void updateTask(VideoProcessingTask task, VideoProcessingTask.Status status) {
task.setStatus(status);
taskRepo.saveAndFlush(task);
if (status == VideoProcessingTask.Status.COMPLETED || status == VideoProcessingTask.Status.FAILED) {
sendTaskCompleteToApi(task);
}
}
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void startWaitingTasks() {
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
for (var task : waitingTasks) {
log.info("Queueing processing of task {}.", task.getId());
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
videoProcessingExecutor.execute(() -> processVideo(task));
}
}
@Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS)
public void removeOldTasks() {
LocalDateTime cutoff = LocalDateTime.now().minusHours(12);
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
for (var task : oldTasks) {
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
log.info("Deleting completed task {}.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.info("Deleting failed task {}.", task.getId());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
log.info("Task {} was in progress for too long; deleting.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
log.info("Task {} was waiting for too long; deleting.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
}
}
}
private void processVideo(VideoProcessingTask task) {
log.info("Started processing task {}.", task.getId());
Path uploadFile = fileStorageService.getStoragePathForFile(task.getUploadFileId());
Path rawUploadFile = uploadFile.resolveSibling(task.getUploadFileId() + "-vid-in");
if (Files.notExists(uploadFile) || !Files.isReadable(uploadFile)) {
log.error("Uploaded video file {} doesn't exist or isn't readable.", uploadFile);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
try {
fileStorageService.copyTo(task.getUploadFileId(), rawUploadFile);
} catch (IOException e) {
log.error("Failed to copy raw video file {} to {}.", uploadFile, rawUploadFile);
e.printStackTrace();
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// Run the actual processing here.
Path videoFile = uploadFile.resolveSibling(task.getUploadFileId() + "-vid-out.mp4");
Path thumbnailFile = uploadFile.resolveSibling(task.getUploadFileId() + "-thm-out.jpeg");
try {
log.info("Processing video for uploaded video file {}.", uploadFile.getFileName());
videoProcessor.processVideo(rawUploadFile, videoFile);
log.info("Generating thumbnail for uploaded video file {}.", uploadFile.getFileName());
thumbnailGenerator.generateThumbnailImage(videoFile, thumbnailFile);
} catch (Exception e) {
e.printStackTrace();
log.error("""
Video processing failed for task {}:
Input file: {}
Output file: {}
Exception message: {}""",
task.getId(),
rawUploadFile,
videoFile,
e.getMessage()
);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// And finally, copy the output to the final location.
try {
// Save the video to a final file location.
var originalMetadata = fileStorageService.getMetadata(task.getUploadFileId());
FileMetadata metadata = new FileMetadata(originalMetadata.filename(), originalMetadata.mimeType(), true);
String videoFileId = fileStorageService.save(videoFile, metadata);
// Save the thumbnail too.
FileMetadata thumbnailMetadata = new FileMetadata("thumbnail.jpeg", "image/jpeg", true);
String thumbnailFileId = fileStorageService.save(thumbnailFile, thumbnailMetadata);
task.setVideoFileId(videoFileId);
task.setThumbnailFileId(thumbnailFileId);
updateTask(task, VideoProcessingTask.Status.COMPLETED);
log.info("Finished processing task {}.", task.getId());
} catch (IOException e) {
log.error("Failed to copy processed video to final storage location.", e);
updateTask(task, VideoProcessingTask.Status.FAILED);
} finally {
deleteAllTaskFiles(task);
}
}
/**
* Sends an update message to the Gymboard API when a task finishes its
* processing. Note that Gymboard API will also eventually poll the CDN's
* own API to get task status if we fail to send it, so there's some
* redundancy built-in.
* @param task The task to send.
*/
private void sendTaskCompleteToApi(VideoProcessingTask task) {
String json;
try {
json = objectMapper.writeValueAsString(new VideoProcessingTaskStatusUpdate(task));
} catch (JsonProcessingException e) {
log.error("JSON error while sending task data to API for task " + task.getId(), e);
return;
}
HttpClient httpClient = HttpClient.newBuilder().build();
HttpRequest request = HttpRequest.newBuilder(URI.create(apiOrigin + "/submissions/video-processing-complete"))
.header("Content-Type", "application/json")
.header("X-Gymboard-Service-Secret", apiSecret)
.timeout(Duration.ofSeconds(3))
.POST(HttpRequest.BodyPublishers.ofString(json))
.build();
try {
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() >= 400) {
log.error("API returned not-ok response {}", response.statusCode());
}
} catch (Exception e) {
log.error("Failed to send HTTP request to API.", e);
}
}
/**
* Helper function to delete all temporary files related to a task's
* processing operations. If the task is FAILED, then files are kept for
* debugging purposes.
* @param task The task to delete files for.
*/
private void deleteAllTaskFiles(VideoProcessingTask task) {
if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.warn("Retaining files for failed task {}, upload id {}.", task.getId(), task.getUploadFileId());
return;
}
Path dir = fileStorageService.getStoragePathForFile(task.getUploadFileId()).getParent();
try (var s = Files.list(dir)) {
var files = s.toList();
for (var file : files) {
String filename = file.getFileName().toString().strip();
if (Files.isRegularFile(file) && filename.startsWith(task.getUploadFileId())) {
try {
Files.delete(file);
} catch (IOException e) {
log.error("Failed to delete file " + file + " related to task " + task.getId(), e);
}
}
}
} catch (IOException e) {
log.error("Failed to list files in " + dir + " when deleting files for task " + task.getId(), e);
}
}
}

View File

@ -1,42 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import nl.andrewlalis.gymboardcdn.uploads.service.CommandFailedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public abstract class FfmpegCommandExecutor {
private static final Logger log = LoggerFactory.getLogger(FfmpegCommandExecutor.class);
protected abstract String[] buildCommand(Path inputFile, Path outputFile);
public void run(String label, Path inputFile, Path outputFile) throws IOException {
String inputFilename = inputFile.getFileName().toString().strip();
Path stdout = inputFile.resolveSibling(inputFilename + "-ffmpeg-" + label + "-out.log");
Path stderr = inputFile.resolveSibling(inputFilename + "-ffmpeg-" + label + "-err.log");
String[] command = buildCommand(inputFile, outputFile);
Process process = new ProcessBuilder(buildCommand(inputFile, outputFile))
.redirectOutput(stdout.toFile())
.redirectError(stderr.toFile())
.start();
try {
int result = process.waitFor();
if (result != 0) {
throw new CommandFailedException(command, result, stdout, stderr);
}
} catch (InterruptedException e) {
throw new IOException("Interrupted while waiting for ffmpeg to finish.", e);
}
// Try to clean up output files when the command exited successfully.
try {
Files.deleteIfExists(stdout);
Files.deleteIfExists(stderr);
} catch (IOException e) {
log.warn("Failed to delete output files after successful ffmpeg execution.", e);
}
}
}

View File

@ -1,21 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public class FfmpegThumbnailGenerator extends FfmpegCommandExecutor implements ThumbnailGenerator {
@Override
public void generateThumbnailImage(Path videoInputFile, Path outputFilePath) throws IOException {
super.run("thm", videoInputFile, outputFilePath);
}
@Override
protected String[] buildCommand(Path inputFile, Path outputFile) {
return new String[]{
"ffmpeg",
"-i", inputFile.toString(),
"-vframes", "1",
outputFile.toString()
};
}
}

View File

@ -1,48 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
public class FfmpegVideoProcessor extends FfmpegCommandExecutor implements VideoProcessor {
private static final Logger log = LoggerFactory.getLogger(FfmpegVideoProcessor.class);
@Override
public void processVideo(Path inputFilePath, Path outputFilePath) throws IOException {
Instant start = Instant.now();
long inputFileSize = Files.size(inputFilePath);
super.run("vid", inputFilePath, outputFilePath);
Duration duration = Duration.between(start, Instant.now());
long outputFileSize = Files.size(outputFilePath);
double reductionFactor = inputFileSize / (double) outputFileSize;
double durationSeconds = duration.toMillis() / 1000.0;
log.info(
"Processed video {} from {} to {} bytes, {} reduction in {} seconds.",
inputFilePath.getFileName().toString(),
inputFileSize,
outputFileSize,
String.format("%.3f%%", reductionFactor),
String.format("%.3f", durationSeconds)
);
}
@Override
protected String[] buildCommand(Path inputFile, Path outputFile) {
return new String[]{
"ffmpeg",
"-i", inputFile.toString(),
"-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264",
"-crf", "28",
"-f", "mp4",
outputFile.toString()
};
}
}

View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public interface ThumbnailGenerator {
void generateThumbnailImage(Path videoInputFile, Path outputFilePath) throws IOException;
}

View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public interface VideoProcessor {
void processVideo(Path inputFilePath, Path outputFilePath) throws IOException;
}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.files.util;
package nl.andrewlalis.gymboardcdn.util;
/*
* sulky-modules - several general-purpose modules.

View File

@ -6,9 +6,7 @@ spring.jpa.hibernate.ddl-auto=update
server.port=8082
# A secret header token that other services must provide to use service-only endpoints.
app.service-secret=testing
app.web-origin=http://localhost:9000
app.api-origin=http://localhost:8080
app.api-secret=testing
app.files.storage-dir=./cdn-files/
app.files.temp-dir=./cdn-files/tmp/

View File

@ -1,11 +1,22 @@
package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.ServletInputStream;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.AdditionalAnswers.returnsFirstArg;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
public class UploadServiceTest {
/**
@ -16,30 +27,31 @@ public class UploadServiceTest {
*/
@Test
public void processableVideoUploadSuccess() throws IOException {
// TODO: Refactor all of this!
// VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
// when(videoTaskRepository.save(any(VideoProcessingTask.class)))
// .then(returnsFirstArg());
// FileService fileService = Mockito.mock(FileService.class);
// when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
// .thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
//
// when(fileService.createNewFileIdentifier()).thenReturn("abc");
//
// UploadService uploadService = new UploadService(
// videoTaskRepository,
// fileService
// );
// HttpServletRequest mockRequest = mock(HttpServletRequest.class);
// when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
// when(mockRequest.getHeader("Content-Length")).thenReturn("123");
// ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
// when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
// var expectedResponse = new FileUploadResponse("abc");
// var response = uploadService.processableVideoUpload(mockRequest);
// assertEquals(expectedResponse, response);
// verify(fileService, times(1)).saveToTempFile(any(), any());
// verify(videoTaskRepository, times(1)).save(any());
// verify(fileService, times(1)).createNewFileIdentifier();
StoredFileRepository storedFileRepository = Mockito.mock(StoredFileRepository.class);
VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
when(videoTaskRepository.save(any(VideoProcessingTask.class)))
.then(returnsFirstArg());
FileService fileService = Mockito.mock(FileService.class);
when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
.thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
when(fileService.createNewFileIdentifier()).thenReturn("abc");
UploadService uploadService = new UploadService(
storedFileRepository,
videoTaskRepository,
fileService
);
HttpServletRequest mockRequest = mock(HttpServletRequest.class);
when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
when(mockRequest.getHeader("Content-Length")).thenReturn("123");
ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
var expectedResponse = new FileUploadResponse("abc");
var response = uploadService.processableVideoUpload(mockRequest);
assertEquals(expectedResponse, response);
verify(fileService, times(1)).saveToTempFile(any(), any());
verify(videoTaskRepository, times(1)).save(any());
verify(fileService, times(1)).createNewFileIdentifier();
}
}

View File

@ -1,6 +1,7 @@
import std.stdio;
import cli;
import command;
import services;
import consolecolors;
@ -9,11 +10,12 @@ void main() {
ServiceManager serviceManager = new ServiceManager();
CliHandler cliHandler = new CliHandler();
cliHandler.register("service", new ServiceCommand(serviceManager));
cwriteln("Gymboard CLI: Type <cyan>help</cyan> for more information. Type <red>exit</red> to exit the CLI.");
while (!cliHandler.shouldExit) {
cwriteln("\n<blue>Gymboard CLI</blue>: <grey>Command-line interface for managing Gymboard services.</grey>");
cwriteln(" Type <cyan>help</cyan> for more information.\n Type <red>exit</red> to exit the CLI.\n");
while (!cliHandler.isExitRequested) {
cwrite("&gt; ".blue);
cliHandler.readAndHandleCommand();
}
serviceManager.stopAll();
cwriteln("Goodbye!".green);
cwriteln("Goodbye!".blue);
}

View File

@ -7,45 +7,7 @@ import std.typecons;
import consolecolors;
interface CliCommand {
void handle(string[] args);
}
class CliHandler {
private CliCommand[string] commands;
public bool shouldExit = false;
public void register(string name, CliCommand command) {
this.commands[name] = command;
}
public void readAndHandleCommand() {
string[] commandAndArgs = readln().strip().split!isWhite();
if (commandAndArgs.length == 0) return;
string command = commandAndArgs[0].toLower();
if (command == "help") {
showHelp();
} else if (command == "exit") {
shouldExit = true;
} else if (command in commands) {
commands[command].handle(commandAndArgs.length > 1 ? commandAndArgs[1 .. $] : []);
} else {
cwritefln("Unknown command: %s".red, command.orange);
}
}
}
void showHelp() {
writeln(q"HELP
Gymboard CLI: A tool for streamlining development.
Commands:
help Shows this message.
exit Exits the CLI, stopping any running services.
HELP");
}
import command.base;
import services;
class ServiceCommand : CliCommand {
@ -86,6 +48,14 @@ class ServiceCommand : CliCommand {
}
}
string name() const {
return "Service";
}
string description() const {
return "bleh";
}
/**
* Validates that a service command contains as its second argument a valid
* service name.

View File

@ -0,0 +1,102 @@
module command.base;
import consolecolors;
import std.stdio;
import std.string;
import std.uni;
interface CliCommand {
void handle(string[] args);
string name() const;
string description() const;
}
class CliHandler {
private CliCommand[string] commands;
private bool exitRequested = false;
this() {
commands["help"] = new HelpCommand(this);
commands["exit"] = new ExitCommand(this);
}
void register(string name, CliCommand command) {
this.commands[name.strip().toLower()] = command;
}
void readAndHandleCommand() {
string[] commandAndArgs = readln().strip().split!isWhite();
if (commandAndArgs.length == 0) return;
string command = commandAndArgs[0].toLower();
if (command in commands) {
commands[command].handle(commandAndArgs.length > 1 ? commandAndArgs[1 .. $] : []);
} else {
cwritefln("Unknown command: %s".red, command.orange);
}
}
void setExitRequested() {
this.exitRequested = true;
}
bool isExitRequested() const {
return this.exitRequested;
}
CliCommand[string] getCommands() {
return this.commands;
}
}
class HelpCommand : CliCommand {
private CliHandler handler;
this(CliHandler handler) {
this.handler = handler;
}
void handle(string[] args) {
import std.algorithm;
cwriteln("<blue>Gymboard CLI Help</blue>: <grey>Information about how to use this program.</grey>");
string[] commandNames = this.handler.getCommands().keys;
sort(commandNames);
uint longestCommandNameLength = commandNames.map!(n => cast(uint) n.length).maxElement;
foreach (name; commandNames) {
CliCommand command = this.handler.getCommands()[name];
string formattedName = cyan(leftJustify(name, longestCommandNameLength + 1, ' '));
cwriteln(formattedName, command.description().grey);
}
}
string name() const {
return "help";
}
string description() const {
return "Shows help information.";
}
}
class ExitCommand : CliCommand {
private CliHandler handler;
this(CliHandler handler) {
this.handler = handler;
}
void handle(string[] args) {
this.handler.setExitRequested();
}
string name() const {
return "exit";
}
string description() const {
return "Exits the CLI, gracefully stopping any services or running jobs.";
}
}

View File

@ -0,0 +1,3 @@
module command;
public import command.base;