Compare commits

..

13 Commits
main ... nodb

88 changed files with 1951 additions and 1298 deletions

1
.gitignore vendored
View File

@ -1,2 +1 @@
build/
cli*

View File

@ -1,53 +1,7 @@
# Gymboard
Leaderboards and social lifting for your local community gym.
Gymboard is a platform for sharing videos of your gym lifts with the world,
from your local gym to the world's stage.
## Architecture Overview
Gymboard is designed as a sort of hybrid architecture combining the best of
microservices and monoliths. Here's a short list of each project that makes
up the Gymboard ecosystem, and what they do:
| Project | Type | Description |
| --- | --- | --- |
| 💻 **gymboard-app** | TS, VueJS, Quasar | A front-end web application for users accessing Gymboard. |
| 🧬 **gymboard-api** | Java, Spring | The *main* backend service that the app talks to. Includes auth and most business logic. |
| 🔍 **gymboard-search** | Java, Lucene | An indexing and searching service that scrapes data from *gymboard-api* to build Lucene search indexes. |
| 🗂 **gymboard-cdn** | Java, Spring | A minimal content-delivery service that manages general file storage, including video processing. |
| 🛠 **gymboard-cli** | D | **WIP** command-line-interface for managing all services for development and deployment. |
| 📸 **gymboard-uploads** | D, Handy-Httpd | **WIP** dedicated service for video upload processing, to extract functionality from *gymboard-cdn*. |
Leaderboards for your local community gym.
## Development
Gymboard is comprised of a variety of components, each in its own directory, and with its own project format. Follow the instructions in the README of the respective project to set that one up.
Gymboard is comprised of a variety of components, each in its own directory,
and with its own project format. Follow the instructions in the README of the
respective project to set that one up.
A `docker-compose.yml` file is defined in this directory, and it defines a set
of services that may be used by one or more services. Install docker on your
system if you haven't already, and run `docker-compose up -d` to start the
services.
Run `./build-cli.d` to build and prepare a `cli` executable that you can use to
run the Gymboard CLI.
**WIP:**
A `build-apps.d` script is available to try and build all projects and collect
their artifacts in a `build/` directory for deployment.
> Eventually, this functionality will be merged into *gymboard-cli*.
### Local Environment
The requirements to develop each project depend of course on the type of
project. But in general, the following software recommendations should hold:
| Type | Requirements |
| --- | --- |
| **Java** | [Latest LTS version](https://adoptium.net/en-GB/temurin/releases/), latest Maven version |
| **VueJS** | Vue 3, with a recent version of NodeJS and NPM or similar. |
| **D** | [D toolchain](https://dlang.org/download.html) (compiler + dub) for D version >= 2.103, DMD recommended compiler |
[Docker](https://www.docker.com/) is recommended for running local dependencies
like DB, mail server, message queues, etc.
A `docker-compose.yml` file is defined in this directory, and it defines a set of services that may be used by one or more services. Install docker on your system if you haven't already, and run `docker-compose up -d` to start the services.

View File

@ -1,42 +0,0 @@
#!/usr/bin/env rdmd
/**
* Run this script with `./build.d` to prepare the latest version of the CLI
* for use. It compiles the CLI application, and copies a "cli" executable to
* the project's root directory for use.
*/
module build_cli;
import std.stdio;
import std.process;
import std.file;
import std.path;
int main() {
writeln("Building...");
const mainDir = getcwd();
chdir("gymboard-cli");
auto result = executeShell("dub build --build=release");
if (result.status != 0) {
stderr.writefln!"Build failed: %d"(result.status);
stderr.writeln(result.output);
return result.status;
}
string finalPath = buildPath("..", "cli");
if (exists(finalPath)) std.file.remove(finalPath);
version (Posix) {
string sourceExecutable = "gymboard-cli";
}
version (Windows) {
string sourceExecutable = "gymboard-cli.exe";
}
std.file.copy(sourceExecutable, finalPath);
version (Posix) {
result = executeShell("chmod +x " ~ finalPath);
if (result.status != 0) {
stderr.writefln!"Failed to enable executable permission: %d"(result.status);
return result.status;
}
}
writeln("Done! Run ./cli to start using Gymboard CLI.");
return 0;
}

View File

@ -6,8 +6,6 @@
* thread.
*
* Supply the "clean" command to remove all build artifacts instead of building.
*
* Eventually, this will be migrated to gymboard-cli.
*/
module build_apps;
@ -22,8 +20,7 @@ enum BUILDS_DIR = "build";
int main(string[] args) {
if (args.length > 1 && args[1] == "clean") {
writeln("Cleaning builds");
if (exists(BUILDS_DIR)) rmdirRecurse(BUILDS_DIR);
rmdirRecurse(BUILDS_DIR);
return 0;
}
Thread[] buildThreads = [
@ -94,10 +91,8 @@ int runBuild(const BuildSpec spec) {
string buildDir = buildPath(BUILDS_DIR, spec.name);
if (exists(buildDir)) rmdirRecurse(buildDir);
mkdirRecurse(buildDir);
const logFilePath = buildPath(buildDir, "build.log");
const errorLogFilePath = buildPath(buildDir, "build-error.log");
File buildLogFile = File(logFilePath, "w");
File buildErrorLogFile = File(errorLogFilePath, "w");
File buildLogFile = File(buildPath(buildDir, "build.log"), "w");
File buildErrorLogFile = File(buildPath(buildDir, "build-error.log"), "w");
Pid pid = spawnShell(
spec.buildCommand,
std.stdio.stdin,
@ -109,14 +104,7 @@ int runBuild(const BuildSpec spec) {
nativeShell()
);
int result = wait(pid);
if (result != 0) {
writefln!"Build command failed for build \"%s\". Check %s for more info."(spec.name, errorLogFilePath);
return result;
}
// Clean up unused log files.
if (getSize(logFilePath) == 0) std.file.remove(logFilePath);
if (getSize(errorLogFilePath) == 0) std.file.remove(errorLogFilePath);
if (result != 0) return result;
// Find and extract artifacts.
bool artifactFound = false;

View File

@ -62,7 +62,8 @@ public class SecurityConfig {
"/auth/token",
"/auth/register",
"/auth/activate",
"/auth/reset-password"
"/auth/reset-password",
"/submissions/video-processing-complete"
).permitAll()
// Everything else must be authenticated, just to be safe.
.anyRequest().authenticated();

View File

@ -0,0 +1,39 @@
package nl.andrewlalis.gymboard_api.config;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.method.HandlerMethod;
import org.springframework.web.servlet.HandlerInterceptor;
import java.lang.reflect.Method;
/**
* An interceptor that checks that requests to endpoints annotated with
* {@link ServiceOnly} have a valid service secret header value.
*/
@Component
public class ServiceAccessInterceptor implements HandlerInterceptor {
public static final String HEADER_NAME = "X-Gymboard-Service-Secret";
@Value("${app.service-secret}")
private String serviceSecret;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Method handlerMethod = ((HandlerMethod) handler).getMethod();
Class<?> handlerClass = handlerMethod.getDeclaringClass();
ServiceOnly methodAnnotation = handlerMethod.getAnnotation(ServiceOnly.class);
ServiceOnly classAnnotation = handlerClass.getAnnotation(ServiceOnly.class);
if (methodAnnotation != null || classAnnotation != null) {
String secret = request.getHeader(HEADER_NAME);
if (secret == null || !secret.trim().equals(serviceSecret)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
return false;
}
}
return true;
}
}

View File

@ -0,0 +1,16 @@
package nl.andrewlalis.gymboard_api.config;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation that can be applied to a controller or controller method to
* restrict access to only requests from another service that provide a
* legitimate service secret.
* @see ServiceAccessInterceptor
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ServiceOnly {}

View File

@ -22,9 +22,11 @@ public class WebComponents {
@Value("${app.cdn-origin}")
private String cdnOrigin;
@Value("${app.cdn-secret}")
private String cdnSecret;
@Bean
public CdnClient cdnClient() {
return new CdnClient(cdnOrigin);
return new CdnClient(cdnOrigin, cdnSecret);
}
}

View File

@ -1,8 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.GymService;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;

View File

@ -1,6 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.LeaderboardService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;

View File

@ -1,6 +1,6 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.UserSubmissionService;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;

View File

@ -1,12 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
import java.time.LocalDateTime;
public record SubmissionPayload(
String exerciseShortName,
LocalDateTime performedAt,
float weight,
String weightUnit,
int reps,
String videoFileId
) {}

View File

@ -1,37 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.dto.UserResponse;
import nl.andrewlalis.gymboard_api.util.StandardDateFormatter;
public record SubmissionResponse(
String id,
String createdAt,
GymSimpleResponse gym,
ExerciseResponse exercise,
UserResponse user,
String performedAt,
String videoFileId,
double rawWeight,
String weightUnit,
double metricWeight,
int reps,
boolean verified
) {
public SubmissionResponse(Submission submission) {
this(
submission.getId(),
StandardDateFormatter.format(submission.getCreatedAt()),
new GymSimpleResponse(submission.getGym()),
new ExerciseResponse(submission.getExercise()),
new UserResponse(submission.getUser()),
StandardDateFormatter.format(submission.getPerformedAt()),
submission.getVideoFileId(),
submission.getRawWeight().doubleValue(),
submission.getWeightUnit().name(),
submission.getMetricWeight().doubleValue(),
submission.getReps(),
submission.isVerified()
);
}
}

View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.dto;
public record VideoProcessingCompletePayload(
long taskId,
String status,
String videoFileId,
String thumbnailFileId
) {}

View File

@ -1,137 +0,0 @@
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.hibernate.annotations.CreationTimestamp;
import java.math.BigDecimal;
import java.time.LocalDateTime;
@Entity
@Table(name = "submission")
public class Submission {
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Gym gym;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Exercise exercise;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private User user;
@Column(nullable = false)
private LocalDateTime performedAt;
/**
* The id of the video file that was submitted for this submission. It lives
* on the <em>gymboard-cdn</em> service as a stored file, which can be
* accessed via <code>GET https://CDN-HOST/files/{videoFileId}</code>.
*/
@Column(nullable = false, updatable = false, length = 26)
private String videoFileId;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal rawWeight;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private WeightUnit weightUnit;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal metricWeight;
@Column(nullable = false)
private int reps;
@Column(nullable = false)
private boolean verified;
public Submission() {}
public Submission(
String id,
Gym gym,
Exercise exercise,
User user,
LocalDateTime performedAt,
String videoFileId,
BigDecimal rawWeight,
WeightUnit unit,
BigDecimal metricWeight,
int reps
) {
this.id = id;
this.gym = gym;
this.exercise = exercise;
this.videoFileId = videoFileId;
this.user = user;
this.performedAt = performedAt;
this.rawWeight = rawWeight;
this.weightUnit = unit;
this.metricWeight = metricWeight;
this.reps = reps;
this.verified = false;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Gym getGym() {
return gym;
}
public Exercise getExercise() {
return exercise;
}
public String getVideoFileId() {
return videoFileId;
}
public User getUser() {
return user;
}
public LocalDateTime getPerformedAt() {
return performedAt;
}
public BigDecimal getRawWeight() {
return rawWeight;
}
public WeightUnit getWeightUnit() {
return weightUnit;
}
public BigDecimal getMetricWeight() {
return metricWeight;
}
public int getReps() {
return reps;
}
public boolean isVerified() {
return verified;
}
public void setVerified(boolean verified) {
this.verified = verified;
}
}

View File

@ -1,10 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.api.service;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymResponse;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.util.PredicateBuilder;
import org.slf4j.Logger;
@ -42,13 +42,14 @@ public class GymService {
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return submissionRepository.findAll((root, query, criteriaBuilder) -> {
query.orderBy(
criteriaBuilder.desc(root.get("performedAt")),
criteriaBuilder.desc(root.get("properties").get("performedAt")),
criteriaBuilder.desc(root.get("createdAt"))
);
query.distinct(true);
return PredicateBuilder.and(criteriaBuilder)
.with(criteriaBuilder.equal(root.get("gym"), gym))
.with(criteriaBuilder.isTrue(root.get("verified")))
.with(criteriaBuilder.isFalse(root.get("processing")))
.build();
}, PageRequest.of(0, 5))
.map(SubmissionResponse::new)

View File

@ -1,10 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.api.service;
import nl.andrewlalis.gymboard_api.domains.api.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.LeaderboardTimeframe;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
@ -55,8 +55,10 @@ public class LeaderboardService {
query.distinct(true);
query.orderBy(criteriaBuilder.desc(root.get("metricWeight")));
// Basic predicates that should always hold.
PredicateBuilder pb = PredicateBuilder.and(criteriaBuilder)
.with(criteriaBuilder.isTrue(root.get("verified")));
.with(criteriaBuilder.isTrue(root.get("verified")))
.with(criteriaBuilder.isFalse(root.get("processing")));
cutoffTime.ifPresent(time -> pb.with(criteriaBuilder.greaterThan(root.get("performedAt"), time)));
optionalExercise.ifPresent(exercise -> pb.with(criteriaBuilder.equal(root.get("exercise"), exercise)));

View File

@ -1,6 +1,7 @@
package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboard_api.config.ServiceAccessInterceptor;
import java.io.IOException;
import java.net.URI;
@ -13,23 +14,28 @@ import java.time.Duration;
public class CdnClient {
private final HttpClient httpClient;
private final String baseUrl;
private final String cdnSecret;
private final ObjectMapper objectMapper;
public final UploadsClient uploads;
public final FilesClient files;
public CdnClient(String baseUrl) {
public CdnClient(String baseUrl, String cdnSecret) {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(3))
.followRedirects(HttpClient.Redirect.NORMAL)
.build();
this.baseUrl = baseUrl;
this.cdnSecret = cdnSecret;
this.objectMapper = new ObjectMapper();
this.uploads = new UploadsClient(this);
this.files = new FilesClient(this);
}
public <T> T get(String urlPath, Class<T> responseType) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.GET()
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() == 200) {
@ -46,8 +52,31 @@ public class CdnClient {
.POST(HttpRequest.BodyPublishers.ofFile(filePath))
.header("Content-Type", contentType)
.header("X-Gymboard-Filename", filePath.getFileName().toString())
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
return objectMapper.readValue(response.body(), responseType);
}
public void post(String urlPath) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.POST(HttpRequest.BodyPublishers.noBody())
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<Void> response = httpClient.send(req, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() != 200) {
throw new IOException("Request failed with code " + response.statusCode());
}
}
public void delete(String urlPath) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.DELETE()
.header(ServiceAccessInterceptor.HEADER_NAME, cdnSecret)
.build();
HttpResponse<Void> response = httpClient.send(req, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() >= 400) {
throw new IOException("Request failed with code " + response.statusCode());
}
}
}

View File

@ -0,0 +1,18 @@
package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
public record FilesClient(CdnClient client) {
public record FileMetadataResponse(
String filename,
String mimeType,
long size,
String createdAt
) {}
public FileMetadataResponse getFileMetadata(String id) throws Exception {
return client.get("/files/" + id + "/metadata", FileMetadataResponse.class);
}
public void deleteFile(String id) throws Exception {
client.delete("/files/" + id);
}
}

View File

@ -3,26 +3,22 @@ package nl.andrewlalis.gymboard_api.domains.api.service.cdn_client;
import java.nio.file.Path;
public record UploadsClient(CdnClient client) {
public record FileUploadResponse(String id) {}
public record VideoProcessingTaskStatusResponse(String status) {}
public record FileMetadataResponse(
String filename,
String mimeType,
long size,
String uploadedAt,
boolean availableForDownload
public record FileUploadResponse(long taskId) {}
public record VideoProcessingTaskStatusResponse(
String status,
String videoFileId,
String thumbnailFileId
) {}
public FileUploadResponse uploadVideo(Path filePath, String contentType) throws Exception {
return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class);
public long uploadVideo(Path filePath, String contentType) throws Exception {
return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class).taskId();
}
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) throws Exception {
public VideoProcessingTaskStatusResponse getVideoProcessingTaskStatus(long id) throws Exception {
return client.get("/uploads/video/" + id + "/status", VideoProcessingTaskStatusResponse.class);
}
public FileMetadataResponse getFileMetadata(String id) throws Exception {
return client.get("/files/" + id + "/metadata", FileMetadataResponse.class);
public void startTask(long taskId) throws Exception {
client.post("/uploads/video/" + taskId + "/start");
}
}

View File

@ -2,26 +2,33 @@ package nl.andrewlalis.gymboard_api.domains.api.service.submission;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dto.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionPayload;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.CdnClient;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.UploadsClient;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionProperties;
import nl.andrewlalis.gymboard_api.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.math.BigDecimal;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeParseException;
import java.util.concurrent.TimeUnit;
/**
* Service which handles the rather mundane tasks associated with exercise
@ -80,32 +87,53 @@ public class ExerciseSubmissionService {
}
// Create the submission.
LocalDateTime performedAt = payload.performedAt();
if (performedAt == null) performedAt = LocalDateTime.now();
LocalDateTime performedAt = LocalDateTime.now();
if (payload.performedAt() != null) {
performedAt = LocalDate.parse(payload.performedAt()).atTime(performedAt.toLocalTime());
}
BigDecimal rawWeight = BigDecimal.valueOf(payload.weight());
WeightUnit weightUnit = WeightUnit.parse(payload.weightUnit());
BigDecimal metricWeight = BigDecimal.valueOf(payload.weight());
if (weightUnit == WeightUnit.POUNDS) {
metricWeight = WeightUnit.toKilograms(rawWeight);
}
Submission submission = submissionRepository.saveAndFlush(new Submission(
ulid.nextULID(), gym, exercise, user,
SubmissionProperties properties = new SubmissionProperties(
exercise,
performedAt,
payload.videoFileId(),
rawWeight, weightUnit, metricWeight, payload.reps()
));
rawWeight,
weightUnit,
payload.reps()
);
Submission submission = new Submission(ulid.nextULID(), gym, user, payload.taskId(), properties);
try {
cdnClient.uploads.startTask(submission.getVideoProcessingTaskId());
submission.setProcessing(true);
} catch (Exception e) {
log.error("Failed to start video processing task for submission.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to start video processing.");
}
submission = submissionRepository.saveAndFlush(submission);
return new SubmissionResponse(submission);
}
private ValidationResponse validateSubmissionData(Gym gym, User user, Exercise exercise, SubmissionPayload data) {
ValidationResponse response = new ValidationResponse();
LocalDateTime now = LocalDateTime.now();
LocalDateTime cutoff = LocalDateTime.now().minusDays(3);
if (data.performedAt() != null && data.performedAt().isAfter(LocalDateTime.now())) {
if (data.performedAt() != null) {
try {
LocalDateTime performedAt = LocalDate.parse(data.performedAt()).atTime(now.toLocalTime());
if (performedAt.isAfter(now)) {
response.addMessage("Cannot submit an exercise from the future.");
}
if (data.performedAt() != null && data.performedAt().isBefore(cutoff)) {
if (performedAt.isBefore(cutoff)) {
response.addMessage("Cannot submit an exercise too far in the past.");
}
} catch (DateTimeParseException e) {
response.addMessage("Invalid performedAt format.");
}
}
if (data.reps() < 1 || data.reps() > 500) {
response.addMessage("Invalid rep count.");
}
@ -118,17 +146,13 @@ public class ExerciseSubmissionService {
}
try {
UploadsClient.FileMetadataResponse metadata = cdnClient.uploads.getFileMetadata(data.videoFileId());
if (metadata == null) {
response.addMessage("Missing video file.");
} else if (!metadata.availableForDownload()) {
response.addMessage("File not yet available for download.");
} else if (!"video/mp4".equals(metadata.mimeType())) {
response.addMessage("Invalid video file format.");
var status = cdnClient.uploads.getVideoProcessingTaskStatus(data.taskId());
if (status == null || !status.status().equalsIgnoreCase("NOT_STARTED")) {
response.addMessage("Invalid video processing task.");
}
} catch (Exception e) {
log.error("Error fetching file metadata.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error fetching uploaded video file metadata.");
log.error("Error fetching task status.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Error fetching uploaded video task status.");
}
return response;
}
@ -140,7 +164,133 @@ public class ExerciseSubmissionService {
if (!submission.getUser().getId().equals(user.getId())) {
throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Cannot delete other user's submission.");
}
// TODO: Find a secure way to delete the associated video.
try {
if (submission.getVideoFileId() != null) {
cdnClient.files.deleteFile(submission.getVideoFileId());
}
if (submission.getThumbnailFileId() != null) {
cdnClient.files.deleteFile(submission.getThumbnailFileId());
}
} catch (Exception e) {
log.error("Couldn't delete CDN content for submission " + submissionId, e);
}
submissionRepository.delete(submission);
}
/**
* This method is invoked when the CDN calls this API's endpoint to notify
* us that a video processing task has completed. If the task completed
* successfully, we can set any related submissions' video and thumbnail
* file ids and remove its "processing" flag. Otherwise, we should delete
* the failed submission.
* @param payload The information about the task.
*/
@Transactional
public void handleVideoProcessingComplete(VideoProcessingCompletePayload payload) {
var submissionsToUpdate = submissionRepository.findUnprocessedByTaskId(payload.taskId());
log.info("Received video processing complete message from CDN: {}, affecting {} submissions.", payload, submissionsToUpdate.size());
for (var submission : submissionsToUpdate) {
if (payload.status().equalsIgnoreCase("COMPLETED")) {
submission.setVideoFileId(payload.videoFileId());
submission.setThumbnailFileId(payload.thumbnailFileId());
submission.setProcessing(false);
submissionRepository.save(submission);
// TODO: Send notification of successful processing to the user!
} else if (payload.status().equalsIgnoreCase("FAILED")) {
submissionRepository.delete(submission);
// TODO: Send notification of failed video processing to the user!
}
}
}
/**
* A scheduled task that checks and resolves issues with any submission that
* stays in the "processing" state for too long.
* TODO: Find some way to clean up this mess of logic!
*/
@Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
public void checkProcessingSubmissions() {
var processingSubmissions = submissionRepository.findAllByProcessingTrue();
LocalDateTime actionCutoff = LocalDateTime.now().minus(Duration.ofMinutes(3));
LocalDateTime deleteCutoff = LocalDateTime.now().minus(Duration.ofMinutes(30));
for (var submission : processingSubmissions) {
if (submission.getCreatedAt().isBefore(actionCutoff)) {
// Sanity check to remove any inconsistent submission that doesn't have a task id for whatever reason.
if (submission.getVideoProcessingTaskId() == null) {
log.warn(
"Removing long-processing submission {} for user {} because it doesn't have a task id.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
continue;
}
try {
var status = cdnClient.uploads.getVideoProcessingTaskStatus(submission.getVideoProcessingTaskId());
if (status == null) {
// The task no longer exists on the CDN, so remove the submission.
log.warn(
"Removing long-processing submission {} for user {} because its task no longer exists on the CDN.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("FAILED")) {
// The task failed, so we should remove the submission.
log.warn(
"Removing long-processing submission {} for user {} because its task failed.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("COMPLETED")) {
// The submission should be marked as complete.
submission.setVideoFileId(status.videoFileId());
submission.setThumbnailFileId(status.thumbnailFileId());
submission.setProcessing(false);
submissionRepository.save(submission);
// TODO: Send notification to user.
} else if (status.status().equalsIgnoreCase("NOT_STARTED")) {
// If for whatever reason the submission's video processing never started, start now.
try {
cdnClient.uploads.startTask(submission.getVideoProcessingTaskId());
} catch (Exception e) {
log.error("Failed to start processing task " + submission.getVideoProcessingTaskId(), e);
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
} else {
// The task is waiting or processing, so delete the submission if it's been in that state for an unreasonably long time.
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
} catch (Exception e) {
log.error("Couldn't fetch status of long-processing submission " + submission.getId() + " for user " + submission.getUser().getEmail(), e);
// We can't reliably remove this submission yet, so we'll try again on the next pass.
if (submission.getCreatedAt().isBefore(deleteCutoff)) {
log.warn(
"Removing long-processing submission {} for user {} because it is waiting or processing for too long.",
submission.getId(), submission.getUser().getEmail()
);
submissionRepository.delete(submission);
// TODO: Send notification to user.
}
}
}
}
}
}

View File

@ -1,7 +1,7 @@
package nl.andrewlalis.gymboard_api.domains.api.service.submission;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.auth.service.UserAccessService;
@ -9,10 +9,8 @@ import nl.andrewlalis.gymboard_api.util.PredicateBuilder;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.util.List;

View File

@ -1,10 +1,15 @@
package nl.andrewlalis.gymboard_api.domains.auth.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.auth.model.UserAccountDataRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface UserAccountDataRequestRepository extends JpaRepository<UserAccountDataRequest, Long> {
boolean existsByUserIdAndFulfilledFalse(String userId);
@Modifying
void deleteAllByUser(User user);
}

View File

@ -116,10 +116,15 @@ public class TokenService {
public Jws<Claims> getToken(String token) {
if (token == null) return null;
try {
var builder = Jwts.parserBuilder()
.setSigningKey(this.getPrivateKey())
.requireIssuer(ISSUER);
return builder.build().parseClaimsJws(token);
} catch (Exception e) {
log.warn("Error parsing JWT.", e);
return null;
}
}
private PrivateKey getPrivateKey() {

View File

@ -1,8 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.auth.service;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionReportRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.auth.dao.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionVoteRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
@ -19,6 +21,9 @@ public class UserAccountDeletionService {
private final EmailResetCodeRepository emailResetCodeRepository;
private final PasswordResetCodeRepository passwordResetCodeRepository;
private final SubmissionRepository submissionRepository;
private final SubmissionReportRepository submissionReportRepository;
private final SubmissionVoteRepository submissionVoteRepository;
private final UserAccountDataRequestRepository accountDataRequestRepository;
public UserAccountDeletionService(UserRepository userRepository,
UserReportRepository userReportRepository,
@ -26,7 +31,10 @@ public class UserAccountDeletionService {
UserActivationCodeRepository userActivationCodeRepository,
EmailResetCodeRepository emailResetCodeRepository,
PasswordResetCodeRepository passwordResetCodeRepository,
SubmissionRepository submissionRepository) {
SubmissionRepository submissionRepository,
SubmissionReportRepository submissionReportRepository,
SubmissionVoteRepository submissionVoteRepository,
UserAccountDataRequestRepository accountDataRequestRepository) {
this.userRepository = userRepository;
this.userReportRepository = userReportRepository;
this.userFollowingRepository = userFollowingRepository;
@ -34,6 +42,9 @@ public class UserAccountDeletionService {
this.emailResetCodeRepository = emailResetCodeRepository;
this.passwordResetCodeRepository = passwordResetCodeRepository;
this.submissionRepository = submissionRepository;
this.submissionReportRepository = submissionReportRepository;
this.submissionVoteRepository = submissionVoteRepository;
this.accountDataRequestRepository = accountDataRequestRepository;
}
@Transactional
@ -46,6 +57,9 @@ public class UserAccountDeletionService {
userReportRepository.deleteAllByUserOrReportedBy(user, user);
userFollowingRepository.deleteAllByFollowedUserOrFollowingUser(user, user);
submissionRepository.deleteAllByUser(user);
submissionReportRepository.deleteAllByUser(user);
submissionVoteRepository.deleteAllByUser(user);
accountDataRequestRepository.deleteAllByUser(user);
userRepository.deleteById(user.getId());
}
}

View File

@ -1,6 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.controller;
package nl.andrewlalis.gymboard_api.domains.submission.controller;
import nl.andrewlalis.gymboard_api.domains.api.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.config.ServiceOnly;
import nl.andrewlalis.gymboard_api.domains.submission.dto.SubmissionResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.VideoProcessingCompletePayload;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.springframework.http.ResponseEntity;
@ -26,4 +28,10 @@ public class SubmissionController {
submissionService.deleteSubmission(submissionId, user);
return ResponseEntity.noContent().build();
}
@PostMapping(path = "/video-processing-complete") @ServiceOnly
public ResponseEntity<Void> handleVideoProcessingComplete(@RequestBody VideoProcessingCompletePayload taskStatus) {
submissionService.handleVideoProcessingComplete(taskStatus);
return ResponseEntity.noContent().build();
}
}

View File

@ -0,0 +1,13 @@
package nl.andrewlalis.gymboard_api.domains.submission.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionReport;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface SubmissionReportRepository extends JpaRepository<SubmissionReport, Long> {
@Modifying
void deleteAllByUser(User user);
}

View File

@ -1,14 +1,24 @@
package nl.andrewlalis.gymboard_api.domains.api.dao.submission;
package nl.andrewlalis.gymboard_api.domains.submission.dao;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface SubmissionRepository extends JpaRepository<Submission, String>, JpaSpecificationExecutor<Submission> {
@Modifying
void deleteAllByUser(User user);
@Query("SELECT s FROM Submission s " +
"WHERE s.videoProcessingTaskId = :taskId AND " +
"s.processing = TRUE")
List<Submission> findUnprocessedByTaskId(long taskId);
List<Submission> findAllByProcessingTrue();
}

View File

@ -0,0 +1,13 @@
package nl.andrewlalis.gymboard_api.domains.submission.dao;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionVote;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.stereotype.Repository;
@Repository
public interface SubmissionVoteRepository extends JpaRepository<SubmissionVote, Long> {
@Modifying
void deleteAllByUser(User user);
}

View File

@ -0,0 +1,10 @@
package nl.andrewlalis.gymboard_api.domains.submission.dto;
public record SubmissionPayload(
String exerciseShortName,
String performedAt,
float weight,
String weightUnit,
int reps,
long taskId
) {}

View File

@ -0,0 +1,48 @@
package nl.andrewlalis.gymboard_api.domains.submission.dto;
import nl.andrewlalis.gymboard_api.domains.api.dto.ExerciseResponse;
import nl.andrewlalis.gymboard_api.domains.api.dto.GymSimpleResponse;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.auth.dto.UserResponse;
import nl.andrewlalis.gymboard_api.util.StandardDateFormatter;
public record SubmissionResponse(
String id,
String createdAt,
GymSimpleResponse gym,
UserResponse user,
long videoProcessingTaskId,
String videoFileId,
String thumbnailFileId,
boolean processing,
boolean verified,
// From SubmissionProperties
ExerciseResponse exercise,
String performedAt,
double rawWeight,
String weightUnit,
double metricWeight,
int reps
) {
public SubmissionResponse(Submission submission) {
this(
submission.getId(),
StandardDateFormatter.format(submission.getCreatedAt()),
new GymSimpleResponse(submission.getGym()),
new UserResponse(submission.getUser()),
submission.getVideoProcessingTaskId(),
submission.getVideoFileId(),
submission.getThumbnailFileId(),
submission.isProcessing(),
submission.isVerified(),
new ExerciseResponse(submission.getProperties().getExercise()),
StandardDateFormatter.format(submission.getCreatedAt()),
submission.getProperties().getRawWeight().doubleValue(),
submission.getProperties().getWeightUnit().name(),
submission.getProperties().getMetricWeight().doubleValue(),
submission.getProperties().getReps()
);
}
}

View File

@ -0,0 +1,159 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* The Submission entity represents a user's posted video of a lift they did at
* a gym.
* <p>
* A submission is created in the front-end using the following flow:
* </p>
* <ol>
* <li>User uploads a raw video of their lift.</li>
* <li>User enters some basic information about the lift.</li>
* <li>User submits the lift.</li>
* <li>API validates the information.</li>
* <li>API creates a new Submission, and tells the CDN service to process
* the uploaded video.</li>
* <li>Once processing completes successfully, the CDN sends the final video
* and thumbnail file ids to the API and the Submission's "processing" flag
* is removed.</li>
* <li>If for whatever reason the CDN's video processing fails or never
* completes, the Submission is deleted and the user is notified of the
* issue.</li>
* </ol>
*/
@Entity
@Table(name = "submission")
public class Submission {
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Gym gym;
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private User user;
/**
* The id of the video processing task that a user gives to us when they
* create the submission, so that when the task finishes processing, we can
* route its data to the right submission.
*/
@Column
private Long videoProcessingTaskId;
/**
* The id of the video file that was submitted for this submission. It lives
* on the <em>gymboard-cdn</em> service as a stored file, which can be
* accessed via <code>GET https://CDN-HOST/files/{videoFileId}</code>.
*/
@Column(length = 26)
private String videoFileId = null;
/**
* The id of the thumbnail file that was generated for this submission.
* Similarly to the video file id, it refers to a file managed by the CDN.
*/
@Column(length = 26)
private String thumbnailFileId = null;
/**
* The user-specified properties of the submission.
*/
@Embedded
private SubmissionProperties properties;
/**
* A flag that indicates whether this submission is currently processing.
* A submission is processing until its associated processing task completes
* either successfully or unsuccessfully.
*/
@Column(nullable = false)
private boolean processing;
@Column(nullable = false)
private boolean verified;
public Submission() {}
public Submission(
String id,
Gym gym,
User user,
long videoProcessingTaskId,
SubmissionProperties properties
) {
this.id = id;
this.gym = gym;
this.user = user;
this.videoProcessingTaskId = videoProcessingTaskId;
this.properties = properties;
this.verified = false;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Gym getGym() {
return gym;
}
public Long getVideoProcessingTaskId() {
return videoProcessingTaskId;
}
public String getVideoFileId() {
return videoFileId;
}
public String getThumbnailFileId() {
return thumbnailFileId;
}
public void setVideoFileId(String videoFileId) {
this.videoFileId = videoFileId;
}
public void setThumbnailFileId(String thumbnailFileId) {
this.thumbnailFileId = thumbnailFileId;
}
public User getUser() {
return user;
}
public SubmissionProperties getProperties() {
return properties;
}
public boolean isProcessing() {
return processing;
}
public void setProcessing(boolean processing) {
this.processing = processing;
}
public boolean isVerified() {
return verified;
}
public void setVerified(boolean verified) {
this.verified = verified;
}
}

View File

@ -0,0 +1,74 @@
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import java.math.BigDecimal;
import java.time.LocalDateTime;
/**
* Basic user-specified properties about a Submission.
*/
@Embeddable
public class SubmissionProperties {
@ManyToOne(optional = false, fetch = FetchType.LAZY)
private Exercise exercise;
@Column(nullable = false)
private LocalDateTime performedAt;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal rawWeight;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private WeightUnit weightUnit;
@Column(nullable = false, precision = 7, scale = 2)
private BigDecimal metricWeight;
@Column(nullable = false)
private int reps;
public SubmissionProperties() {}
public SubmissionProperties(
Exercise exercise,
LocalDateTime performedAt,
BigDecimal rawWeight,
WeightUnit weightUnit,
int reps
) {
this.exercise = exercise;
this.performedAt = performedAt;
this.rawWeight = rawWeight;
this.weightUnit = weightUnit;
this.metricWeight = WeightUnit.toKilograms(rawWeight, weightUnit);
this.reps = reps;
}
public Exercise getExercise() {
return exercise;
}
public LocalDateTime getPerformedAt() {
return performedAt;
}
public BigDecimal getRawWeight() {
return rawWeight;
}
public WeightUnit getWeightUnit() {
return weightUnit;
}
public BigDecimal getMetricWeight() {
return metricWeight;
}
public int getReps() {
return reps;
}
}

View File

@ -1,7 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;

View File

@ -1,7 +1,8 @@
package nl.andrewlalis.gymboard_api.domains.api.model.submission;
package nl.andrewlalis.gymboard_api.domains.submission.model;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
@Entity
@Table(

View File

@ -56,7 +56,7 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
gen.generate();
completed.add(gen);
} catch (Exception e) {
throw new RuntimeException("Generator failed: " + gen.getClass().getSimpleName());
throw new RuntimeException("Generator failed: " + gen.getClass().getSimpleName(), e);
}
}
}

View File

@ -1,21 +1,25 @@
package nl.andrewlalis.gymboard_api.util.sample_data;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.ExerciseRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.submission.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.domains.submission.dao.SubmissionRepository;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.Gym;
import nl.andrewlalis.gymboard_api.domains.api.model.WeightUnit;
import nl.andrewlalis.gymboard_api.domains.api.model.Exercise;
import nl.andrewlalis.gymboard_api.domains.api.model.submission.Submission;
import nl.andrewlalis.gymboard_api.domains.submission.model.Submission;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.CdnClient;
import nl.andrewlalis.gymboard_api.domains.api.service.submission.ExerciseSubmissionService;
import nl.andrewlalis.gymboard_api.domains.api.service.cdn_client.UploadsClient;
import nl.andrewlalis.gymboard_api.domains.auth.dao.UserRepository;
import nl.andrewlalis.gymboard_api.domains.auth.model.User;
import nl.andrewlalis.gymboard_api.domains.submission.model.SubmissionProperties;
import nl.andrewlalis.gymboard_api.util.ULID;
import org.springframework.beans.factory.annotation.Value;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Profile;
import org.springframework.data.util.Pair;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Path;
import java.time.Duration;
@ -25,35 +29,31 @@ import java.util.*;
@Component
@Profile("development")
public class SampleSubmissionGenerator implements SampleDataGenerator {
private static final Logger log = LoggerFactory.getLogger(SampleSubmissionGenerator.class);
private final GymRepository gymRepository;
private final UserRepository userRepository;
private final ExerciseRepository exerciseRepository;
private final ExerciseSubmissionService submissionService;
private final SubmissionRepository submissionRepository;
private final ULID ulid;
private final CdnClient cdnClient;
@Value("${app.cdn-origin}")
private String cdnOrigin;
public SampleSubmissionGenerator(GymRepository gymRepository, UserRepository userRepository, ExerciseRepository exerciseRepository, ExerciseSubmissionService submissionService, SubmissionRepository submissionRepository, ULID ulid) {
public SampleSubmissionGenerator(GymRepository gymRepository, UserRepository userRepository, ExerciseRepository exerciseRepository, SubmissionRepository submissionRepository, ULID ulid, CdnClient cdnClient) {
this.gymRepository = gymRepository;
this.userRepository = userRepository;
this.exerciseRepository = exerciseRepository;
this.submissionService = submissionService;
this.submissionRepository = submissionRepository;
this.ulid = ulid;
this.cdnClient = cdnClient;
}
@Override
public void generate() throws Exception {
final CdnClient cdnClient = new CdnClient(cdnOrigin);
List<String> videoIds = new ArrayList<>();
var video1 = cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_curl.mp4"), "video/mp4");
var video2 = cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_ohp.mp4"), "video/mp4");
videoIds.add(video1.id());
videoIds.add(video2.id());
// First we generate a small set of uploaded files that all the
// submissions can link to, instead of having them all upload new content.
var uploads = generateUploads();
// Now that uploads are complete, we can proceed with generating the submissions.
List<Gym> gyms = gymRepository.findAll();
List<User> users = userRepository.findAll();
List<Exercise> exercises = exerciseRepository.findAll();
@ -65,15 +65,16 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
Random random = new Random(1);
List<Submission> submissions = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
submissions.add(generateRandomSubmission(
Submission submission = generateRandomSubmission(
gyms,
users,
exercises,
videoIds,
uploads,
earliestSubmission,
latestSubmission,
random
));
);
submissions.add(submission);
}
submissionRepository.saveAll(submissions);
}
@ -82,7 +83,7 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
List<Gym> gyms,
List<User> users,
List<Exercise> exercises,
List<String> videoIds,
Map<Long, Pair<String, String>> uploads,
LocalDateTime earliestSubmission,
LocalDateTime latestSubmission,
Random random
@ -95,20 +96,26 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
weightUnit = WeightUnit.POUNDS;
rawWeight = metricWeight.multiply(new BigDecimal("2.2046226218"));
}
SubmissionProperties properties = new SubmissionProperties(
randomChoice(exercises, random),
time,
rawWeight,
weightUnit,
random.nextInt(13) + 1
);
var submission = new Submission(
ulid.nextULID(),
randomChoice(gyms, random),
randomChoice(exercises, random),
randomChoice(users, random),
time,
randomChoice(videoIds, random),
rawWeight,
weightUnit,
metricWeight,
random.nextInt(13) + 1
randomChoice(new ArrayList<>(uploads.keySet()), random),
properties
);
submission.setVerified(true);
submission.setProcessing(false);
var uploadData = uploads.get(submission.getVideoProcessingTaskId());
submission.setVideoFileId(uploadData.getFirst());
submission.setThumbnailFileId(uploadData.getSecond());
return submission;
}
@ -125,4 +132,46 @@ public class SampleSubmissionGenerator implements SampleDataGenerator {
Duration dur = Duration.between(start, end);
return start.plusSeconds(rand.nextLong(dur.toSeconds() + 1));
}
/**
* Generates a set of sample video uploads to use for all the sample
* submissions.
* @return A map containing keys representing video processing task ids, and
* values being a pair of video and thumbnail file ids.
* @throws Exception If an error occurs.
*/
private Map<Long, Pair<String, String>> generateUploads() throws Exception {
List<Long> taskIds = new ArrayList<>();
taskIds.add(cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_curl.mp4"), "video/mp4"));
taskIds.add(cdnClient.uploads.uploadVideo(Path.of("sample_data", "sample_video_ohp.mp4"), "video/mp4"));
Map<Long, UploadsClient.VideoProcessingTaskStatusResponse> taskStatus = new HashMap<>();
for (long taskId : taskIds) {
cdnClient.uploads.startTask(taskId);
taskStatus.put(taskId, cdnClient.uploads.getVideoProcessingTaskStatus(taskId));
}
// Wait for all video uploads to complete.
while (
taskStatus.values().stream()
.map(UploadsClient.VideoProcessingTaskStatusResponse::status)
.anyMatch(status -> !List.of("COMPLETED", "FAILED").contains(status.toUpperCase()))
) {
log.info("Waiting for sample video upload tasks to finish...");
Thread.sleep(1000);
for (long taskId : taskIds) taskStatus.put(taskId, cdnClient.uploads.getVideoProcessingTaskStatus(taskId));
}
// If any upload failed, throw an exception and cancel this generator.
if (taskStatus.values().stream().anyMatch(r -> r.status().equalsIgnoreCase("FAILED"))) {
throw new IOException("Video upload task processing failed.");
}
// Prepare the final data structure.
Map<Long, Pair<String, String>> finalResults = new HashMap<>();
for (var entry : taskStatus.entrySet()) {
finalResults.put(entry.getKey(), Pair.of(entry.getValue().videoFileId(), entry.getValue().thumbnailFileId()));
}
return finalResults;
}
}

View File

@ -15,7 +15,9 @@ spring.mail.protocol=smtp
spring.mail.properties.mail.smtp.timeout=10000
app.auth.private-key-location=./private_key.der
app.service-secret=testing
app.web-origin=http://localhost:9000
app.cdn-origin=http://localhost:8082
app.cdn-secret=testing
#logging.level.root=DEBUG

View File

@ -8,6 +8,7 @@ const api = axios.create({
});
export enum VideoProcessingStatus {
NOT_STARTED = 'NOT_STARTED',
WAITING = 'WAITING',
IN_PROGRESS = 'IN_PROGRESS',
COMPLETED = 'COMPLETED',
@ -18,24 +19,21 @@ export interface FileMetadata {
filename: string;
mimeType: string;
size: number;
uploadedAt: string;
availableForDownload: boolean;
createdAt: string;
}
export async function uploadVideoToCDN(file: File): Promise<string> {
export async function uploadVideoToCDN(file: File): Promise<number> {
const response = await api.post('/uploads/video', file, {
headers: {
'Content-Type': file.type,
},
});
return response.data.id;
return response.data.taskId;
}
export async function getVideoProcessingStatus(
id: string
): Promise<VideoProcessingStatus | null> {
export async function getVideoProcessingStatus(taskId: number): Promise<VideoProcessingStatus | null> {
try {
const response = await api.get(`/uploads/video/${id}/status`);
const response = await api.get(`/uploads/video/${taskId}/status`);
return response.data.status;
} catch (error: any) {
if (error.response && error.response.status === 404) {
@ -45,16 +43,14 @@ export async function getVideoProcessingStatus(
}
}
export async function waitUntilVideoProcessingComplete(
id: string
): Promise<VideoProcessingStatus> {
export async function waitUntilVideoProcessingComplete(taskId: number): Promise<VideoProcessingStatus> {
let failureCount = 0;
let attemptCount = 0;
while (failureCount < 5 && attemptCount < 60) {
await sleep(1000);
attemptCount++;
try {
const status = await getVideoProcessingStatus(id);
const status = await getVideoProcessingStatus(taskId);
failureCount = 0;
if (
status === VideoProcessingStatus.COMPLETED ||

View File

@ -1,5 +1,5 @@
import { GeoPoint } from 'src/api/main/models';
import SubmissionsModule, { ExerciseSubmission, parseSubmission } from 'src/api/main/submission';
import SubmissionsModule, { Submission, parseSubmission } from 'src/api/main/submission';
import { api } from 'src/api/main/index';
import { GymRoutable } from 'src/router/gym-routing';
@ -51,7 +51,7 @@ class GymsModule {
public async getRecentSubmissions(
gym: GymRoutable
): Promise<Array<ExerciseSubmission>> {
): Promise<Array<Submission>> {
const response = await api.get(
`/gyms/${gym.countryCode}_${gym.cityShortName}_${gym.shortName}/recent-submissions`
);

View File

@ -1,4 +1,4 @@
import { ExerciseSubmission, parseSubmission } from 'src/api/main/submission';
import { Submission, parseSubmission } from 'src/api/main/submission';
import { getGymCompoundId, GymRoutable } from 'src/router/gym-routing';
import { api } from 'src/api/main/index';
@ -29,7 +29,7 @@ interface RequestParams {
class LeaderboardsModule {
public async getLeaderboard(
params: LeaderboardParams
): Promise<Array<ExerciseSubmission>> {
): Promise<Array<Submission>> {
const requestParams: RequestParams = {};
if (params.exerciseShortName) {
requestParams.exercise = params.exerciseShortName;

View File

@ -14,7 +14,7 @@ export interface ExerciseSubmissionPayload {
weight: number;
weightUnit: string;
reps: number;
videoFileId: string;
taskId: number;
}
export enum WeightUnit {
@ -29,30 +29,35 @@ export class WeightUnitUtil {
}
}
export interface ExerciseSubmission {
export interface Submission {
id: string;
createdAt: DateTime;
gym: SimpleGym;
exercise: Exercise;
user: User;
videoFileId: string | null;
thumbnailFileId: string | null;
processing: boolean;
verified: boolean;
exercise: Exercise;
performedAt: DateTime;
videoFileId: string;
rawWeight: number;
weightUnit: WeightUnit;
metricWeight: number;
reps: number;
}
export function parseSubmission(data: any): ExerciseSubmission {
export function parseSubmission(data: any): Submission {
data.createdAt = DateTime.fromISO(data.createdAt);
data.performedAt = DateTime.fromISO(data.performedAt);
return data as ExerciseSubmission;
return data as Submission;
}
class SubmissionsModule {
public async getSubmission(
submissionId: string
): Promise<ExerciseSubmission> {
): Promise<Submission> {
const response = await api.get(`/submissions/${submissionId}`);
return parseSubmission(response.data);
}
@ -61,7 +66,7 @@ class SubmissionsModule {
gym: GymRoutable,
payload: ExerciseSubmissionPayload,
authStore: AuthStoreType
): Promise<ExerciseSubmission> {
): Promise<Submission> {
const gymId = getGymCompoundId(gym);
const response = await api.post(`/gyms/${gymId}/submissions`, payload, authStore.axiosConfig);
return parseSubmission(response.data);

View File

@ -1,15 +1,15 @@
import {api} from 'src/api/main';
import {AuthStoreType} from 'stores/auth-store';
import {ExerciseSubmission, parseSubmission} from 'src/api/main/submission';
import {Submission, parseSubmission} from 'src/api/main/submission';
import {defaultPaginationOptions, Page, PaginationOptions, toQueryParams} from 'src/api/main/models';
class UsersModule {
public async getRecentSubmissions(userId: string, authStore: AuthStoreType): Promise<Array<ExerciseSubmission>> {
public async getRecentSubmissions(userId: string, authStore: AuthStoreType): Promise<Array<Submission>> {
const response = await api.get(`/users/${userId}/recent-submissions`, authStore.axiosConfig);
return response.data.map(parseSubmission);
}
public async getSubmissions(userId: string, authStore: AuthStoreType, paginationOptions: PaginationOptions = defaultPaginationOptions()): Promise<Page<ExerciseSubmission>> {
public async getSubmissions(userId: string, authStore: AuthStoreType, paginationOptions: PaginationOptions = defaultPaginationOptions()): Promise<Page<Submission>> {
const config = structuredClone(authStore.axiosConfig);
config.params = toQueryParams(paginationOptions);
const response = await api.get(`/users/${userId}/submissions`, config);

View File

@ -19,11 +19,11 @@
</template>
<script setup lang="ts">
import { ExerciseSubmission, WeightUnitUtil } from 'src/api/main/submission';
import { Submission, WeightUnitUtil } from 'src/api/main/submission';
import { DateTime } from 'luxon';
interface Props {
submission: ExerciseSubmission;
submission: Submission;
showName?: boolean;
showGym?: boolean;
}

View File

@ -51,8 +51,8 @@ export default {
upload: 'Video File to Upload',
submit: 'Submit',
submitUploading: 'Uploading video...',
submitUploadFailed: 'Video upload failed.',
submitCreatingSubmission: 'Creating submission...',
submitVideoProcessing: 'Processing...',
submitComplete: 'Submission complete!',
submitFailed: 'Submission processing failed. Please try again later.',
},

View File

@ -2,6 +2,7 @@
<q-page>
<StandardCenteredPage v-if="submission">
<video
v-if="!submission.processing"
class="submission-video"
:src="getFileUrl(submission.videoFileId)"
loop
@ -10,6 +11,9 @@
preload="metadata"
autoplay
/>
<div v-if="submission.processing">
<p>This submission is still processing.</p>
</div>
<h3>
{{ submission.rawWeight }}&nbsp;{{ WeightUnitUtil.toAbbreviation(submission.weightUnit) }}
{{ submission.exercise.displayName }}
@ -23,7 +27,7 @@
<!-- Deletion button is only visible if the user who submitted it is viewing it. -->
<q-btn
v-if="authStore.user && authStore.user.id === submission.user.id"
v-if="authStore.user && authStore.user.id === submission.user.id && !submission.processing"
label="Delete"
@click="deleteSubmission"
/>
@ -34,18 +38,18 @@
<script setup lang="ts">
import api from 'src/api/main';
import StandardCenteredPage from 'src/components/StandardCenteredPage.vue';
import { ExerciseSubmission, WeightUnitUtil } from 'src/api/main/submission';
import { Submission, WeightUnitUtil } from 'src/api/main/submission';
import { onMounted, ref, Ref } from 'vue';
import { useRoute, useRouter } from 'vue-router';
import { DateTime } from 'luxon';
import { getFileUrl } from 'src/api/cdn';
import { getGymRoute } from 'src/router/gym-routing';
import {useAuthStore} from 'stores/auth-store';
import {confirm, showApiErrorToast} from 'src/utils';
import {confirm, showApiErrorToast, showInfoToast} from 'src/utils';
import {useI18n} from 'vue-i18n';
import {useQuasar} from 'quasar';
const submission: Ref<ExerciseSubmission | undefined> = ref();
const submission: Ref<Submission | undefined> = ref();
const route = useRoute();
const router = useRouter();
@ -54,14 +58,24 @@ const i18n = useI18n();
const quasar = useQuasar();
onMounted(async () => {
await loadSubmission();
if (submission.value?.processing) {
showInfoToast('This submission is still processing.');
}
});
async function loadSubmission() {
const submissionId = route.params.submissionId as string;
try {
submission.value = await api.gyms.submissions.getSubmission(submissionId);
if (submission.value.processing) {
setTimeout(loadSubmission, 3000);
}
} catch (error) {
console.error(error);
showApiErrorToast(error);
await router.push('/');
}
});
}
/**
* Shows a confirmation dialog asking the user if they really want to delete

View File

@ -43,7 +43,7 @@
<script setup lang="ts">
import { nextTick, onMounted, ref, Ref } from 'vue';
import { ExerciseSubmission } from 'src/api/main/submission';
import { Submission } from 'src/api/main/submission';
import api from 'src/api/main';
import { getGymFromRoute } from 'src/router/gym-routing';
import ExerciseSubmissionListItem from 'components/ExerciseSubmissionListItem.vue';
@ -51,7 +51,7 @@ import { Gym } from 'src/api/main/gyms';
import 'leaflet/dist/leaflet.css';
import { Map, Marker, TileLayer } from 'leaflet';
const recentSubmissions: Ref<Array<ExerciseSubmission>> = ref([]);
const recentSubmissions: Ref<Array<Submission>> = ref([]);
const gym: Ref<Gym | undefined> = ref();
const TILE_URL = 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png';

View File

@ -33,13 +33,13 @@ import api from 'src/api/main';
import { Exercise } from 'src/api/main/exercises';
import { Gym } from 'src/api/main/gyms';
import { LeaderboardTimeframe } from 'src/api/main/leaderboards';
import { ExerciseSubmission } from 'src/api/main/submission';
import { Submission } from 'src/api/main/submission';
import ExerciseSubmissionListItem from 'src/components/ExerciseSubmissionListItem.vue';
import { getGymFromRoute } from 'src/router/gym-routing';
import { sleep } from 'src/utils';
import { onMounted, ref, Ref, watch, computed } from 'vue';
const submissions: Ref<Array<ExerciseSubmission>> = ref([]);
const submissions: Ref<Array<Submission>> = ref([]);
const gym: Ref<Gym | undefined> = ref();
const exercises: Ref<Array<Exercise>> = ref([]);

View File

@ -48,7 +48,7 @@ A high-level overview of the submission process is as follows:
</div>
<div class="row">
<q-input
v-model="submissionModel.date"
v-model="submissionModel.performedAt"
type="date"
:label="$t('gymPage.submitPage.date')"
class="col-12"
@ -91,28 +91,22 @@ A high-level overview of the submission process is as follows:
</template>
<script setup lang="ts">
import { onMounted, ref, Ref } from 'vue';
import { getGymFromRoute } from 'src/router/gym-routing';
import {onMounted, ref, Ref} from 'vue';
import {getGymFromRoute} from 'src/router/gym-routing';
import SlimForm from 'components/SlimForm.vue';
import api from 'src/api/main';
import { Gym } from 'src/api/main/gyms';
import { Exercise } from 'src/api/main/exercises';
import { useRoute, useRouter } from 'vue-router';
import { showApiErrorToast, sleep } from 'src/utils';
import {
uploadVideoToCDN,
VideoProcessingStatus,
waitUntilVideoProcessingComplete,
} from 'src/api/cdn';
import { useAuthStore } from 'stores/auth-store';
import { useI18n } from 'vue-i18n';
import { useQuasar } from 'quasar';
import {Gym} from 'src/api/main/gyms';
import {Exercise} from 'src/api/main/exercises';
import {useRoute, useRouter} from 'vue-router';
import {showApiErrorToast, showWarningToast, sleep} from 'src/utils';
import {uploadVideoToCDN,} from 'src/api/cdn';
import {useAuthStore} from 'stores/auth-store';
import {useI18n} from 'vue-i18n';
const authStore = useAuthStore();
const router = useRouter();
const route = useRoute();
const i18n = useI18n();
const quasar = useQuasar();
interface Option {
value: string;
@ -127,8 +121,8 @@ let submissionModel = ref({
weight: 100,
weightUnit: 'Kg',
reps: 1,
videoFileId: '',
date: new Date().toLocaleDateString('en-CA'),
performedAt: new Date().toLocaleDateString('en-CA'),
taskId: -1
});
const selectedVideoFile: Ref<File | undefined> = ref<File>();
const weightUnits = ['KG', 'LBS'];
@ -169,59 +163,53 @@ async function onSubmitted() {
if (!selectedVideoFile.value || !gym.value) return;
submitting.value = true;
if (await uploadVideo()) {
await createSubmission();
}
submitting.value = false;
}
/**
* Uploads the selected video and returns true if successful.
*/
async function uploadVideo(): Promise<boolean> {
if (!selectedVideoFile.value) return false;
try {
// 1. Upload the video to the CDN.
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitUploading');
await sleep(1000);
submissionModel.value.videoFileId = await uploadVideoToCDN(
selectedVideoFile.value
);
// 2. Wait for the video to be processed.
submitButtonLabel.value = i18n.t(
'gymPage.submitPage.submitVideoProcessing'
);
const processingStatus = await waitUntilVideoProcessingComplete(
submissionModel.value.videoFileId
);
// 3. If successful upload, create the submission.
if (processingStatus === VideoProcessingStatus.COMPLETED) {
try {
submitButtonLabel.value = i18n.t(
'gymPage.submitPage.submitCreatingSubmission'
);
submissionModel.value.taskId = await uploadVideoToCDN(selectedVideoFile.value);
return true;
} catch (error) {
showApiErrorToast(error);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitUploadFailed');
await sleep(1000);
const submission = await api.gyms.submissions.createSubmission(
gym.value,
submissionModel.value,
authStore
);
selectedVideoFile.value = undefined;
submitButtonLabel.value = i18n.t('gymPage.submitPage.submit');
return false;
}
}
/**
* Tries to create a new submission, and if successful, redirects the user to it.
*/
async function createSubmission() {
if (!gym.value) return;
try {
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitCreatingSubmission');
await sleep(1000);
const submission = await api.gyms.submissions.createSubmission(gym.value, submissionModel.value, authStore);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitComplete');
await sleep(2000);
await router.push(`/submissions/${submission.id}`);
} catch (error: any) {
if (error.response && error.response.status === 400) {
quasar.notify({
message: error.response.data.message,
type: 'warning',
position: 'top',
});
showWarningToast(error.response.data.message);
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitFailed');
await sleep(3000);
} else {
showApiErrorToast(error);
}
}
// Otherwise, report the failed submission and give up.
} else {
submitButtonLabel.value = i18n.t('gymPage.submitPage.submitFailed');
await sleep(3000);
}
} catch (error: any) {
showApiErrorToast(error);
} finally {
submitting.value = false;
submitButtonLabel.value = i18n.t('gymPage.submitPage.submit');
}
}

View File

@ -21,7 +21,7 @@ import ExerciseSubmissionListItem from 'components/ExerciseSubmissionListItem.vu
import {showApiErrorToast} from 'src/utils';
import {PaginationHelpers} from 'src/api/main/models';
import InfinitePageLoader from 'src/api/infinite-page-loader';
import {ExerciseSubmission} from 'src/api/main/submission';
import {Submission} from 'src/api/main/submission';
interface Props {
userId: string;
@ -30,7 +30,7 @@ const props = defineProps<Props>();
const authStore = useAuthStore();
const submissions: Ref<ExerciseSubmission[]> = ref([]);
const submissions: Ref<Submission[]> = ref([]);
const loader = new InfinitePageLoader(submissions, async paginationOptions => {
try {
return await api.users.getSubmissions(props.userId, authStore, paginationOptions);
@ -41,7 +41,7 @@ const loader = new InfinitePageLoader(submissions, async paginationOptions => {
onMounted(async () => {
loader.registerWindowScrollListener();
await loader.setPagination(PaginationHelpers.sortedDescBy('performedAt'));
await loader.setPagination(PaginationHelpers.sortedDescBy('properties.performedAt'));
});
</script>

View File

@ -1,6 +1,12 @@
package nl.andrewlalis.gymboardcdn;
import nl.andrewlalis.gymboardcdn.util.ULID;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.files.util.ULID;
import nl.andrewlalis.gymboardcdn.uploads.service.process.FfmpegThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.FfmpegVideoProcessor;
import nl.andrewlalis.gymboardcdn.uploads.service.process.ThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.VideoProcessor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -8,17 +14,32 @@ import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.web.cors.CorsConfiguration;
import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
import org.springframework.web.filter.CorsFilter;
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import java.util.Arrays;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
@Configuration
@EnableScheduling
public class Config {
public class Config implements WebMvcConfigurer {
@Value("${app.web-origin}")
private String webOrigin;
@Value("${app.api-origin}")
private String apiOrigin;
private final ServiceAccessInterceptor serviceAccessInterceptor;
public Config(ServiceAccessInterceptor serviceAccessInterceptor) {
this.serviceAccessInterceptor = serviceAccessInterceptor;
}
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(serviceAccessInterceptor);
}
@Bean
public CorsFilter corsFilter() {
final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
@ -36,4 +57,29 @@ public class Config {
public ULID ulid() {
return new ULID();
}
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
}
@Bean
public FileStorageService fileStorageService() {
return new FileStorageService(ulid(), objectMapper(), "cdn-files");
}
@Bean
public VideoProcessor videoProcessor() {
return new FfmpegVideoProcessor();
}
@Bean
public ThumbnailGenerator thumbnailGenerator() {
return new FfmpegThumbnailGenerator();
}
@Bean
public Executor videoProcessingExecutor() {
return Executors.newFixedThreadPool(1);
}
}

View File

@ -0,0 +1,39 @@
package nl.andrewlalis.gymboardcdn;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.method.HandlerMethod;
import org.springframework.web.servlet.HandlerInterceptor;
import java.lang.reflect.Method;
/**
* An interceptor that checks that requests to endpoints annotated with
* {@link ServiceOnly} have a valid service secret header value.
*/
@Component
public class ServiceAccessInterceptor implements HandlerInterceptor {
private static final String HEADER_NAME = "X-Gymboard-Service-Secret";
@Value("${app.service-secret}")
private String serviceSecret;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
Method handlerMethod = ((HandlerMethod) handler).getMethod();
Class<?> handlerClass = handlerMethod.getDeclaringClass();
ServiceOnly methodAnnotation = handlerMethod.getAnnotation(ServiceOnly.class);
ServiceOnly classAnnotation = handlerClass.getAnnotation(ServiceOnly.class);
if (methodAnnotation != null || classAnnotation != null) {
String secret = request.getHeader(HEADER_NAME);
if (secret == null || !secret.trim().equals(serviceSecret)) {
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
return false;
}
}
return true;
}
}

View File

@ -0,0 +1,16 @@
package nl.andrewlalis.gymboardcdn;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation that can be applied to a controller or controller method to
* restrict access to only requests from another service that provide a
* legitimate service secret.
* @see ServiceAccessInterceptor
*/
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
public @interface ServiceOnly {}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.api;
package nl.andrewlalis.gymboardcdn;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;

View File

@ -1,5 +0,0 @@
package nl.andrewlalis.gymboardcdn.api;
public record FileUploadResponse(
String id
) {}

View File

@ -1,5 +0,0 @@
package nl.andrewlalis.gymboardcdn.api;
public record VideoProcessingTaskStatusResponse(
String status
) {}

View File

@ -0,0 +1,49 @@
package nl.andrewlalis.gymboardcdn.files;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.ServiceOnly;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
/**
* Controller for general-purpose file access.
*/
@RestController
public class FileController {
private final FileStorageService fileStorageService;
public FileController(FileStorageService fileStorageService) {
this.fileStorageService = fileStorageService;
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
fileStorageService.streamToHttpResponse(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FullFileMetadata getFileMetadata(@PathVariable String id) {
try {
var data = fileStorageService.getMetadata(id);
if (data == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND);
return data;
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Couldn't read file metadata.", e);
}
}
@DeleteMapping(path = "/files/{id}") @ServiceOnly
public void deleteFile(@PathVariable String id) {
try {
fileStorageService.delete(id);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to delete file.", e);
}
}
}

View File

@ -0,0 +1,7 @@
package nl.andrewlalis.gymboardcdn.files;
public record FileMetadata (
String filename,
String mimeType,
boolean accessible
) {}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.api;
package nl.andrewlalis.gymboardcdn.files;
public record FileMetadataResponse(
String filename,

View File

@ -0,0 +1,223 @@
package nl.andrewlalis.gymboardcdn.files;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.files.util.ULID;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* This service acts as a low-level driver for interacting with the storage
* system. This includes reading and writing files and their metadata.
* <p>
* Files are stored in a top-level directory, then in 3 sub-directories
* according to their creation date. So if a file is created on 2023-04-02,
* then it will be stored in BASE_DIR/2023/04/02/. All files are uniquely
* identified by a ULID; a monotonic, time-sorted id.
* </p>
* <p>
* Each file has a 1Kb (1024 bytes) metadata block baked into it when it's
* first saved. This metadata block stores a JSON-serialized set of metadata
* properties about the file.
* </p>
*/
public class FileStorageService {
private static final int HEADER_SIZE = 1024;
private final ULID ulid;
private final ObjectMapper objectMapper;
private final String baseStorageDir;
public FileStorageService(ULID ulid, ObjectMapper objectMapper, String baseStorageDir) {
this.ulid = ulid;
this.objectMapper = objectMapper;
this.baseStorageDir = baseStorageDir;
}
public String generateFileId() {
return ulid.nextULID();
}
public String save(Path inputFile, FileMetadata metadata) throws IOException {
try (var in = Files.newInputStream(inputFile)) {
return save(in, metadata, -1);
}
}
/**
* Saves a new file to the storage.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(InputStream in, FileMetadata metadata, long maxSize) throws IOException {
ULID.Value id = ulid.nextValue();
return save(id, in, metadata, maxSize);
}
/**
* Saves a new file to the storage using a specific file id.
* @param id The file id to save to.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(ULID.Value id, InputStream in, FileMetadata metadata, long maxSize) throws IOException {
Path filePath = getStoragePathForFile(id.toString());
Files.createDirectories(filePath.getParent());
try (var out = Files.newOutputStream(filePath)) {
writeMetadata(out, metadata);
byte[] buffer = new byte[8192];
int bytesRead;
long totalBytesWritten = 0;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
totalBytesWritten += bytesRead;
if (maxSize > 0 && totalBytesWritten > maxSize) {
out.close();
Files.delete(filePath);
throw new IOException("File too large.");
}
}
}
return id.toString();
}
/**
* Gets metadata for a file identified by the given id.
* @param rawId The file's id.
* @return The metadata for the file, or null if no file is found.
* @throws IOException If an error occurs while reading metadata.
*/
public FullFileMetadata getMetadata(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) return null;
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
LocalDateTime date = dateFromULID(ULID.parseULID(rawId));
return new FullFileMetadata(
metadata.filename(),
metadata.mimeType(),
Files.size(filePath) - HEADER_SIZE,
date.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
);
}
}
/**
* Streams a stored file to an HTTP response. A NOT_FOUND response is sent
* if the file doesn't exist. Responses include a cache-control header to
* allow clients to cache the response for a long time, as stored files are
* considered to be immutable, unless rarely deleted.
* @param rawId The file's id.
* @param response The HTTP response to write to.
*/
public void streamToHttpResponse(String rawId, HttpServletResponse response) {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return;
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
response.setContentType(metadata.mimeType());
response.setContentLengthLong(Files.size(filePath) - HEADER_SIZE);
response.addHeader("Cache-Control", "max-age=604800, immutable");
var out = response.getOutputStream();
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
}
}
/**
* Gets the path to the physical location of the file with the given id.
* Note that this method makes no guarantee as to whether the file exists.
* @param rawId The id of the file.
* @return The path to the location where the file is stored.
*/
public Path getStoragePathForFile(String rawId) {
ULID.Value id = ULID.parseULID(rawId);
LocalDateTime time = dateFromULID(id);
Path dir = Path.of(baseStorageDir)
.resolve(String.format("%04d", time.getYear()))
.resolve(String.format("%02d", time.getMonthValue()))
.resolve(String.format("%02d", time.getDayOfMonth()));
return dir.resolve(rawId);
}
/**
* Deletes the file with a given id, if it exists.
* @param rawId The file's id.
* @throws IOException If an error occurs.
*/
public void delete(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
Files.deleteIfExists(filePath);
}
public void copyTo(String fileId, Path filePath) throws IOException {
Path inputFilePath = getStoragePathForFile(fileId);
if (Files.notExists(inputFilePath)) {
throw new IOException("File " + fileId + " not found.");
}
try (
var in = Files.newInputStream(inputFilePath);
var out = Files.newOutputStream(filePath)
) {
readMetadata(in);
in.transferTo(out);
}
}
private static LocalDateTime dateFromULID(ULID.Value value) {
return Instant.ofEpochMilli(value.timestamp())
.atOffset(ZoneOffset.UTC)
.toLocalDateTime();
}
private FileMetadata readMetadata(InputStream in) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
int readCount = in.read(buffer.array(), 0, HEADER_SIZE);
if (readCount != HEADER_SIZE) throw new IOException("Invalid header. Read " + readCount + " bytes instead of " + HEADER_SIZE);
short metadataBytesLength = buffer.getShort();
byte[] metadataBytes = new byte[metadataBytesLength];
buffer.get(metadataBytes);
return objectMapper.readValue(metadataBytes, FileMetadata.class);
}
private void writeMetadata(OutputStream out, FileMetadata metadata) throws IOException {
ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE);
byte[] metadataBytes = objectMapper.writeValueAsBytes(metadata);
if (metadataBytes.length > HEADER_SIZE - 2) {
throw new IOException("Metadata is too large.");
}
headerBuffer.putShort((short) metadataBytes.length);
headerBuffer.put(metadataBytes);
out.write(headerBuffer.array());
}
}

View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboardcdn.files;
public record FullFileMetadata(
String filename,
String mimeType,
long size,
String createdAt
) {}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.util;
package nl.andrewlalis.gymboardcdn.files.util;
/*
* sulky-modules - several general-purpose modules.

View File

@ -1,81 +0,0 @@
package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
@Entity
@Table(name = "stored_file")
public class StoredFile {
/**
* ULID-based unique file identifier.
*/
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
/**
* The timestamp at which the file was originally uploaded.
*/
@Column(nullable = false, updatable = false)
private LocalDateTime uploadedAt;
/**
* The original filename.
*/
@Column(nullable = false, updatable = false)
private String name;
/**
* The type of the file.
*/
@Column(updatable = false)
private String mimeType;
/**
* The file's size on the disk.
*/
@Column(nullable = false, updatable = false)
private long size;
public StoredFile() {}
public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) {
this.id = id;
this.name = name;
this.mimeType = mimeType;
this.size = size;
this.uploadedAt = uploadedAt;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getName() {
return name;
}
public String getMimeType() {
return mimeType;
}
public long getSize() {
return size;
}
public LocalDateTime getUploadedAt() {
return uploadedAt;
}
}

View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboardcdn.model;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface StoredFileRepository extends JpaRepository<StoredFile, String> {
}

View File

@ -1,88 +0,0 @@
package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* An entity to keep track of a task for processing a raw video into a better
* format for Gymboard to serve.
*/
@Entity
@Table(name = "task_video_processing")
public class VideoProcessingTask {
public enum Status {
WAITING,
IN_PROGRESS,
COMPLETED,
FAILED
}
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private Status status;
/**
* The original filename.
*/
@Column(nullable = false)
private String filename;
/**
* The path to the temporary file that we'll use as input.
*/
@Column(nullable = false)
private String tempFilePath;
/**
* The identifier that will be used to identify the final video, if it
* is processed successfully.
*/
@Column(nullable = false, updatable = false, length = 26)
private String videoIdentifier;
public VideoProcessingTask() {}
public VideoProcessingTask(Status status, String filename, String tempFilePath, String videoIdentifier) {
this.status = status;
this.filename = filename;
this.tempFilePath = tempFilePath;
this.videoIdentifier = videoIdentifier;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getFilename() {
return filename;
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getTempFilePath() {
return tempFilePath;
}
public String getVideoIdentifier() {
return videoIdentifier;
}
}

View File

@ -1,116 +0,0 @@
package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.concurrent.TimeUnit;
/**
* The service that manages storing and retrieving files from a base filesystem.
*/
@Service
public class FileService {
private static final Logger log = LoggerFactory.getLogger(FileService.class);
@Value("${app.files.storage-dir}")
private String storageDir;
@Value("${app.files.temp-dir}")
private String tempDir;
private final StoredFileRepository storedFileRepository;
private final ULID ulid;
public FileService(StoredFileRepository storedFileRepository, ULID ulid) {
this.storedFileRepository = storedFileRepository;
this.ulid = ulid;
}
public Path getStoragePathForFile(StoredFile file) throws IOException {
LocalDateTime time = file.getUploadedAt();
Path dir = Path.of(storageDir)
.resolve(Integer.toString(time.getYear()))
.resolve(Integer.toString(time.getMonthValue()))
.resolve(Integer.toString(time.getDayOfMonth()));
if (Files.notExists(dir)) Files.createDirectories(dir);
return dir.resolve(file.getId());
}
public String createNewFileIdentifier() {
return ulid.nextULID();
}
public Path saveToTempFile(InputStream in, String filename) throws IOException {
Path tempDir = getTempDir();
String suffix = null;
if (filename != null) {
int idx = filename.lastIndexOf('.');
if (idx >= 0) {
suffix = filename.substring(idx);
}
}
Path tempFile = Files.createTempFile(tempDir, null, suffix);
try (var out = Files.newOutputStream(tempFile)) {
in.transferTo(out);
}
return tempFile;
}
private Path getStorageDir() throws IOException {
Path dir = Path.of(storageDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
private Path getTempDir() throws IOException {
Path dir = Path.of(tempDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
/**
* Scheduled task that removes any StoredFile entities for which no more
* physical file exists.
*/
@Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
@Transactional
public void removeOrphanedFiles() {
Pageable pageable = PageRequest.of(0, 100, Sort.by(Sort.Direction.DESC, "createdAt"));
Page<StoredFile> page = storedFileRepository.findAll(pageable);
while (!page.isEmpty()) {
for (var storedFile : page) {
try {
Path filePath = getStoragePathForFile(storedFile);
if (Files.notExists(filePath)) {
log.warn("Removing stored file {} because it no longer exists on the disk.", storedFile.getId());
storedFileRepository.delete(storedFile);
}
} catch (IOException e) {
log.error("Couldn't get storage path for stored file.", e);
}
}
pageable = pageable.next();
page = storedFileRepository.findAll(pageable);
}
}
}

View File

@ -1,131 +0,0 @@
package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.format.DateTimeFormatter;
@Service
public class UploadService {
private static final Logger log = LoggerFactory.getLogger(UploadService.class);
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final StoredFileRepository storedFileRepository;
private final VideoProcessingTaskRepository videoTaskRepository;
private final FileService fileService;
public UploadService(StoredFileRepository storedFileRepository,
VideoProcessingTaskRepository videoTaskRepository,
FileService fileService) {
this.storedFileRepository = storedFileRepository;
this.videoTaskRepository = videoTaskRepository;
this.fileService = fileService;
}
/**
* Handles uploading of a processable video file that will be processed
* before being stored in the system.
* @param request The request from which we can read the file.
* @return A response that contains an identifier that can be used to check
* the status of the video processing, and eventually fetch the video.
*/
@Transactional
public FileUploadResponse processableVideoUpload(HttpServletRequest request) {
String contentLengthStr = request.getHeader("Content-Length");
if (contentLengthStr == null || !contentLengthStr.matches("\\d+")) {
throw new ResponseStatusException(HttpStatus.LENGTH_REQUIRED);
}
long contentLength = Long.parseUnsignedLong(contentLengthStr);
if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
}
Path tempFile;
String filename = request.getHeader("X-Gymboard-Filename");
if (filename == null) filename = "unnamed.mp4";
try {
tempFile = fileService.saveToTempFile(request.getInputStream(), filename);
} catch (IOException e) {
log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
String identifier = fileService.createNewFileIdentifier();
videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.WAITING,
filename,
tempFile.toString(),
identifier
));
return new FileUploadResponse(identifier);
}
/**
* Gets the status of a video processing task.
* @param id The video identifier.
* @return The status of the video processing task.
*/
@Transactional(readOnly = true)
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) {
VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(task.getStatus().name());
}
/**
* Streams the contents of a stored file to a client via the Http response.
* @param id The file's unique identifier.
* @param response The response to stream the content to.
*/
@Transactional(readOnly = true)
public void streamFile(String id, HttpServletResponse response) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = fileService.getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
@Transactional(readOnly = true)
public FileMetadataResponse getFileMetadata(String id) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
try {
Path filePath = fileService.getStoragePathForFile(file);
boolean exists = Files.exists(filePath);
return new FileMetadataResponse(
file.getName(),
file.getMimeType(),
file.getSize(),
file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
exists
);
} catch (IOException e) {
log.error("Couldn't get path to stored file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}

View File

@ -1,173 +0,0 @@
package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@Service
public class VideoProcessingService {
private static final Logger log = LoggerFactory.getLogger(VideoProcessingService.class);
private final Executor taskExecutor;
private final VideoProcessingTaskRepository taskRepo;
private final StoredFileRepository storedFileRepository;
private final FileService fileService;
public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, StoredFileRepository storedFileRepository, FileService fileService) {
this.taskExecutor = taskExecutor;
this.taskRepo = taskRepo;
this.storedFileRepository = storedFileRepository;
this.fileService = fileService;
}
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void startWaitingTasks() {
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
for (var task : waitingTasks) {
log.info("Queueing processing of video {}.", task.getVideoIdentifier());
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
taskExecutor.execute(() -> processVideo(task));
}
}
@Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS)
public void removeOldTasks() {
LocalDateTime cutoff = LocalDateTime.now().minusHours(12);
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
for (var task : oldTasks) {
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
log.info("Deleting completed task for video {}.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.info("Deleting failed task for video {}.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
log.info("Task for video {} was in progress for too long; deleting.", task.getVideoIdentifier());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
log.info("Task for video {} was waiting for too long; deleting.", task.getVideoIdentifier());
taskRepo.delete(task);
}
}
}
private void processVideo(VideoProcessingTask task) {
log.info("Started processing video {}.", task.getVideoIdentifier());
Path tempFile = Path.of(task.getTempFilePath());
if (Files.notExists(tempFile) || !Files.isReadable(tempFile)) {
log.error("Temp file {} doesn't exist or isn't readable.", tempFile);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// Then begin running the actual FFMPEG processing.
Path tempDir = tempFile.getParent();
Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier());
try {
processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile);
} catch (Exception e) {
e.printStackTrace();
log.error("""
Video processing failed for video {}:
Input file: {}
Output file: {}
Exception message: {}""",
task.getVideoIdentifier(),
tempFile,
ffmpegOutputFile,
e.getMessage()
);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// And finally, copy the output to the final location.
try {
StoredFile storedFile = new StoredFile(
task.getVideoIdentifier(),
task.getFilename(),
"video/mp4",
Files.size(ffmpegOutputFile),
task.getCreatedAt()
);
Path finalFilePath = fileService.getStoragePathForFile(storedFile);
Files.move(ffmpegOutputFile, finalFilePath);
Files.deleteIfExists(tempFile);
Files.deleteIfExists(ffmpegOutputFile);
storedFileRepository.saveAndFlush(storedFile);
updateTask(task, VideoProcessingTask.Status.COMPLETED);
log.info("Finished processing video {}.", task.getVideoIdentifier());
} catch (IOException e) {
log.error("Failed to copy processed video to final storage location.", e);
updateTask(task, VideoProcessingTask.Status.FAILED);
}
}
/**
* Uses the `ffmpeg` system command to process a raw input video and produce
* a compressed, reduced-size output video that's ready for usage in the
* application.
* @param dir The working directory.
* @param inFile The input file to read from.
* @param outFile The output file to write to. MUST have a ".mp4" extension.
* @throws IOException If a filesystem error occurs.
* @throws CommandFailedException If the ffmpeg command fails.
* @throws InterruptedException If the ffmpeg command is interrupted.
*/
private void processVideoWithFFMPEG(Path dir, Path inFile, Path outFile) throws IOException, InterruptedException {
Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log");
Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log");
final String[] command = {
"ffmpeg",
"-i", inFile.getFileName().toString(),
"-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264",
"-crf", "28",
"-f", "mp4",
outFile.getFileName().toString()
};
long startSize = Files.size(inFile);
Instant startTime = Instant.now();
Process ffmpegProcess = new ProcessBuilder()
.command(command)
.redirectOutput(tmpStdout.toFile())
.redirectError(tmpStderr.toFile())
.directory(dir.toFile())
.start();
int result = ffmpegProcess.waitFor();
if (result != 0) throw new CommandFailedException(command, result, tmpStdout, tmpStderr);
long endSize = Files.size(outFile);
Duration dur = Duration.between(startTime, Instant.now());
double reductionFactor = startSize / (double) endSize;
String reductionFactorStr = String.format("%.3f%%", reductionFactor * 100);
log.info("Processed video from {} bytes to {} bytes in {} seconds, {} reduction.", startSize, endSize, dur.getSeconds(), reductionFactorStr);
// Delete the logs if everything was successful.
Files.deleteIfExists(tmpStdout);
Files.deleteIfExists(tmpStderr);
}
private void updateTask(VideoProcessingTask task, VideoProcessingTask.Status status) {
task.setStatus(status);
taskRepo.saveAndFlush(task);
}
}

View File

@ -1,8 +1,8 @@
package nl.andrewlalis.gymboardcdn.api;
package nl.andrewlalis.gymboardcdn.uploads.api;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.UploadService;
import nl.andrewlalis.gymboardcdn.ServiceOnly;
import nl.andrewlalis.gymboardcdn.uploads.service.UploadService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@ -17,22 +17,17 @@ public class UploadController {
}
@PostMapping(path = "/uploads/video", consumes = {"video/mp4"})
public FileUploadResponse uploadVideo(HttpServletRequest request) {
public VideoUploadResponse uploadVideo(HttpServletRequest request) {
return uploadService.processableVideoUpload(request);
}
@GetMapping(path = "/uploads/video/{id}/status")
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) {
return uploadService.getVideoProcessingStatus(id);
@PostMapping(path = "/uploads/video/{taskId}/start") @ServiceOnly
public void startVideoProcessing(@PathVariable long taskId) {
uploadService.startVideoProcessing(taskId);
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
uploadService.streamFile(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
return uploadService.getFileMetadata(id);
@GetMapping(path = "/uploads/video/{taskId}/status")
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable long taskId) {
return uploadService.getVideoProcessingStatus(taskId);
}
}

View File

@ -0,0 +1,7 @@
package nl.andrewlalis.gymboardcdn.uploads.api;
public record VideoProcessingTaskStatusResponse(
String status,
String videoFileId,
String thumbnailFileId
) {}

View File

@ -0,0 +1,5 @@
package nl.andrewlalis.gymboardcdn.uploads.api;
public record VideoUploadResponse(
long taskId
) {}

View File

@ -0,0 +1,96 @@
package nl.andrewlalis.gymboardcdn.uploads.model;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* An entity to keep track of a task for processing a raw video into a better
* format for Gymboard to serve. Generally, tasks are processed like so:
* <ol>
* <li>A video is uploaded, and a new task is created with the NOT_STARTED status.</li>
* <li>Once the Gymboard API verifies the associated submission, it'll
* request to start the task, bringing it to the WAITING status.</li>
* <li>When a task executor picks up the waiting task, its status changes to IN_PROGRESS.</li>
* <li>If the video is processed successfully, then the task is COMPLETED, otherwise FAILED.</li>
* </ol>
*/
@Entity
@Table(name = "task_video_processing")
public class VideoProcessingTask {
public enum Status {
NOT_STARTED,
WAITING,
IN_PROGRESS,
COMPLETED,
FAILED
}
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private Status status;
/**
* The file id for the original, raw user-uploaded video file that needs to
* be processed.
*/
@Column(nullable = false, updatable = false, length = 26)
private String uploadFileId;
@Column(length = 26)
private String videoFileId;
@Column(length = 26)
private String thumbnailFileId;
public VideoProcessingTask() {}
public VideoProcessingTask(Status status, String uploadFileId) {
this.status = status;
this.uploadFileId = uploadFileId;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getUploadFileId() {
return uploadFileId;
}
public String getVideoFileId() {
return videoFileId;
}
public void setVideoFileId(String videoFileId) {
this.videoFileId = videoFileId;
}
public String getThumbnailFileId() {
return thumbnailFileId;
}
public void setThumbnailFileId(String thumbnailFileId) {
this.thumbnailFileId = thumbnailFileId;
}
}

View File

@ -1,16 +1,13 @@
package nl.andrewlalis.gymboardcdn.model;
package nl.andrewlalis.gymboardcdn.uploads.model;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@Repository
public interface VideoProcessingTaskRepository extends JpaRepository<VideoProcessingTask, Long> {
Optional<VideoProcessingTask> findByVideoIdentifier(String identifier);
List<VideoProcessingTask> findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status status);
List<VideoProcessingTask> findAllByCreatedAtBefore(LocalDateTime cutoff);

View File

@ -0,0 +1,12 @@
package nl.andrewlalis.gymboardcdn.uploads.model;
public record VideoProcessingTaskStatusUpdate(
long taskId,
String status,
String videoFileId,
String thumbnailFileId
) {
public VideoProcessingTaskStatusUpdate(VideoProcessingTask task) {
this(task.getId(), task.getStatus().name(), task.getVideoFileId(), task.getThumbnailFileId());
}
}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboardcdn.service;
package nl.andrewlalis.gymboardcdn.uploads.service;
import java.io.IOException;
import java.nio.file.Path;

View File

@ -0,0 +1,103 @@
package nl.andrewlalis.gymboardcdn.uploads.service;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.uploads.api.VideoUploadResponse;
import nl.andrewlalis.gymboardcdn.uploads.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.files.FileMetadata;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
@Service
public class UploadService {
private static final Logger log = LoggerFactory.getLogger(UploadService.class);
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final VideoProcessingTaskRepository videoTaskRepository;
private final FileStorageService fileStorageService;
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileStorageService fileStorageService) {
this.videoTaskRepository = videoTaskRepository;
this.fileStorageService = fileStorageService;
}
/**
* Handles uploading of a processable video file that will be processed
* before being stored in the system.
* @param request The request from which we can read the file.
* @return A response containing the id of the video processing task, to be
* given to the Gymboard API so that it can further manage processing after
* a submission is completed.
*/
@Transactional
public VideoUploadResponse processableVideoUpload(HttpServletRequest request) {
String contentLengthStr = request.getHeader("Content-Length");
if (contentLengthStr == null || !contentLengthStr.matches("\\d+")) {
throw new ResponseStatusException(HttpStatus.LENGTH_REQUIRED);
}
long contentLength = Long.parseUnsignedLong(contentLengthStr);
if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
}
String filename = request.getHeader("X-Gymboard-Filename");
if (filename == null) filename = "unnamed.mp4";
FileMetadata metadata = new FileMetadata(
filename,
request.getContentType(),
false
);
String uploadFileId;
try {
uploadFileId = fileStorageService.save(request.getInputStream(), metadata, contentLength);
} catch (IOException e) {
log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
var task = videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.NOT_STARTED,
uploadFileId
));
return new VideoUploadResponse(task.getId());
}
/**
* Gets the status of a video processing task.
* @param id The task id.
* @return The status of the video processing task.
*/
@Transactional(readOnly = true)
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(long id) {
VideoProcessingTask task = videoTaskRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(
task.getStatus().name(),
task.getVideoFileId(),
task.getThumbnailFileId()
);
}
/**
* Marks this task as waiting to be picked up for processing. The Gymboard
* API should send a message itself to start processing of an uploaded video
* once it validates a submission.
* @param taskId The task id.
*/
@Transactional
public void startVideoProcessing(long taskId) {
VideoProcessingTask task = videoTaskRepository.findById(taskId)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
if (task.getStatus() == VideoProcessingTask.Status.NOT_STARTED) {
task.setStatus(VideoProcessingTask.Status.WAITING);
videoTaskRepository.save(task);
}
}
}

View File

@ -0,0 +1,229 @@
package nl.andrewlalis.gymboardcdn.uploads.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.andrewlalis.gymboardcdn.files.FileMetadata;
import nl.andrewlalis.gymboardcdn.files.FileStorageService;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskRepository;
import nl.andrewlalis.gymboardcdn.uploads.model.VideoProcessingTaskStatusUpdate;
import nl.andrewlalis.gymboardcdn.uploads.service.process.ThumbnailGenerator;
import nl.andrewlalis.gymboardcdn.uploads.service.process.VideoProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@Service
public class VideoProcessingService {
private static final Logger log = LoggerFactory.getLogger(VideoProcessingService.class);
private final Executor videoProcessingExecutor;
private final VideoProcessingTaskRepository taskRepo;
private final FileStorageService fileStorageService;
private final VideoProcessor videoProcessor;
private final ThumbnailGenerator thumbnailGenerator;
private final ObjectMapper objectMapper;
@Value("${app.api-origin}")
private String apiOrigin;
@Value("${app.api-secret}")
private String apiSecret;
public VideoProcessingService(Executor videoProcessingExecutor,
VideoProcessingTaskRepository taskRepo,
FileStorageService fileStorageService,
VideoProcessor videoProcessor,
ThumbnailGenerator thumbnailGenerator, ObjectMapper objectMapper) {
this.videoProcessingExecutor = videoProcessingExecutor;
this.taskRepo = taskRepo;
this.fileStorageService = fileStorageService;
this.videoProcessor = videoProcessor;
this.thumbnailGenerator = thumbnailGenerator;
this.objectMapper = objectMapper;
}
private void updateTask(VideoProcessingTask task, VideoProcessingTask.Status status) {
task.setStatus(status);
taskRepo.saveAndFlush(task);
if (status == VideoProcessingTask.Status.COMPLETED || status == VideoProcessingTask.Status.FAILED) {
sendTaskCompleteToApi(task);
}
}
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void startWaitingTasks() {
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
for (var task : waitingTasks) {
log.info("Queueing processing of task {}.", task.getId());
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
videoProcessingExecutor.execute(() -> processVideo(task));
}
}
@Scheduled(fixedRate = 1, timeUnit = TimeUnit.HOURS)
public void removeOldTasks() {
LocalDateTime cutoff = LocalDateTime.now().minusHours(12);
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
for (var task : oldTasks) {
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
log.info("Deleting completed task {}.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.info("Deleting failed task {}.", task.getId());
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
log.info("Task {} was in progress for too long; deleting.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
log.info("Task {} was waiting for too long; deleting.", task.getId());
deleteAllTaskFiles(task);
taskRepo.delete(task);
}
}
}
private void processVideo(VideoProcessingTask task) {
log.info("Started processing task {}.", task.getId());
Path uploadFile = fileStorageService.getStoragePathForFile(task.getUploadFileId());
Path rawUploadFile = uploadFile.resolveSibling(task.getUploadFileId() + "-vid-in");
if (Files.notExists(uploadFile) || !Files.isReadable(uploadFile)) {
log.error("Uploaded video file {} doesn't exist or isn't readable.", uploadFile);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
try {
fileStorageService.copyTo(task.getUploadFileId(), rawUploadFile);
} catch (IOException e) {
log.error("Failed to copy raw video file {} to {}.", uploadFile, rawUploadFile);
e.printStackTrace();
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// Run the actual processing here.
Path videoFile = uploadFile.resolveSibling(task.getUploadFileId() + "-vid-out.mp4");
Path thumbnailFile = uploadFile.resolveSibling(task.getUploadFileId() + "-thm-out.jpeg");
try {
log.info("Processing video for uploaded video file {}.", uploadFile.getFileName());
videoProcessor.processVideo(rawUploadFile, videoFile);
log.info("Generating thumbnail for uploaded video file {}.", uploadFile.getFileName());
thumbnailGenerator.generateThumbnailImage(videoFile, thumbnailFile);
} catch (Exception e) {
e.printStackTrace();
log.error("""
Video processing failed for task {}:
Input file: {}
Output file: {}
Exception message: {}""",
task.getId(),
rawUploadFile,
videoFile,
e.getMessage()
);
updateTask(task, VideoProcessingTask.Status.FAILED);
return;
}
// And finally, copy the output to the final location.
try {
// Save the video to a final file location.
var originalMetadata = fileStorageService.getMetadata(task.getUploadFileId());
FileMetadata metadata = new FileMetadata(originalMetadata.filename(), originalMetadata.mimeType(), true);
String videoFileId = fileStorageService.save(videoFile, metadata);
// Save the thumbnail too.
FileMetadata thumbnailMetadata = new FileMetadata("thumbnail.jpeg", "image/jpeg", true);
String thumbnailFileId = fileStorageService.save(thumbnailFile, thumbnailMetadata);
task.setVideoFileId(videoFileId);
task.setThumbnailFileId(thumbnailFileId);
updateTask(task, VideoProcessingTask.Status.COMPLETED);
log.info("Finished processing task {}.", task.getId());
} catch (IOException e) {
log.error("Failed to copy processed video to final storage location.", e);
updateTask(task, VideoProcessingTask.Status.FAILED);
} finally {
deleteAllTaskFiles(task);
}
}
/**
* Sends an update message to the Gymboard API when a task finishes its
* processing. Note that Gymboard API will also eventually poll the CDN's
* own API to get task status if we fail to send it, so there's some
* redundancy built-in.
* @param task The task to send.
*/
private void sendTaskCompleteToApi(VideoProcessingTask task) {
String json;
try {
json = objectMapper.writeValueAsString(new VideoProcessingTaskStatusUpdate(task));
} catch (JsonProcessingException e) {
log.error("JSON error while sending task data to API for task " + task.getId(), e);
return;
}
HttpClient httpClient = HttpClient.newBuilder().build();
HttpRequest request = HttpRequest.newBuilder(URI.create(apiOrigin + "/submissions/video-processing-complete"))
.header("Content-Type", "application/json")
.header("X-Gymboard-Service-Secret", apiSecret)
.timeout(Duration.ofSeconds(3))
.POST(HttpRequest.BodyPublishers.ofString(json))
.build();
try {
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() >= 400) {
log.error("API returned not-ok response {}", response.statusCode());
}
} catch (Exception e) {
log.error("Failed to send HTTP request to API.", e);
}
}
/**
* Helper function to delete all temporary files related to a task's
* processing operations. If the task is FAILED, then files are kept for
* debugging purposes.
* @param task The task to delete files for.
*/
private void deleteAllTaskFiles(VideoProcessingTask task) {
if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.warn("Retaining files for failed task {}, upload id {}.", task.getId(), task.getUploadFileId());
return;
}
Path dir = fileStorageService.getStoragePathForFile(task.getUploadFileId()).getParent();
try (var s = Files.list(dir)) {
var files = s.toList();
for (var file : files) {
String filename = file.getFileName().toString().strip();
if (Files.isRegularFile(file) && filename.startsWith(task.getUploadFileId())) {
try {
Files.delete(file);
} catch (IOException e) {
log.error("Failed to delete file " + file + " related to task " + task.getId(), e);
}
}
}
} catch (IOException e) {
log.error("Failed to list files in " + dir + " when deleting files for task " + task.getId(), e);
}
}
}

View File

@ -0,0 +1,42 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import nl.andrewlalis.gymboardcdn.uploads.service.CommandFailedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public abstract class FfmpegCommandExecutor {
private static final Logger log = LoggerFactory.getLogger(FfmpegCommandExecutor.class);
protected abstract String[] buildCommand(Path inputFile, Path outputFile);
public void run(String label, Path inputFile, Path outputFile) throws IOException {
String inputFilename = inputFile.getFileName().toString().strip();
Path stdout = inputFile.resolveSibling(inputFilename + "-ffmpeg-" + label + "-out.log");
Path stderr = inputFile.resolveSibling(inputFilename + "-ffmpeg-" + label + "-err.log");
String[] command = buildCommand(inputFile, outputFile);
Process process = new ProcessBuilder(buildCommand(inputFile, outputFile))
.redirectOutput(stdout.toFile())
.redirectError(stderr.toFile())
.start();
try {
int result = process.waitFor();
if (result != 0) {
throw new CommandFailedException(command, result, stdout, stderr);
}
} catch (InterruptedException e) {
throw new IOException("Interrupted while waiting for ffmpeg to finish.", e);
}
// Try to clean up output files when the command exited successfully.
try {
Files.deleteIfExists(stdout);
Files.deleteIfExists(stderr);
} catch (IOException e) {
log.warn("Failed to delete output files after successful ffmpeg execution.", e);
}
}
}

View File

@ -0,0 +1,21 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public class FfmpegThumbnailGenerator extends FfmpegCommandExecutor implements ThumbnailGenerator {
@Override
public void generateThumbnailImage(Path videoInputFile, Path outputFilePath) throws IOException {
super.run("thm", videoInputFile, outputFilePath);
}
@Override
protected String[] buildCommand(Path inputFile, Path outputFile) {
return new String[]{
"ffmpeg",
"-i", inputFile.toString(),
"-vframes", "1",
outputFile.toString()
};
}
}

View File

@ -0,0 +1,48 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
public class FfmpegVideoProcessor extends FfmpegCommandExecutor implements VideoProcessor {
private static final Logger log = LoggerFactory.getLogger(FfmpegVideoProcessor.class);
@Override
public void processVideo(Path inputFilePath, Path outputFilePath) throws IOException {
Instant start = Instant.now();
long inputFileSize = Files.size(inputFilePath);
super.run("vid", inputFilePath, outputFilePath);
Duration duration = Duration.between(start, Instant.now());
long outputFileSize = Files.size(outputFilePath);
double reductionFactor = inputFileSize / (double) outputFileSize;
double durationSeconds = duration.toMillis() / 1000.0;
log.info(
"Processed video {} from {} to {} bytes, {} reduction in {} seconds.",
inputFilePath.getFileName().toString(),
inputFileSize,
outputFileSize,
String.format("%.3f%%", reductionFactor),
String.format("%.3f", durationSeconds)
);
}
@Override
protected String[] buildCommand(Path inputFile, Path outputFile) {
return new String[]{
"ffmpeg",
"-i", inputFile.toString(),
"-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264",
"-crf", "28",
"-f", "mp4",
outputFile.toString()
};
}
}

View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public interface ThumbnailGenerator {
void generateThumbnailImage(Path videoInputFile, Path outputFilePath) throws IOException;
}

View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboardcdn.uploads.service.process;
import java.io.IOException;
import java.nio.file.Path;
public interface VideoProcessor {
void processVideo(Path inputFilePath, Path outputFilePath) throws IOException;
}

View File

@ -6,7 +6,9 @@ spring.jpa.hibernate.ddl-auto=update
server.port=8082
# A secret header token that other services must provide to use service-only endpoints.
app.service-secret=testing
app.web-origin=http://localhost:9000
app.api-origin=http://localhost:8080
app.files.storage-dir=./cdn-files/
app.files.temp-dir=./cdn-files/tmp/
app.api-secret=testing

View File

@ -1,22 +1,11 @@
package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.ServletInputStream;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.AdditionalAnswers.returnsFirstArg;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
public class UploadServiceTest {
/**
@ -27,31 +16,30 @@ public class UploadServiceTest {
*/
@Test
public void processableVideoUploadSuccess() throws IOException {
StoredFileRepository storedFileRepository = Mockito.mock(StoredFileRepository.class);
VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
when(videoTaskRepository.save(any(VideoProcessingTask.class)))
.then(returnsFirstArg());
FileService fileService = Mockito.mock(FileService.class);
when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
.thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
when(fileService.createNewFileIdentifier()).thenReturn("abc");
UploadService uploadService = new UploadService(
storedFileRepository,
videoTaskRepository,
fileService
);
HttpServletRequest mockRequest = mock(HttpServletRequest.class);
when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
when(mockRequest.getHeader("Content-Length")).thenReturn("123");
ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
var expectedResponse = new FileUploadResponse("abc");
var response = uploadService.processableVideoUpload(mockRequest);
assertEquals(expectedResponse, response);
verify(fileService, times(1)).saveToTempFile(any(), any());
verify(videoTaskRepository, times(1)).save(any());
verify(fileService, times(1)).createNewFileIdentifier();
// TODO: Refactor all of this!
// VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
// when(videoTaskRepository.save(any(VideoProcessingTask.class)))
// .then(returnsFirstArg());
// FileService fileService = Mockito.mock(FileService.class);
// when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
// .thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
//
// when(fileService.createNewFileIdentifier()).thenReturn("abc");
//
// UploadService uploadService = new UploadService(
// videoTaskRepository,
// fileService
// );
// HttpServletRequest mockRequest = mock(HttpServletRequest.class);
// when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
// when(mockRequest.getHeader("Content-Length")).thenReturn("123");
// ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
// when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
// var expectedResponse = new FileUploadResponse("abc");
// var response = uploadService.processableVideoUpload(mockRequest);
// assertEquals(expectedResponse, response);
// verify(fileService, times(1)).saveToTempFile(any(), any());
// verify(videoTaskRepository, times(1)).save(any());
// verify(fileService, times(1)).createNewFileIdentifier();
}
}

View File

@ -1,7 +1,6 @@
import std.stdio;
import cli;
import command;
import services;
import consolecolors;
@ -10,12 +9,11 @@ void main() {
ServiceManager serviceManager = new ServiceManager();
CliHandler cliHandler = new CliHandler();
cliHandler.register("service", new ServiceCommand(serviceManager));
cwriteln("\n<blue>Gymboard CLI</blue>: <grey>Command-line interface for managing Gymboard services.</grey>");
cwriteln(" Type <cyan>help</cyan> for more information.\n Type <red>exit</red> to exit the CLI.\n");
while (!cliHandler.isExitRequested) {
cwriteln("Gymboard CLI: Type <cyan>help</cyan> for more information. Type <red>exit</red> to exit the CLI.");
while (!cliHandler.shouldExit) {
cwrite("&gt; ".blue);
cliHandler.readAndHandleCommand();
}
serviceManager.stopAll();
cwriteln("Goodbye!".blue);
cwriteln("Goodbye!".green);
}

View File

@ -7,7 +7,45 @@ import std.typecons;
import consolecolors;
import command.base;
interface CliCommand {
void handle(string[] args);
}
class CliHandler {
private CliCommand[string] commands;
public bool shouldExit = false;
public void register(string name, CliCommand command) {
this.commands[name] = command;
}
public void readAndHandleCommand() {
string[] commandAndArgs = readln().strip().split!isWhite();
if (commandAndArgs.length == 0) return;
string command = commandAndArgs[0].toLower();
if (command == "help") {
showHelp();
} else if (command == "exit") {
shouldExit = true;
} else if (command in commands) {
commands[command].handle(commandAndArgs.length > 1 ? commandAndArgs[1 .. $] : []);
} else {
cwritefln("Unknown command: %s".red, command.orange);
}
}
}
void showHelp() {
writeln(q"HELP
Gymboard CLI: A tool for streamlining development.
Commands:
help Shows this message.
exit Exits the CLI, stopping any running services.
HELP");
}
import services;
class ServiceCommand : CliCommand {
@ -48,14 +86,6 @@ class ServiceCommand : CliCommand {
}
}
string name() const {
return "Service";
}
string description() const {
return "bleh";
}
/**
* Validates that a service command contains as its second argument a valid
* service name.

View File

@ -1,102 +0,0 @@
module command.base;
import consolecolors;
import std.stdio;
import std.string;
import std.uni;
interface CliCommand {
void handle(string[] args);
string name() const;
string description() const;
}
class CliHandler {
private CliCommand[string] commands;
private bool exitRequested = false;
this() {
commands["help"] = new HelpCommand(this);
commands["exit"] = new ExitCommand(this);
}
void register(string name, CliCommand command) {
this.commands[name.strip().toLower()] = command;
}
void readAndHandleCommand() {
string[] commandAndArgs = readln().strip().split!isWhite();
if (commandAndArgs.length == 0) return;
string command = commandAndArgs[0].toLower();
if (command in commands) {
commands[command].handle(commandAndArgs.length > 1 ? commandAndArgs[1 .. $] : []);
} else {
cwritefln("Unknown command: %s".red, command.orange);
}
}
void setExitRequested() {
this.exitRequested = true;
}
bool isExitRequested() const {
return this.exitRequested;
}
CliCommand[string] getCommands() {
return this.commands;
}
}
class HelpCommand : CliCommand {
private CliHandler handler;
this(CliHandler handler) {
this.handler = handler;
}
void handle(string[] args) {
import std.algorithm;
cwriteln("<blue>Gymboard CLI Help</blue>: <grey>Information about how to use this program.</grey>");
string[] commandNames = this.handler.getCommands().keys;
sort(commandNames);
uint longestCommandNameLength = commandNames.map!(n => cast(uint) n.length).maxElement;
foreach (name; commandNames) {
CliCommand command = this.handler.getCommands()[name];
string formattedName = cyan(leftJustify(name, longestCommandNameLength + 1, ' '));
cwriteln(formattedName, command.description().grey);
}
}
string name() const {
return "help";
}
string description() const {
return "Shows help information.";
}
}
class ExitCommand : CliCommand {
private CliHandler handler;
this(CliHandler handler) {
this.handler = handler;
}
void handle(string[] args) {
this.handler.setExitRequested();
}
string name() const {
return "exit";
}
string description() const {
return "Exits the CLI, gracefully stopping any services or running jobs.";
}
}

View File

@ -1,3 +0,0 @@
module command;
public import command.base;