From d60f7142e8a801e8c007921589a9a5a04fb7b423 Mon Sep 17 00:00:00 2001 From: Andrew Lalis Date: Fri, 3 Feb 2023 14:09:38 +0100 Subject: [PATCH] Added CDN client to sample data loader, and finalized base implementation of CDN. --- gymboard-api/README.md | 2 +- gymboard-api/pom.xml | 1 + .../controller/GymController.java | 22 +- .../controller/SubmissionController.java | 6 - .../dto/ExerciseSubmissionPayload.java | 2 +- .../dto/ExerciseSubmissionResponse.java | 4 +- .../controller/dto/UploadedFileResponse.java | 3 - .../dao/StoredFileRepository.java | 9 - .../ExerciseSubmissionRepository.java | 3 - .../ExerciseSubmissionTempFileRepository.java | 17 - ...ExerciseSubmissionVideoFileRepository.java | 18 - .../gymboard_api/model/StoredFile.java | 68 --- .../model/exercise/ExerciseSubmission.java | 57 +-- .../exercise/ExerciseSubmissionTempFile.java | 58 --- .../exercise/ExerciseSubmissionVideoFile.java | 41 -- .../gymboard_api/service/UploadService.java | 76 --- .../service/cdn_client/CdnClient.java | 47 ++ .../service/cdn_client/UploadsClient.java | 16 + .../submission/ExerciseSubmissionService.java | 53 +- .../SubmissionProcessingService.java | 237 --------- .../{model => util}/SampleDataLoader.java | 87 ++-- .../application-development.properties | 1 + gymboard-cdn/.gitignore | 2 + .../nl/andrewlalis/gymboardcdn/Config.java | 10 +- .../gymboardcdn/api/FileMetadataResponse.java | 9 + .../gymboardcdn/api/FileUploadResponse.java | 2 +- .../gymboardcdn/api/UploadController.java | 31 +- .../gymboardcdn/model/StoredFile.java | 29 +- .../model/StoredFileRepository.java | 6 +- .../model/VideoProcessingTask.java | 2 +- .../gymboardcdn/service/FileService.java | 54 +-- .../gymboardcdn/service/UploadService.java | 72 ++- .../service/VideoProcessingService.java | 24 +- .../nl/andrewlalis/gymboardcdn/util/ULID.java | 456 ++++++++++++++++++ .../application-development.properties | 1 + .../src/test/resources/application.properties | 1 + 36 files changed, 755 insertions(+), 772 deletions(-) delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/UploadedFileResponse.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/StoredFileRepository.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionTempFileRepository.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionVideoFileRepository.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/StoredFile.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionTempFile.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionVideoFile.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/UploadService.java create mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/CdnClient.java create mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/UploadsClient.java delete mode 100644 gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/SubmissionProcessingService.java rename gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/{model => util}/SampleDataLoader.java (71%) create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileMetadataResponse.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/util/ULID.java diff --git a/gymboard-api/README.md b/gymboard-api/README.md index 343e41b..581dd03 100644 --- a/gymboard-api/README.md +++ b/gymboard-api/README.md @@ -4,7 +4,7 @@ An HTTP/REST API powered by Java and Spring Boot. This API serves as the main en ## Development -To ease development, `nl.andrewlalis.gymboard_api.model.SampleDataLoader` will run on startup and populate the database with some sample entities. You can regenerate this data by manually deleting the database, and deleting the `.sample_data` marker file that's generated in the project directory. +To ease development, `nl.andrewlalis.gymboard_api.util.SampleDataLoader` will run on startup and populate the database with some sample entities. You can regenerate this data by manually deleting the database, and deleting the `.sample_data` marker file that's generated in the project directory. ## ULIDs diff --git a/gymboard-api/pom.xml b/gymboard-api/pom.xml index 43d5ff0..39829b3 100644 --- a/gymboard-api/pom.xml +++ b/gymboard-api/pom.xml @@ -42,6 +42,7 @@ spring-boot-starter-mail + org.postgresql postgresql diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/GymController.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/GymController.java index 351fa94..5cf4cbd 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/GymController.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/GymController.java @@ -1,12 +1,12 @@ package nl.andrewlalis.gymboard_api.controller; -import nl.andrewlalis.gymboard_api.controller.dto.*; +import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; +import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload; +import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse; +import nl.andrewlalis.gymboard_api.controller.dto.GymResponse; import nl.andrewlalis.gymboard_api.service.GymService; -import nl.andrewlalis.gymboard_api.service.UploadService; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; -import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; import java.util.List; @@ -17,14 +17,10 @@ import java.util.List; @RequestMapping(path = "/gyms/{compoundId}") public class GymController { private final GymService gymService; - private final UploadService uploadService; private final ExerciseSubmissionService submissionService; - public GymController(GymService gymService, - UploadService uploadService, - ExerciseSubmissionService submissionService) { + public GymController(GymService gymService, ExerciseSubmissionService submissionService) { this.gymService = gymService; - this.uploadService = uploadService; this.submissionService = submissionService; } @@ -45,12 +41,4 @@ public class GymController { ) { return submissionService.createSubmission(CompoundGymId.parse(compoundId), payload); } - - @PostMapping(path = "/submissions/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public UploadedFileResponse uploadVideo( - @PathVariable String compoundId, - @RequestParam MultipartFile file - ) { - return uploadService.handleSubmissionUpload(CompoundGymId.parse(compoundId), file); - } } diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/SubmissionController.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/SubmissionController.java index ddebd8e..3721c7e 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/SubmissionController.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/SubmissionController.java @@ -1,6 +1,5 @@ package nl.andrewlalis.gymboard_api.controller; -import jakarta.servlet.http.HttpServletResponse; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; import org.springframework.web.bind.annotation.GetMapping; @@ -21,9 +20,4 @@ public class SubmissionController { public ExerciseSubmissionResponse getSubmission(@PathVariable String submissionId) { return submissionService.getSubmission(submissionId); } - - @GetMapping(path = "/{submissionId}/video") - public void getSubmissionVideo(@PathVariable String submissionId, HttpServletResponse response) { - submissionService.streamVideo(submissionId, response); - } } diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionPayload.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionPayload.java index 2d8ae27..690e405 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionPayload.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionPayload.java @@ -6,5 +6,5 @@ public record ExerciseSubmissionPayload( float weight, String weightUnit, int reps, - long videoId + String videoFileId ) {} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionResponse.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionResponse.java index 891556a..50c0190 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionResponse.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/ExerciseSubmissionResponse.java @@ -9,7 +9,7 @@ public record ExerciseSubmissionResponse( String createdAt, GymSimpleResponse gym, ExerciseResponse exercise, - String status, + String videoFileId, String submitterName, double rawWeight, String weightUnit, @@ -22,7 +22,7 @@ public record ExerciseSubmissionResponse( submission.getCreatedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), new GymSimpleResponse(submission.getGym()), new ExerciseResponse(submission.getExercise()), - submission.getStatus().name(), + submission.getVideoFileId(), submission.getSubmitterName(), submission.getRawWeight().doubleValue(), submission.getWeightUnit().name(), diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/UploadedFileResponse.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/UploadedFileResponse.java deleted file mode 100644 index b3f3e0b..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/controller/dto/UploadedFileResponse.java +++ /dev/null @@ -1,3 +0,0 @@ -package nl.andrewlalis.gymboard_api.controller.dto; - -public record UploadedFileResponse(long id) {} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/StoredFileRepository.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/StoredFileRepository.java deleted file mode 100644 index 0c1cfe0..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/StoredFileRepository.java +++ /dev/null @@ -1,9 +0,0 @@ -package nl.andrewlalis.gymboard_api.dao; - -import nl.andrewlalis.gymboard_api.model.StoredFile; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; - -@Repository -public interface StoredFileRepository extends JpaRepository { -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionRepository.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionRepository.java index 3f86920..2f02130 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionRepository.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionRepository.java @@ -5,9 +5,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.stereotype.Repository; -import java.util.List; - @Repository public interface ExerciseSubmissionRepository extends JpaRepository, JpaSpecificationExecutor { - List findAllByStatus(ExerciseSubmission.Status status); } diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionTempFileRepository.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionTempFileRepository.java deleted file mode 100644 index 12d2948..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionTempFileRepository.java +++ /dev/null @@ -1,17 +0,0 @@ -package nl.andrewlalis.gymboard_api.dao.exercise; - -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; - -import java.time.LocalDateTime; -import java.util.List; -import java.util.Optional; - -@Repository -public interface ExerciseSubmissionTempFileRepository extends JpaRepository { - List findAllByCreatedAtBefore(LocalDateTime timestamp); - Optional findBySubmission(ExerciseSubmission submission); - boolean existsByPath(String path); -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionVideoFileRepository.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionVideoFileRepository.java deleted file mode 100644 index 4a47ba5..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/dao/exercise/ExerciseSubmissionVideoFileRepository.java +++ /dev/null @@ -1,18 +0,0 @@ -package nl.andrewlalis.gymboard_api.dao.exercise; - -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile; -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Query; -import org.springframework.stereotype.Repository; - -import java.util.Optional; - -@Repository -public interface ExerciseSubmissionVideoFileRepository extends JpaRepository { - Optional findBySubmission(ExerciseSubmission submission); - - @Query("SELECT f FROM ExerciseSubmissionVideoFile f WHERE " + - "f.submission.id = :submissionId AND f.submission.complete = true") - Optional findByCompletedSubmissionId(String submissionId); -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/StoredFile.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/StoredFile.java deleted file mode 100644 index 6154a00..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/StoredFile.java +++ /dev/null @@ -1,68 +0,0 @@ -package nl.andrewlalis.gymboard_api.model; - -import jakarta.persistence.*; -import org.hibernate.annotations.CreationTimestamp; - -import java.time.LocalDateTime; - -/** - * Base class for file storage. Files (mostly gym videos) are stored in the - * database as blobs, after they've been pre-processed with compression and/or - * resizing. - */ -@Entity -@Table(name = "stored_file") -public class StoredFile { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @CreationTimestamp - private LocalDateTime createdAt; - - @Column(nullable = false, updatable = false) - private String filename; - - @Column(nullable = false, updatable = false) - private String mimeType; - - @Column(nullable = false, updatable = false) - private long size; - - @Lob - @Column(nullable = false, updatable = false) - private byte[] content; - - public StoredFile() {} - - public StoredFile(String filename, String mimeType, long size, byte[] content) { - this.filename = filename; - this.mimeType = mimeType; - this.size = size; - this.content = content; - } - - public Long getId() { - return id; - } - - public LocalDateTime getCreatedAt() { - return createdAt; - } - - public String getFilename() { - return filename; - } - - public String getMimeType() { - return mimeType; - } - - public long getSize() { - return size; - } - - public byte[] getContent() { - return content; - } -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmission.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmission.java index 58a7e00..d767918 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmission.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmission.java @@ -10,24 +10,6 @@ import java.time.LocalDateTime; @Entity @Table(name = "exercise_submission") public class ExerciseSubmission { - /** - * The status of a submission. - *
    - *
  • Each submission starts as WAITING.
    • - *
  • The status changes to PROCESSING once it's picked up for processing.
    • - *
  • If processing fails, the status changes to FAILED.
    • - *
  • If processing is successful, the status changes to COMPLETED.
    • - *
  • Once a completed submission is verified either automatically or manually, it's set to VERIFIED.
    • - *
- */ - public enum Status { - WAITING, - PROCESSING, - FAILED, - COMPLETED, - VERIFIED - } - public enum WeightUnit { KG, LBS @@ -46,9 +28,13 @@ public class ExerciseSubmission { @ManyToOne(optional = false, fetch = FetchType.LAZY) private Exercise exercise; - @Enumerated(EnumType.STRING) - @Column(nullable = false) - private Status status; + /** + * The id of the video file that was submitted for this submission. It lives + * on the gymboard-cdn service as a stored file, which can be + * accessed via GET https://CDN-HOST/files/{videoFileId}. + */ + @Column(nullable = false, updatable = false, length = 26) + private String videoFileId; @Column(nullable = false, updatable = false, length = 63) private String submitterName; @@ -66,27 +52,18 @@ public class ExerciseSubmission { @Column(nullable = false) private int reps; - /** - * Marker that's used to simplify queries where we just want submissions - * that are in a status that's not WAITING, PROCESSING, or FAILED, i.e. - * a successful submission that's been processed. - */ - @Column(nullable = false) - private boolean complete; - public ExerciseSubmission() {} - public ExerciseSubmission(String id, Gym gym, Exercise exercise, String submitterName, BigDecimal rawWeight, WeightUnit unit, BigDecimal metricWeight, int reps) { + public ExerciseSubmission(String id, Gym gym, Exercise exercise, String videoFileId, String submitterName, BigDecimal rawWeight, WeightUnit unit, BigDecimal metricWeight, int reps) { this.id = id; this.gym = gym; this.exercise = exercise; + this.videoFileId = videoFileId; this.submitterName = submitterName; this.rawWeight = rawWeight; this.weightUnit = unit; this.metricWeight = metricWeight; this.reps = reps; - this.status = Status.WAITING; - this.complete = false; } public String getId() { @@ -105,12 +82,8 @@ public class ExerciseSubmission { return exercise; } - public Status getStatus() { - return status; - } - - public void setStatus(Status status) { - this.status = status; + public String getVideoFileId() { + return videoFileId; } public String getSubmitterName() { @@ -132,12 +105,4 @@ public class ExerciseSubmission { public int getReps() { return reps; } - - public boolean isComplete() { - return complete; - } - - public void setComplete(boolean complete) { - this.complete = complete; - } } diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionTempFile.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionTempFile.java deleted file mode 100644 index fdf292e..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionTempFile.java +++ /dev/null @@ -1,58 +0,0 @@ -package nl.andrewlalis.gymboard_api.model.exercise; - -import jakarta.persistence.*; -import org.hibernate.annotations.CreationTimestamp; - -import java.time.LocalDateTime; - -/** - * Tracks the temporary file on disk that's stored while a user is preparing - * their submission. This file will be removed after the submission is - * processed. - */ -@Entity -@Table(name = "exercise_submission_temp_file") -public class ExerciseSubmissionTempFile { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @CreationTimestamp - private LocalDateTime createdAt; - - @Column(nullable = false, updatable = false, length = 1024) - private String path; - - /** - * The submission that this temporary file is for. This will initially be - * null, but will be set as soon as the submission is finalized. - */ - @OneToOne(fetch = FetchType.LAZY) - private ExerciseSubmission submission; - - public ExerciseSubmissionTempFile() {} - - public ExerciseSubmissionTempFile(String path) { - this.path = path; - } - - public Long getId() { - return id; - } - - public LocalDateTime getCreatedAt() { - return createdAt; - } - - public String getPath() { - return path; - } - - public ExerciseSubmission getSubmission() { - return submission; - } - - public void setSubmission(ExerciseSubmission submission) { - this.submission = submission; - } -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionVideoFile.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionVideoFile.java deleted file mode 100644 index 076aaaf..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/exercise/ExerciseSubmissionVideoFile.java +++ /dev/null @@ -1,41 +0,0 @@ -package nl.andrewlalis.gymboard_api.model.exercise; - -import jakarta.persistence.*; -import nl.andrewlalis.gymboard_api.model.StoredFile; - -/** - * An entity which links an {@link ExerciseSubmission} to a {@link nl.andrewlalis.gymboard_api.model.StoredFile} - * containing the video that was submitted along with the submission. - */ -@Entity -@Table(name = "exercise_submission_video_file") -public class ExerciseSubmissionVideoFile { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; - - @OneToOne(optional = false, fetch = FetchType.LAZY) - private ExerciseSubmission submission; - - @OneToOne(optional = false, fetch = FetchType.LAZY, orphanRemoval = true) - private StoredFile file; - - public ExerciseSubmissionVideoFile() {} - - public ExerciseSubmissionVideoFile(ExerciseSubmission submission, StoredFile file) { - this.submission = submission; - this.file = file; - } - - public Long getId() { - return id; - } - - public ExerciseSubmission getSubmission() { - return submission; - } - - public StoredFile getFile() { - return file; - } -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/UploadService.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/UploadService.java deleted file mode 100644 index 6095139..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/UploadService.java +++ /dev/null @@ -1,76 +0,0 @@ -package nl.andrewlalis.gymboard_api.service; - -import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; -import nl.andrewlalis.gymboard_api.controller.dto.UploadedFileResponse; -import nl.andrewlalis.gymboard_api.dao.GymRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository; -import nl.andrewlalis.gymboard_api.model.Gym; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.server.ResponseStatusException; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -/** - * Service for handling large file uploads. - */ -@Service -public class UploadService { - public static final Path SUBMISSION_TEMP_FILE_DIR = Path.of("exercise_submission_temp_files"); - private static final String[] ALLOWED_VIDEO_TYPES = { - "video/mp4" - }; - - private final ExerciseSubmissionTempFileRepository tempFileRepository; - private final GymRepository gymRepository; - - public UploadService(ExerciseSubmissionTempFileRepository tempFileRepository, GymRepository gymRepository) { - this.tempFileRepository = tempFileRepository; - this.gymRepository = gymRepository; - } - - /** - * Handles the upload of an exercise submission's video file by saving the - * file to a temporary location, and recording that location in the - * database for when the exercise submission is completed. We'll only do - * the computationally expensive video processing if a user successfully - * submits their submission; otherwise, the raw video is discarded after a - * while. - * @param gymId The gym's id. - * @param multipartFile The uploaded file. - * @return A response containing the uploaded file's id, to be included in - * the user's submission. - */ - @Transactional - public UploadedFileResponse handleSubmissionUpload(CompoundGymId gymId, MultipartFile multipartFile) { - Gym gym = gymRepository.findByCompoundId(gymId) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); - // TODO: Check that user is allowed to upload. - boolean fileTypeAcceptable = false; - for (String allowedType : ALLOWED_VIDEO_TYPES) { - if (allowedType.equalsIgnoreCase(multipartFile.getContentType())) { - fileTypeAcceptable = true; - break; - } - } - if (!fileTypeAcceptable) { - throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid content type."); - } - try { - if (!Files.exists(SUBMISSION_TEMP_FILE_DIR)) { - Files.createDirectory(SUBMISSION_TEMP_FILE_DIR); - } - Path tempFilePath = Files.createTempFile(SUBMISSION_TEMP_FILE_DIR, null, null); - multipartFile.transferTo(tempFilePath); - ExerciseSubmissionTempFile tempFileEntity = tempFileRepository.save(new ExerciseSubmissionTempFile(tempFilePath.toString())); - return new UploadedFileResponse(tempFileEntity.getId()); - } catch (IOException e) { - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "File upload failed.", e); - } - } -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/CdnClient.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/CdnClient.java new file mode 100644 index 0000000..d328eae --- /dev/null +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/CdnClient.java @@ -0,0 +1,47 @@ +package nl.andrewlalis.gymboard_api.service.cdn_client; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; + +public class CdnClient { + private final HttpClient httpClient; + private final String baseUrl; + private final ObjectMapper objectMapper; + + public final UploadsClient uploads; + + public CdnClient(String baseUrl) { + this.httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(3)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + this.baseUrl = baseUrl; + this.objectMapper = new ObjectMapper(); + this.uploads = new UploadsClient(this); + } + + public T get(String urlPath, Class responseType) throws IOException, InterruptedException { + HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath)) + .GET() + .build(); + HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); + return objectMapper.readValue(response.body(), responseType); + } + + public T postFile(String urlPath, Path filePath, String contentType, Class responseType) throws IOException, InterruptedException { + HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath)) + .POST(HttpRequest.BodyPublishers.ofFile(filePath)) + .header("Content-Type", contentType) + .header("X-Gymboard-Filename", filePath.getFileName().toString()) + .build(); + HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); + return objectMapper.readValue(response.body(), responseType); + } +} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/UploadsClient.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/UploadsClient.java new file mode 100644 index 0000000..1bebc65 --- /dev/null +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/cdn_client/UploadsClient.java @@ -0,0 +1,16 @@ +package nl.andrewlalis.gymboard_api.service.cdn_client; + +import java.nio.file.Path; + +public record UploadsClient(CdnClient client) { + public record FileUploadResponse(String id) {} + public record VideoProcessingTaskStatusResponse(String status) {} + + public FileUploadResponse uploadVideo(Path filePath, String contentType) throws Exception { + return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class); + } + + public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) throws Exception { + return client.get("/uploads/video/" + id + "/status", VideoProcessingTaskStatusResponse.class); + } +} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/ExerciseSubmissionService.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/ExerciseSubmissionService.java index bc11936..96ff7b5 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/ExerciseSubmissionService.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/ExerciseSubmissionService.java @@ -1,19 +1,14 @@ package nl.andrewlalis.gymboard_api.service.submission; -import jakarta.servlet.http.HttpServletResponse; import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse; import nl.andrewlalis.gymboard_api.dao.GymRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionVideoFileRepository; import nl.andrewlalis.gymboard_api.model.Gym; import nl.andrewlalis.gymboard_api.model.exercise.Exercise; import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile; import nl.andrewlalis.gymboard_api.util.ULID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +17,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.server.ResponseStatusException; -import java.io.IOException; import java.math.BigDecimal; /** @@ -36,21 +30,15 @@ public class ExerciseSubmissionService { private final GymRepository gymRepository; private final ExerciseRepository exerciseRepository; private final ExerciseSubmissionRepository exerciseSubmissionRepository; - private final ExerciseSubmissionTempFileRepository tempFileRepository; - private final ExerciseSubmissionVideoFileRepository submissionVideoFileRepository; private final ULID ulid; public ExerciseSubmissionService(GymRepository gymRepository, ExerciseRepository exerciseRepository, ExerciseSubmissionRepository exerciseSubmissionRepository, - ExerciseSubmissionTempFileRepository tempFileRepository, - ExerciseSubmissionVideoFileRepository submissionVideoFileRepository, ULID ulid) { this.gymRepository = gymRepository; this.exerciseRepository = exerciseRepository; this.exerciseSubmissionRepository = exerciseSubmissionRepository; - this.tempFileRepository = tempFileRepository; - this.submissionVideoFileRepository = submissionVideoFileRepository; this.ulid = ulid; } @@ -61,33 +49,11 @@ public class ExerciseSubmissionService { return new ExerciseSubmissionResponse(submission); } - @Transactional(readOnly = true) - public void streamVideo(String submissionId, HttpServletResponse response) { - ExerciseSubmissionVideoFile videoFile = submissionVideoFileRepository.findByCompletedSubmissionId(submissionId) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); - response.setContentType(videoFile.getFile().getMimeType()); - response.setContentLengthLong(videoFile.getFile().getSize()); - try { - response.getOutputStream().write(videoFile.getFile().getContent()); - } catch (IOException e) { - log.error("Failed to write submission video file to response.", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); - } - } - /** - * Handles the creation of a new exercise submission. This involves a few steps: - *
    - *
  1. Pre-fetch all of the referenced data, like exercise and video file.
    2. - *
  2. Check that the submission is legitimate.
    3. - *
  3. Save the submission. (With the WAITING status initially.)
    4. - *
  4. Sometime soon, {@link SubmissionProcessingService#processWaitingSubmissions()} will pick up the submission for processing.
    5. - *
- * Once the asynchronous submission processing is complete, the submission - * status will change to COMPLETE. + * Handles the creation of a new exercise submission. * @param id The gym id. * @param payload The submission data. - * @return The saved submission, which will be in the PROCESSING state at first. + * @return The saved submission. */ @Transactional public ExerciseSubmissionResponse createSubmission(CompoundGymId id, ExerciseSubmissionPayload payload) { @@ -95,10 +61,8 @@ public class ExerciseSubmissionService { .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); Exercise exercise = exerciseRepository.findById(payload.exerciseShortName()) .orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid exercise.")); - ExerciseSubmissionTempFile tempFile = tempFileRepository.findById(payload.videoId()) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid video id.")); - validateSubmission(payload, exercise, tempFile); + // TODO: Validate the submission data. // Create the submission. BigDecimal rawWeight = BigDecimal.valueOf(payload.weight()); @@ -107,26 +71,17 @@ public class ExerciseSubmissionService { if (unit == ExerciseSubmission.WeightUnit.LBS) { metricWeight = metricWeight.multiply(new BigDecimal("0.45359237")); } - ExerciseSubmission submission = exerciseSubmissionRepository.saveAndFlush(new ExerciseSubmission( ulid.nextULID(), gym, exercise, + payload.videoFileId(), payload.name(), rawWeight, unit, metricWeight, payload.reps() )); - // Then link it to the temporary video file so the async task can find it. - tempFile.setSubmission(submission); - tempFileRepository.save(tempFile); - // The submission will be picked up eventually to be processed. - return new ExerciseSubmissionResponse(submission); } - - private void validateSubmission(ExerciseSubmissionPayload payload, Exercise exercise, ExerciseSubmissionTempFile tempFile) { - // TODO: Implement this validation. - } } diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/SubmissionProcessingService.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/SubmissionProcessingService.java deleted file mode 100644 index 8690662..0000000 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/service/submission/SubmissionProcessingService.java +++ /dev/null @@ -1,237 +0,0 @@ -package nl.andrewlalis.gymboard_api.service.submission; - -import nl.andrewlalis.gymboard_api.dao.StoredFileRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository; -import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionVideoFileRepository; -import nl.andrewlalis.gymboard_api.model.StoredFile; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile; -import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile; -import nl.andrewlalis.gymboard_api.service.CommandFailedException; -import nl.andrewlalis.gymboard_api.service.UploadService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -/** - * This service is responsible for the logic of processing new exercise - * submissions and tasks immediately related to that. - */ -@Service -public class SubmissionProcessingService { - private static final Logger log = LoggerFactory.getLogger(SubmissionProcessingService.class); - - private final ExerciseSubmissionRepository exerciseSubmissionRepository; - private final Executor taskExecutor; - private final ExerciseSubmissionTempFileRepository tempFileRepository; - private final ExerciseSubmissionVideoFileRepository videoFileRepository; - private final StoredFileRepository fileRepository; - - public SubmissionProcessingService(ExerciseSubmissionRepository exerciseSubmissionRepository, - Executor taskExecutor, - ExerciseSubmissionTempFileRepository tempFileRepository, - ExerciseSubmissionVideoFileRepository videoFileRepository, - StoredFileRepository fileRepository) { - this.exerciseSubmissionRepository = exerciseSubmissionRepository; - this.taskExecutor = taskExecutor; - this.tempFileRepository = tempFileRepository; - this.videoFileRepository = videoFileRepository; - this.fileRepository = fileRepository; - } - - /** - * Simple scheduled task that periodically checks for new submissions - * that are waiting to be processed, and queues tasks to do so. - */ - @Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS) - public void processWaitingSubmissions() { - List waitingSubmissions = exerciseSubmissionRepository.findAllByStatus(ExerciseSubmission.Status.WAITING); - for (var submission : waitingSubmissions) { - taskExecutor.execute(() -> processSubmission(submission.getId())); - } - } - - /** - * Asynchronous task that's started after a submission is submitted, which - * handles video processing and anything else that might need to be done - * before the submission can be marked as COMPLETED. - *

- * Note: This method is intentionally NOT transactional, since it may - * have a long duration, and we want real-time status updates. - *

- * @param submissionId The submission's id. - */ - private void processSubmission(String submissionId) { - log.info("Starting processing of submission {}.", submissionId); - // First try and fetch the submission. - Optional optionalSubmission = exerciseSubmissionRepository.findById(submissionId); - if (optionalSubmission.isEmpty()) { - log.warn("Submission id {} is not associated with a submission.", submissionId); - return; - } - ExerciseSubmission submission = optionalSubmission.get(); - if (submission.getStatus() != ExerciseSubmission.Status.WAITING) { - log.warn("Submission {} cannot be processed because its status {} is not WAITING.", submission.getId(), submission.getStatus()); - return; - } - - // Set the status to processing. - submission.setStatus(ExerciseSubmission.Status.PROCESSING); - exerciseSubmissionRepository.saveAndFlush(submission); - - // Then try and fetch the temporary video file associated with it. - Optional optionalTempFile = tempFileRepository.findBySubmission(submission); - if (optionalTempFile.isEmpty()) { - log.warn("Submission {} failed because the temporary video file couldn't be found.", submission.getId()); - submission.setStatus(ExerciseSubmission.Status.FAILED); - exerciseSubmissionRepository.save(submission); - return; - } - ExerciseSubmissionTempFile tempFile = optionalTempFile.get(); - Path tempFilePath = Path.of(tempFile.getPath()); - if (!Files.exists(tempFilePath) || !Files.isReadable(tempFilePath)) { - log.error("Submission {} failed because the temporary video file {} isn't readable.", submission.getId(), tempFilePath); - submission.setStatus(ExerciseSubmission.Status.FAILED); - exerciseSubmissionRepository.saveAndFlush(submission); - return; - } - - // Now we can try to process the video file into a compressed format that can be stored in the DB. - Path dir = UploadService.SUBMISSION_TEMP_FILE_DIR; - String tempFileName = tempFilePath.getFileName().toString(); - String tempFileBaseName = tempFileName.substring(0, tempFileName.length() - ".tmp".length()); - Path outFilePath = dir.resolve(tempFileBaseName + "-out.mp4"); - StoredFile file; - try { - processVideo(dir, tempFilePath, outFilePath); - file = fileRepository.save(new StoredFile( - "compressed.mp4", - "video/mp4", - Files.size(outFilePath), - Files.readAllBytes(outFilePath) - )); - } catch (Exception e) { - log.error(""" - Video processing failed for submission {}: - Input file: {} - Output file: {} - Exception message: {}""", - submission.getId(), - tempFilePath, - outFilePath, - e.getMessage() - ); - submission.setStatus(ExerciseSubmission.Status.FAILED); - exerciseSubmissionRepository.saveAndFlush(submission); - return; - } - - // After we've saved the processed file, we can link it to the submission, and set the submission's status. - videoFileRepository.save(new ExerciseSubmissionVideoFile( - submission, - file - )); - submission.setStatus(ExerciseSubmission.Status.COMPLETED); - submission.setComplete(true); - exerciseSubmissionRepository.save(submission); - // And delete the temporary files. - try { - Files.delete(tempFilePath); - Files.delete(outFilePath); - tempFileRepository.delete(tempFile); - } catch (IOException e) { - log.error("Couldn't delete temporary files after submission completed.", e); - } - log.info("Processing of submission {} complete.", submission.getId()); - } - - /** - * Uses the `ffmpeg` system command to process a raw input video and produce - * a compressed, reduced-size output video that's ready for usage in the - * application. - * @param dir The working directory. - * @param inFile The input file to read from. - * @param outFile The output file to write to. MUST have a ".mp4" extension. - * @throws IOException If a filesystem error occurs. - * @throws CommandFailedException If the ffmpeg command fails. - * @throws InterruptedException If the ffmpeg command is interrupted. - */ - private void processVideo(Path dir, Path inFile, Path outFile) throws IOException, InterruptedException { - Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log"); - Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log"); - final String[] command = { - "ffmpeg", "-i", inFile.getFileName().toString(), - "-vf", "scale=640x480:flags=lanczos", - "-vcodec", "libx264", - "-crf", "28", - outFile.getFileName().toString() - }; - - long startSize = Files.size(inFile); - Instant startTime = Instant.now(); - - Process ffmpegProcess = new ProcessBuilder() - .command(command) - .redirectOutput(tmpStdout.toFile()) - .redirectError(tmpStderr.toFile()) - .directory(dir.toFile()) - .start(); - int result = ffmpegProcess.waitFor(); - if (result != 0) throw new CommandFailedException(command, result, tmpStdout, tmpStderr); - - long endSize = Files.size(outFile); - Duration dur = Duration.between(startTime, Instant.now()); - double reductionFactor = startSize / (double) endSize; - String reductionFactorStr = String.format("%.3f%%", reductionFactor * 100); - log.info("Processed video from {} bytes to {} bytes in {} seconds, {} reduction.", startSize, endSize, dur.getSeconds(), reductionFactorStr); - - // Delete the logs if everything was successful. - Files.deleteIfExists(tmpStdout); - Files.deleteIfExists(tmpStderr); - } - - @Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES) - public void removeOldUploadedFiles() { - // First remove any temp files older than 10 minutes. - LocalDateTime cutoff = LocalDateTime.now().minusMinutes(10); - var tempFiles = tempFileRepository.findAllByCreatedAtBefore(cutoff); - for (var file : tempFiles) { - try { - Files.deleteIfExists(Path.of(file.getPath())); - tempFileRepository.delete(file); - log.info("Removed temporary submission file {} at {}.", file.getId(), file.getPath()); - } catch (IOException e) { - log.error(String.format("Could not delete submission temp file %d at %s.", file.getId(), file.getPath()), e); - } - } - - // Then remove any files in the directory which don't correspond to a valid file in the db. - if (Files.notExists(UploadService.SUBMISSION_TEMP_FILE_DIR)) return; - try (var s = Files.list(UploadService.SUBMISSION_TEMP_FILE_DIR)) { - for (var path : s.toList()) { - if (!tempFileRepository.existsByPath(path.toString())) { - try { - Files.delete(path); - } catch (IOException e) { - log.error("Couldn't delete orphan temp file: " + path, e); - } - } - } - } catch (IOException e) { - log.error("Couldn't get list of temp files.", e); - } - } -} diff --git a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/SampleDataLoader.java b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/util/SampleDataLoader.java similarity index 71% rename from gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/SampleDataLoader.java rename to gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/util/SampleDataLoader.java index 9433701..3a3d491 100644 --- a/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/model/SampleDataLoader.java +++ b/gymboard-api/src/main/java/nl/andrewlalis/gymboard_api/util/SampleDataLoader.java @@ -1,4 +1,4 @@ -package nl.andrewlalis.gymboard_api.model; +package nl.andrewlalis.gymboard_api.util; import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload; @@ -9,20 +9,24 @@ import nl.andrewlalis.gymboard_api.dao.GymRepository; import nl.andrewlalis.gymboard_api.dao.auth.RoleRepository; import nl.andrewlalis.gymboard_api.dao.auth.UserRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository; +import nl.andrewlalis.gymboard_api.model.City; +import nl.andrewlalis.gymboard_api.model.Country; +import nl.andrewlalis.gymboard_api.model.GeoPoint; +import nl.andrewlalis.gymboard_api.model.Gym; import nl.andrewlalis.gymboard_api.model.auth.Role; import nl.andrewlalis.gymboard_api.model.auth.User; import nl.andrewlalis.gymboard_api.model.exercise.Exercise; import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; -import nl.andrewlalis.gymboard_api.service.UploadService; import nl.andrewlalis.gymboard_api.service.auth.UserService; +import nl.andrewlalis.gymboard_api.service.cdn_client.CdnClient; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.mock.web.MockMultipartFile; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -31,7 +35,8 @@ import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Path; -import java.util.function.Consumer; +import java.util.HashSet; +import java.util.Set; /** * Simple component that loads sample data that's useful when testing the application. @@ -44,25 +49,25 @@ public class SampleDataLoader implements ApplicationListener { exerciseRepository.save(new Exercise(record.get(0), record.get(1))); }); @@ -108,6 +113,13 @@ public class SampleDataLoader implements ApplicationListener videoIds = new HashSet<>(); + loadCsv("submissions", record -> { var exercise = exerciseRepository.findById(record.get(0)).orElseThrow(); BigDecimal weight = new BigDecimal(record.get(1)); @@ -117,26 +129,34 @@ public class SampleDataLoader implements ApplicationListener removalSet = new HashSet<>(); + for (var videoId : videoIds) { + String status = cdnClient.uploads.getVideoProcessingStatus(videoId).status(); + if (status.equalsIgnoreCase("COMPLETED") || status.equalsIgnoreCase("FAILED")) { + removalSet.add(videoId); + } + } + videoIds.removeAll(removalSet); + Thread.sleep(1000); + } + loadCsv("users", record -> { String email = record.get(0); String password = record.get(1); @@ -156,12 +176,21 @@ public class SampleDataLoader implements ApplicationListener recordConsumer) throws IOException { + @FunctionalInterface + interface ThrowableConsumer { + void accept(T item) throws Exception; + } + + private void loadCsv(String csvName, ThrowableConsumer recordConsumer) throws IOException { String path = "sample_data/" + csvName + ".csv"; log.info("Loading data from {}...", path); var reader = new FileReader(path); for (var record : CSVFormat.DEFAULT.parse(reader)) { - recordConsumer.accept(record); + try { + recordConsumer.accept(record); + } catch (Exception e) { + e.printStackTrace(); + } } } } diff --git a/gymboard-api/src/main/resources/application-development.properties b/gymboard-api/src/main/resources/application-development.properties index a891663..580e6ab 100644 --- a/gymboard-api/src/main/resources/application-development.properties +++ b/gymboard-api/src/main/resources/application-development.properties @@ -16,3 +16,4 @@ spring.mail.properties.mail.smtp.timeout=10000 app.auth.private-key-location=./private_key.der app.web-origin=http://localhost:9000 +app.cdn-origin=http://localhost:8082 diff --git a/gymboard-cdn/.gitignore b/gymboard-cdn/.gitignore index 549e00a..15848db 100644 --- a/gymboard-cdn/.gitignore +++ b/gymboard-cdn/.gitignore @@ -31,3 +31,5 @@ build/ ### VS Code ### .vscode/ + +cdn-files/ diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java index eb8a190..46b1121 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java @@ -1,5 +1,6 @@ package nl.andrewlalis.gymboardcdn; +import nl.andrewlalis.gymboardcdn.util.ULID; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,6 +15,8 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource; public class Config { @Value("${app.web-origin}") private String webOrigin; + @Value("${app.api-origin}") + private String apiOrigin; /** * Defines the CORS configuration for this API, which is to say that we @@ -27,11 +30,16 @@ public class Config { final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); final CorsConfiguration config = new CorsConfiguration(); config.setAllowCredentials(true); - // Don't do this in production, use a proper list of allowed origins config.addAllowedOriginPattern(webOrigin); + config.addAllowedOriginPattern(apiOrigin); config.addAllowedHeader("*"); config.addAllowedMethod("*"); source.registerCorsConfiguration("/**", config); return source; } + + @Bean + public ULID ulid() { + return new ULID(); + } } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileMetadataResponse.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileMetadataResponse.java new file mode 100644 index 0000000..6846cfa --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileMetadataResponse.java @@ -0,0 +1,9 @@ +package nl.andrewlalis.gymboardcdn.api; + +public record FileMetadataResponse( + String filename, + String mimeType, + long size, + String uploadedAt, + boolean availableForDownload +) {} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java index d120fb4..29ffb6b 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java @@ -1,5 +1,5 @@ package nl.andrewlalis.gymboardcdn.api; public record FileUploadResponse( - String identifier + String id ) {} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java index daee8ac..fe8ede2 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java @@ -1,9 +1,12 @@ package nl.andrewlalis.gymboardcdn.api; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import nl.andrewlalis.gymboardcdn.service.UploadService; -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; @RestController public class UploadController { @@ -13,13 +16,23 @@ public class UploadController { this.uploadService = uploadService; } - @PostMapping(path = "/uploads/video", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public FileUploadResponse uploadVideo(@RequestParam MultipartFile file) { - return uploadService.processableVideoUpload(file); + @PostMapping(path = "/uploads/video", consumes = {"video/mp4"}) + public FileUploadResponse uploadVideo(HttpServletRequest request) { + return uploadService.processableVideoUpload(request); } - @GetMapping(path = "/uploads/video/{identifier}/status") - public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String identifier) { - return uploadService.getVideoProcessingStatus(identifier); + @GetMapping(path = "/uploads/video/{id}/status") + public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) { + return uploadService.getVideoProcessingStatus(id); + } + + @GetMapping(path = "/files/{id}") + public void getFile(@PathVariable String id, HttpServletResponse response) { + uploadService.streamFile(id, response); + } + + @GetMapping(path = "/files/{id}/metadata") + public FileMetadataResponse getFileMetadata(@PathVariable String id) { + return uploadService.getFileMetadata(id); } } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java index b19fc62..8b87382 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java @@ -1,6 +1,9 @@ package nl.andrewlalis.gymboardcdn.model; -import jakarta.persistence.*; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; import org.hibernate.annotations.CreationTimestamp; import java.time.LocalDateTime; @@ -8,9 +11,12 @@ import java.time.LocalDateTime; @Entity @Table(name = "stored_file") public class StoredFile { + /** + * ULID-based unique file identifier. + */ @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private Long id; + @Column(nullable = false, updatable = false, length = 26) + private String id; @CreationTimestamp private LocalDateTime createdAt; @@ -27,13 +33,6 @@ public class StoredFile { @Column(nullable = false, updatable = false) private String name; - /** - * The internal id that's used to find this file wherever it's placed on - * our service's storage. It is universally unique. - */ - @Column(nullable = false, updatable = false, unique = true) - private String identifier; - /** * The type of the file. */ @@ -48,15 +47,15 @@ public class StoredFile { public StoredFile() {} - public StoredFile(String name, String identifier, String mimeType, long size, LocalDateTime uploadedAt) { + public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) { + this.id = id; this.name = name; - this.identifier = identifier; this.mimeType = mimeType; this.size = size; this.uploadedAt = uploadedAt; } - public Long getId() { + public String getId() { return id; } @@ -68,10 +67,6 @@ public class StoredFile { return name; } - public String getIdentifier() { - return identifier; - } - public String getMimeType() { return mimeType; } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java index 10a8d11..d4da008 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java @@ -3,10 +3,6 @@ package nl.andrewlalis.gymboardcdn.model; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; -import java.util.Optional; - @Repository -public interface StoredFileRepository extends JpaRepository { - Optional findByIdentifier(String identifier); - boolean existsByIdentifier(String identifier); +public interface StoredFileRepository extends JpaRepository { } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java index d8fc7ea..a18752f 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java @@ -46,7 +46,7 @@ public class VideoProcessingTask { * The identifier that will be used to identify the final video, if it * is processed successfully. */ - @Column(nullable = false) + @Column(nullable = false, updatable = false, length = 26) private String videoIdentifier; public VideoProcessingTask() {} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java index 7bbc614..9e6d56e 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java @@ -1,7 +1,7 @@ package nl.andrewlalis.gymboardcdn.service; -import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; -import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; +import nl.andrewlalis.gymboardcdn.model.StoredFile; +import nl.andrewlalis.gymboardcdn.util.ULID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -13,7 +13,6 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; -import java.util.Random; /** * The service that manages storing and retrieving files from a base filesystem. @@ -28,52 +27,29 @@ public class FileService { @Value("${app.files.temp-dir}") private String tempDir; - private final StoredFileRepository storedFileRepository; - private final VideoProcessingTaskRepository videoProcessingTaskRepository; + private final ULID ulid; - public FileService(StoredFileRepository storedFileRepository, VideoProcessingTaskRepository videoProcessingTaskRepository) { - this.storedFileRepository = storedFileRepository; - this.videoProcessingTaskRepository = videoProcessingTaskRepository; + public FileService(ULID ulid) { + this.ulid = ulid; } - public Path getStorageDirForTime(LocalDateTime time) throws IOException { - Path dir = getStorageDir() + public Path getStoragePathForFile(StoredFile file) throws IOException { + LocalDateTime time = file.getUploadedAt(); + Path dir = Path.of(storageDir) .resolve(Integer.toString(time.getYear())) .resolve(Integer.toString(time.getMonthValue())) .resolve(Integer.toString(time.getDayOfMonth())); if (Files.notExists(dir)) Files.createDirectories(dir); - return dir; + return dir.resolve(file.getId()); } public String createNewFileIdentifier() { - String ident = generateRandomIdentifier(); - int attempts = 0; - while (storedFileRepository.existsByIdentifier(ident) || videoProcessingTaskRepository.existsByVideoIdentifier(ident)) { - ident = generateRandomIdentifier(); - attempts++; - if (attempts > 10) { - log.warn("Took more than 10 attempts to generate a unique file identifier."); - } - if (attempts > 100) { - log.error("Couldn't generate a unique file identifier after 100 attempts. Quitting!"); - throw new RuntimeException("Couldn't generate a unique file identifier."); - } - } - return ident; + return ulid.nextULID(); } - private String generateRandomIdentifier() { - StringBuilder sb = new StringBuilder(9); - String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - Random rand = new Random(); - for (int i = 0; i < 9; i++) sb.append(alphabet.charAt(rand.nextInt(alphabet.length()))); - return sb.toString(); - } - - public Path saveToTempFile(MultipartFile file) throws IOException { + public Path saveToTempFile(InputStream in, String filename) throws IOException { Path tempDir = getTempDir(); String suffix = null; - String filename = file.getOriginalFilename(); if (filename != null) { int idx = filename.lastIndexOf('.'); if (idx >= 0) { @@ -81,14 +57,12 @@ public class FileService { } } Path tempFile = Files.createTempFile(tempDir, null, suffix); - file.transferTo(tempFile); + try (var out = Files.newOutputStream(tempFile)) { + in.transferTo(out); + } return tempFile; } - public Path saveToStorage(String filename, InputStream in) throws IOException { - throw new RuntimeException("Not implemented!"); - } - private Path getStorageDir() throws IOException { Path dir = Path.of(storageDir); if (Files.notExists(dir)) { diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java index 784379d..df551cd 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java @@ -1,7 +1,11 @@ package nl.andrewlalis.gymboardcdn.service; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse; import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse; +import nl.andrewlalis.gymboardcdn.model.StoredFile; import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; @@ -14,7 +18,9 @@ import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.time.format.DateTimeFormatter; @Service public class UploadService { @@ -32,11 +38,20 @@ public class UploadService { this.fileService = fileService; } + /** + * Handles uploading of a processable video file that will be processed + * before being stored in the system. + * @param request The request from which we can read the file. + * @return A response that contains an identifier that can be used to check + * the status of the video processing, and eventually fetch the video. + */ @Transactional - public FileUploadResponse processableVideoUpload(MultipartFile file) { + public FileUploadResponse processableVideoUpload(HttpServletRequest request) { Path tempFile; + String filename = request.getHeader("X-Gymboard-Filename"); + if (filename == null) filename = "unnamed.mp4"; try { - tempFile = fileService.saveToTempFile(file); + tempFile = fileService.saveToTempFile(request.getInputStream(), filename); } catch (IOException e) { log.error("Failed to save video upload to temp file.", e); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); @@ -44,17 +59,64 @@ public class UploadService { String identifier = fileService.createNewFileIdentifier(); videoTaskRepository.save(new VideoProcessingTask( VideoProcessingTask.Status.WAITING, - file.getOriginalFilename(), + filename, tempFile.toString(), identifier )); return new FileUploadResponse(identifier); } + /** + * Gets the status of a video processing task. + * @param id The video identifier. + * @return The status of the video processing task. + */ @Transactional(readOnly = true) - public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String identifier) { - VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(identifier) + public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) { + VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(id) .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); return new VideoProcessingTaskStatusResponse(task.getStatus().name()); } + + /** + * Streams the contents of a stored file to a client via the Http response. + * @param id The file's unique identifier. + * @param response The response to stream the content to. + */ + @Transactional(readOnly = true) + public void streamFile(String id, HttpServletResponse response) { + StoredFile file = storedFileRepository.findById(id) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + response.setContentType(file.getMimeType()); + response.setContentLengthLong(file.getSize()); + try { + Path filePath = fileService.getStoragePathForFile(file); + try (var in = Files.newInputStream(filePath)) { + in.transferTo(response.getOutputStream()); + } + } catch (IOException e) { + log.error("Failed to write file to response.", e); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @Transactional(readOnly = true) + public FileMetadataResponse getFileMetadata(String id) { + StoredFile file = storedFileRepository.findById(id) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + try { + Path filePath = fileService.getStoragePathForFile(file); + boolean exists = Files.exists(filePath); + return new FileMetadataResponse( + file.getName(), + file.getMimeType(), + file.getSize(), + file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), + exists + ); + } catch (IOException e) { + log.error("Couldn't get path to stored file.", e); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); + } + } } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java index d3e5799..66bddce 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java @@ -14,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; -import java.time.LocalDateTime; import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -77,21 +76,21 @@ public class VideoProcessingService { } // And finally, copy the output to the final location. - LocalDateTime uploadedAt = task.getCreatedAt(); try { - Path finalFilePath = fileService.getStorageDirForTime(uploadedAt) - .resolve(task.getVideoIdentifier()); + StoredFile storedFile = new StoredFile( + task.getVideoIdentifier(), + task.getFilename(), + "video/mp4", + Files.size(ffmpegOutputFile), + task.getCreatedAt() + ); + Path finalFilePath = fileService.getStoragePathForFile(storedFile); Files.move(ffmpegOutputFile, finalFilePath); Files.deleteIfExists(tempFile); Files.deleteIfExists(ffmpegOutputFile); - storedFileRepository.saveAndFlush(new StoredFile( - task.getFilename(), - task.getVideoIdentifier(), - "video/mp4", - Files.size(ffmpegOutputFile), - uploadedAt - )); + storedFileRepository.saveAndFlush(storedFile); updateTask(task, VideoProcessingTask.Status.COMPLETED); + log.info("Finished processing video {}.", task.getVideoIdentifier()); } catch (IOException e) { log.error("Failed to copy processed video to final storage location.", e); updateTask(task, VideoProcessingTask.Status.FAILED); @@ -113,7 +112,8 @@ public class VideoProcessingService { Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log"); Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log"); final String[] command = { - "ffmpeg", "-i", inFile.getFileName().toString(), + "ffmpeg", + "-i", inFile.getFileName().toString(), "-vf", "scale=640x480:flags=lanczos", "-vcodec", "libx264", "-crf", "28", diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/util/ULID.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/util/ULID.java new file mode 100644 index 0000000..1819795 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/util/ULID.java @@ -0,0 +1,456 @@ +package nl.andrewlalis.gymboardcdn.util; + +/* + * sulky-modules - several general-purpose modules. + * Copyright (C) 2007-2019 Joern Huxhorn + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +/* + * Copyright 2007-2019 Joern Huxhorn + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Serializable; +import java.security.SecureRandom; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; + +/* + * https://github.com/ulid/spec + */ +@SuppressWarnings("PMD.ShortClassName") +public class ULID +{ + private static final char[] ENCODING_CHARS = { + '0','1','2','3','4','5','6','7','8','9', + 'A','B','C','D','E','F','G','H','J','K', + 'M','N','P','Q','R','S','T','V','W','X', + 'Y','Z', + }; + + private static final byte[] DECODING_CHARS = { + // 0 + -1, -1, -1, -1, -1, -1, -1, -1, + // 8 + -1, -1, -1, -1, -1, -1, -1, -1, + // 16 + -1, -1, -1, -1, -1, -1, -1, -1, + // 24 + -1, -1, -1, -1, -1, -1, -1, -1, + // 32 + -1, -1, -1, -1, -1, -1, -1, -1, + // 40 + -1, -1, -1, -1, -1, -1, -1, -1, + // 48 + 0, 1, 2, 3, 4, 5, 6, 7, + // 56 + 8, 9, -1, -1, -1, -1, -1, -1, + // 64 + -1, 10, 11, 12, 13, 14, 15, 16, + // 72 + 17, 1, 18, 19, 1, 20, 21, 0, + // 80 + 22, 23, 24, 25, 26, -1, 27, 28, + // 88 + 29, 30, 31, -1, -1, -1, -1, -1, + // 96 + -1, 10, 11, 12, 13, 14, 15, 16, + // 104 + 17, 1, 18, 19, 1, 20, 21, 0, + // 112 + 22, 23, 24, 25, 26, -1, 27, 28, + // 120 + 29, 30, 31, + }; + + private static final int MASK = 0x1F; + private static final int MASK_BITS = 5; + private static final long TIMESTAMP_OVERFLOW_MASK = 0xFFFF_0000_0000_0000L; + private static final long TIMESTAMP_MSB_MASK = 0xFFFF_FFFF_FFFF_0000L; + private static final long RANDOM_MSB_MASK = 0xFFFFL; + + private final Random random; + + public ULID() + { + this(new SecureRandom()); + } + + public ULID(Random random) + { + Objects.requireNonNull(random, "random must not be null!"); + this.random = random; + } + + public void appendULID(StringBuilder stringBuilder) + { + Objects.requireNonNull(stringBuilder, "stringBuilder must not be null!"); + internalAppendULID(stringBuilder, System.currentTimeMillis(), random); + } + + public String nextULID() + { + return nextULID(System.currentTimeMillis()); + } + + public String nextULID(long timestamp) + { + return internalUIDString(timestamp, random); + } + + public Value nextValue() + { + return nextValue(System.currentTimeMillis()); + } + + public Value nextValue(long timestamp) + { + return internalNextValue(timestamp, random); + } + + /** + * Returns the next monotonic value. If an overflow happened while incrementing + * the random part of the given previous ULID value then the returned value will + * have a zero random part. + * + * @param previousUlid the previous ULID value. + * @return the next monotonic value. + */ + public Value nextMonotonicValue(Value previousUlid) + { + return nextMonotonicValue(previousUlid, System.currentTimeMillis()); + } + + /** + * Returns the next monotonic value. If an overflow happened while incrementing + * the random part of the given previous ULID value then the returned value will + * have a zero random part. + * + * @param previousUlid the previous ULID value. + * @param timestamp the timestamp of the next ULID value. + * @return the next monotonic value. + */ + public Value nextMonotonicValue(Value previousUlid, long timestamp) + { + Objects.requireNonNull(previousUlid, "previousUlid must not be null!"); + if(previousUlid.timestamp() == timestamp) + { + return previousUlid.increment(); + } + return nextValue(timestamp); + } + + /** + * Returns the next monotonic value or empty if an overflow happened while incrementing + * the random part of the given previous ULID value. + * + * @param previousUlid the previous ULID value. + * @return the next monotonic value or empty if an overflow happened. + */ + public Optional nextStrictlyMonotonicValue(Value previousUlid) + { + return nextStrictlyMonotonicValue(previousUlid, System.currentTimeMillis()); + } + + /** + * Returns the next monotonic value or empty if an overflow happened while incrementing + * the random part of the given previous ULID value. + * + * @param previousUlid the previous ULID value. + * @param timestamp the timestamp of the next ULID value. + * @return the next monotonic value or empty if an overflow happened. + */ + public Optional nextStrictlyMonotonicValue(Value previousUlid, long timestamp) + { + Value result = nextMonotonicValue(previousUlid, timestamp); + if(result.compareTo(previousUlid) < 1) + { + return Optional.empty(); + } + return Optional.of(result); + } + + public static Value parseULID(String ulidString) + { + Objects.requireNonNull(ulidString, "ulidString must not be null!"); + if(ulidString.length() != 26) + { + throw new IllegalArgumentException("ulidString must be exactly 26 chars long."); + } + + String timeString = ulidString.substring(0, 10); + long time = internalParseCrockford(timeString); + if ((time & TIMESTAMP_OVERFLOW_MASK) != 0) + { + throw new IllegalArgumentException("ulidString must not exceed '7ZZZZZZZZZZZZZZZZZZZZZZZZZ'!"); + } + String part1String = ulidString.substring(10, 18); + String part2String = ulidString.substring(18); + long part1 = internalParseCrockford(part1String); + long part2 = internalParseCrockford(part2String); + + long most = (time << 16) | (part1 >>> 24); + long least = part2 | (part1 << 40); + return new Value(most, least); + } + + public static Value fromBytes(byte[] data) + { + Objects.requireNonNull(data, "data must not be null!"); + if(data.length != 16) + { + throw new IllegalArgumentException("data must be 16 bytes in length!"); + } + long mostSignificantBits = 0; + long leastSignificantBits = 0; + for (int i=0; i<8; i++) + { + mostSignificantBits = (mostSignificantBits << 8) | (data[i] & 0xff); + } + for (int i=8; i<16; i++) + { + leastSignificantBits = (leastSignificantBits << 8) | (data[i] & 0xff); + } + return new Value(mostSignificantBits, leastSignificantBits); + } + + public static class Value + implements Comparable, Serializable + { + private static final long serialVersionUID = -3563159514112487717L; + + /* + * The most significant 64 bits of this ULID. + */ + private final long mostSignificantBits; + + /* + * The least significant 64 bits of this ULID. + */ + private final long leastSignificantBits; + + public Value(long mostSignificantBits, long leastSignificantBits) + { + this.mostSignificantBits = mostSignificantBits; + this.leastSignificantBits = leastSignificantBits; + } + + /** + * Returns the most significant 64 bits of this ULID's 128 bit value. + * + * @return The most significant 64 bits of this ULID's 128 bit value + */ + public long getMostSignificantBits() { + return mostSignificantBits; + } + + /** + * Returns the least significant 64 bits of this ULID's 128 bit value. + * + * @return The least significant 64 bits of this ULID's 128 bit value + */ + public long getLeastSignificantBits() { + return leastSignificantBits; + } + + + public long timestamp() + { + return mostSignificantBits >>> 16; + } + + public byte[] toBytes() + { + byte[] result=new byte[16]; + for (int i=0; i<8; i++) + { + result[i] = (byte)((mostSignificantBits >> ((7-i)*8)) & 0xFF); + } + for (int i=8; i<16; i++) + { + result[i] = (byte)((leastSignificantBits >> ((15-i)*8)) & 0xFF); + } + + return result; + } + + public Value increment() + { + long lsb = leastSignificantBits; + if(lsb != 0xFFFF_FFFF_FFFF_FFFFL) + { + return new Value(mostSignificantBits, lsb+1); + } + long msb = mostSignificantBits; + if((msb & RANDOM_MSB_MASK) != RANDOM_MSB_MASK) + { + return new Value(msb + 1, 0); + } + return new Value(msb & TIMESTAMP_MSB_MASK, 0); + } + + @Override + public int hashCode() { + long hilo = mostSignificantBits ^ leastSignificantBits; + return ((int)(hilo >> 32)) ^ (int) hilo; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Value value = (Value) o; + + return mostSignificantBits == value.mostSignificantBits + && leastSignificantBits == value.leastSignificantBits; + } + + @Override + public int compareTo(Value val) + { + // The ordering is intentionally set up so that the ULIDs + // can simply be numerically compared as two numbers + return (this.mostSignificantBits < val.mostSignificantBits ? -1 : + (this.mostSignificantBits > val.mostSignificantBits ? 1 : + (this.leastSignificantBits < val.leastSignificantBits ? -1 : + (this.leastSignificantBits > val.leastSignificantBits ? 1 : + 0)))); + } + + @Override + public String toString() + { + char[] buffer = new char[26]; + + internalWriteCrockford(buffer, timestamp(), 10, 0); + long value = ((mostSignificantBits & 0xFFFFL) << 24); + long interim = (leastSignificantBits >>> 40); + value = value | interim; + internalWriteCrockford(buffer, value, 8, 10); + internalWriteCrockford(buffer, leastSignificantBits, 8, 18); + + return new String(buffer); + } + } + + /* + * http://crockford.com/wrmg/base32.html + */ + static void internalAppendCrockford(StringBuilder builder, long value, int count) + { + for(int i = count-1; i >= 0; i--) + { + int index = (int)((value >>> (i * MASK_BITS)) & MASK); + builder.append(ENCODING_CHARS[index]); + } + } + + static long internalParseCrockford(String input) + { + Objects.requireNonNull(input, "input must not be null!"); + int length = input.length(); + if(length > 12) + { + throw new IllegalArgumentException("input length must not exceed 12 but was "+length+"!"); + } + + long result = 0; + for(int i=0;i>> ((count - i - 1) * MASK_BITS)) & MASK); + buffer[offset+i] = ENCODING_CHARS[index]; + } + } + + static String internalUIDString(long timestamp, Random random) + { + checkTimestamp(timestamp); + + char[] buffer = new char[26]; + + internalWriteCrockford(buffer, timestamp, 10, 0); + // could use nextBytes(byte[] bytes) instead + internalWriteCrockford(buffer, random.nextLong(), 8, 10); + internalWriteCrockford(buffer, random.nextLong(), 8, 18); + + return new String(buffer); + } + + static void internalAppendULID(StringBuilder builder, long timestamp, Random random) + { + checkTimestamp(timestamp); + + internalAppendCrockford(builder, timestamp, 10); + // could use nextBytes(byte[] bytes) instead + internalAppendCrockford(builder, random.nextLong(), 8); + internalAppendCrockford(builder, random.nextLong(), 8); + } + + static Value internalNextValue(long timestamp, Random random) + { + checkTimestamp(timestamp); + // could use nextBytes(byte[] bytes) instead + long mostSignificantBits = random.nextLong(); + long leastSignificantBits = random.nextLong(); + mostSignificantBits &= 0xFFFF; + mostSignificantBits |= (timestamp << 16); + return new Value(mostSignificantBits, leastSignificantBits); + } + + private static void checkTimestamp(long timestamp) + { + if((timestamp & TIMESTAMP_OVERFLOW_MASK) != 0) + { + throw new IllegalArgumentException("ULID does not support timestamps after +10889-08-02T05:31:50.655Z!"); + } + } +} diff --git a/gymboard-cdn/src/main/resources/application-development.properties b/gymboard-cdn/src/main/resources/application-development.properties index 7b50361..543e510 100644 --- a/gymboard-cdn/src/main/resources/application-development.properties +++ b/gymboard-cdn/src/main/resources/application-development.properties @@ -7,5 +7,6 @@ spring.jpa.hibernate.ddl-auto=update server.port=8082 app.web-origin=http://localhost:9000 +app.api-origin=http://localhost:8080 app.files.storage-dir=./cdn-files/ app.files.temp-dir=./cdn-files/tmp/ diff --git a/gymboard-cdn/src/test/resources/application.properties b/gymboard-cdn/src/test/resources/application.properties index 9195bbe..b735bf1 100644 --- a/gymboard-cdn/src/test/resources/application.properties +++ b/gymboard-cdn/src/test/resources/application.properties @@ -12,5 +12,6 @@ spring.jpa.hibernate.ddl-auto=update server.port=8082 app.web-origin=http://localhost:9000 +app.api-origin=http://localhost:8080 app.files.storage-dir=./test-cdn-files/ app.files.temp-dir=./test-cdn-files/tmp/