Added CDN client to sample data loader, and finalized base implementation of CDN.

This commit is contained in:
Andrew Lalis 2023-02-03 14:09:38 +01:00
parent 91648a13fa
commit d60f7142e8
36 changed files with 755 additions and 772 deletions

View File

@ -4,7 +4,7 @@ An HTTP/REST API powered by Java and Spring Boot. This API serves as the main en
## Development ## Development
To ease development, `nl.andrewlalis.gymboard_api.model.SampleDataLoader` will run on startup and populate the database with some sample entities. You can regenerate this data by manually deleting the database, and deleting the `.sample_data` marker file that's generated in the project directory. To ease development, `nl.andrewlalis.gymboard_api.util.SampleDataLoader` will run on startup and populate the database with some sample entities. You can regenerate this data by manually deleting the database, and deleting the `.sample_data` marker file that's generated in the project directory.
## ULIDs ## ULIDs

View File

@ -42,6 +42,7 @@
<artifactId>spring-boot-starter-mail</artifactId> <artifactId>spring-boot-starter-mail</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>

View File

@ -1,12 +1,12 @@
package nl.andrewlalis.gymboard_api.controller; package nl.andrewlalis.gymboard_api.controller;
import nl.andrewlalis.gymboard_api.controller.dto.*; import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse;
import nl.andrewlalis.gymboard_api.controller.dto.GymResponse;
import nl.andrewlalis.gymboard_api.service.GymService; import nl.andrewlalis.gymboard_api.service.GymService;
import nl.andrewlalis.gymboard_api.service.UploadService;
import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.List; import java.util.List;
@ -17,14 +17,10 @@ import java.util.List;
@RequestMapping(path = "/gyms/{compoundId}") @RequestMapping(path = "/gyms/{compoundId}")
public class GymController { public class GymController {
private final GymService gymService; private final GymService gymService;
private final UploadService uploadService;
private final ExerciseSubmissionService submissionService; private final ExerciseSubmissionService submissionService;
public GymController(GymService gymService, public GymController(GymService gymService, ExerciseSubmissionService submissionService) {
UploadService uploadService,
ExerciseSubmissionService submissionService) {
this.gymService = gymService; this.gymService = gymService;
this.uploadService = uploadService;
this.submissionService = submissionService; this.submissionService = submissionService;
} }
@ -45,12 +41,4 @@ public class GymController {
) { ) {
return submissionService.createSubmission(CompoundGymId.parse(compoundId), payload); return submissionService.createSubmission(CompoundGymId.parse(compoundId), payload);
} }
@PostMapping(path = "/submissions/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public UploadedFileResponse uploadVideo(
@PathVariable String compoundId,
@RequestParam MultipartFile file
) {
return uploadService.handleSubmissionUpload(CompoundGymId.parse(compoundId), file);
}
} }

View File

@ -1,6 +1,5 @@
package nl.andrewlalis.gymboard_api.controller; package nl.andrewlalis.gymboard_api.controller;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse;
import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@ -21,9 +20,4 @@ public class SubmissionController {
public ExerciseSubmissionResponse getSubmission(@PathVariable String submissionId) { public ExerciseSubmissionResponse getSubmission(@PathVariable String submissionId) {
return submissionService.getSubmission(submissionId); return submissionService.getSubmission(submissionId);
} }
@GetMapping(path = "/{submissionId}/video")
public void getSubmissionVideo(@PathVariable String submissionId, HttpServletResponse response) {
submissionService.streamVideo(submissionId, response);
}
} }

View File

@ -6,5 +6,5 @@ public record ExerciseSubmissionPayload(
float weight, float weight,
String weightUnit, String weightUnit,
int reps, int reps,
long videoId String videoFileId
) {} ) {}

View File

@ -9,7 +9,7 @@ public record ExerciseSubmissionResponse(
String createdAt, String createdAt,
GymSimpleResponse gym, GymSimpleResponse gym,
ExerciseResponse exercise, ExerciseResponse exercise,
String status, String videoFileId,
String submitterName, String submitterName,
double rawWeight, double rawWeight,
String weightUnit, String weightUnit,
@ -22,7 +22,7 @@ public record ExerciseSubmissionResponse(
submission.getCreatedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), submission.getCreatedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
new GymSimpleResponse(submission.getGym()), new GymSimpleResponse(submission.getGym()),
new ExerciseResponse(submission.getExercise()), new ExerciseResponse(submission.getExercise()),
submission.getStatus().name(), submission.getVideoFileId(),
submission.getSubmitterName(), submission.getSubmitterName(),
submission.getRawWeight().doubleValue(), submission.getRawWeight().doubleValue(),
submission.getWeightUnit().name(), submission.getWeightUnit().name(),

View File

@ -1,3 +0,0 @@
package nl.andrewlalis.gymboard_api.controller.dto;
public record UploadedFileResponse(long id) {}

View File

@ -1,9 +0,0 @@
package nl.andrewlalis.gymboard_api.dao;
import nl.andrewlalis.gymboard_api.model.StoredFile;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface StoredFileRepository extends JpaRepository<StoredFile, Long> {
}

View File

@ -5,9 +5,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor; import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import java.util.List;
@Repository @Repository
public interface ExerciseSubmissionRepository extends JpaRepository<ExerciseSubmission, String>, JpaSpecificationExecutor<ExerciseSubmission> { public interface ExerciseSubmissionRepository extends JpaRepository<ExerciseSubmission, String>, JpaSpecificationExecutor<ExerciseSubmission> {
List<ExerciseSubmission> findAllByStatus(ExerciseSubmission.Status status);
} }

View File

@ -1,17 +0,0 @@
package nl.andrewlalis.gymboard_api.dao.exercise;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
@Repository
public interface ExerciseSubmissionTempFileRepository extends JpaRepository<ExerciseSubmissionTempFile, Long> {
List<ExerciseSubmissionTempFile> findAllByCreatedAtBefore(LocalDateTime timestamp);
Optional<ExerciseSubmissionTempFile> findBySubmission(ExerciseSubmission submission);
boolean existsByPath(String path);
}

View File

@ -1,18 +0,0 @@
package nl.andrewlalis.gymboard_api.dao.exercise;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface ExerciseSubmissionVideoFileRepository extends JpaRepository<ExerciseSubmissionVideoFile, Long> {
Optional<ExerciseSubmissionVideoFile> findBySubmission(ExerciseSubmission submission);
@Query("SELECT f FROM ExerciseSubmissionVideoFile f WHERE " +
"f.submission.id = :submissionId AND f.submission.complete = true")
Optional<ExerciseSubmissionVideoFile> findByCompletedSubmissionId(String submissionId);
}

View File

@ -1,68 +0,0 @@
package nl.andrewlalis.gymboard_api.model;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* Base class for file storage. Files (mostly gym videos) are stored in the
* database as blobs, after they've been pre-processed with compression and/or
* resizing.
*/
@Entity
@Table(name = "stored_file")
public class StoredFile {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Column(nullable = false, updatable = false)
private String filename;
@Column(nullable = false, updatable = false)
private String mimeType;
@Column(nullable = false, updatable = false)
private long size;
@Lob
@Column(nullable = false, updatable = false)
private byte[] content;
public StoredFile() {}
public StoredFile(String filename, String mimeType, long size, byte[] content) {
this.filename = filename;
this.mimeType = mimeType;
this.size = size;
this.content = content;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getFilename() {
return filename;
}
public String getMimeType() {
return mimeType;
}
public long getSize() {
return size;
}
public byte[] getContent() {
return content;
}
}

View File

@ -10,24 +10,6 @@ import java.time.LocalDateTime;
@Entity @Entity
@Table(name = "exercise_submission") @Table(name = "exercise_submission")
public class ExerciseSubmission { public class ExerciseSubmission {
/**
* The status of a submission.
* <ul>
* <li>Each submission starts as WAITING.</li>
* <li>The status changes to PROCESSING once it's picked up for processing.</li>
* <li>If processing fails, the status changes to FAILED.</li>
* <li>If processing is successful, the status changes to COMPLETED.</li>
* <li>Once a completed submission is verified either automatically or manually, it's set to VERIFIED.</li>
* </ul>
*/
public enum Status {
WAITING,
PROCESSING,
FAILED,
COMPLETED,
VERIFIED
}
public enum WeightUnit { public enum WeightUnit {
KG, KG,
LBS LBS
@ -46,9 +28,13 @@ public class ExerciseSubmission {
@ManyToOne(optional = false, fetch = FetchType.LAZY) @ManyToOne(optional = false, fetch = FetchType.LAZY)
private Exercise exercise; private Exercise exercise;
@Enumerated(EnumType.STRING) /**
@Column(nullable = false) * The id of the video file that was submitted for this submission. It lives
private Status status; * on the <em>gymboard-cdn</em> service as a stored file, which can be
* accessed via <code>GET https://CDN-HOST/files/{videoFileId}</code>.
*/
@Column(nullable = false, updatable = false, length = 26)
private String videoFileId;
@Column(nullable = false, updatable = false, length = 63) @Column(nullable = false, updatable = false, length = 63)
private String submitterName; private String submitterName;
@ -66,27 +52,18 @@ public class ExerciseSubmission {
@Column(nullable = false) @Column(nullable = false)
private int reps; private int reps;
/**
* Marker that's used to simplify queries where we just want submissions
* that are in a status that's not WAITING, PROCESSING, or FAILED, i.e.
* a successful submission that's been processed.
*/
@Column(nullable = false)
private boolean complete;
public ExerciseSubmission() {} public ExerciseSubmission() {}
public ExerciseSubmission(String id, Gym gym, Exercise exercise, String submitterName, BigDecimal rawWeight, WeightUnit unit, BigDecimal metricWeight, int reps) { public ExerciseSubmission(String id, Gym gym, Exercise exercise, String videoFileId, String submitterName, BigDecimal rawWeight, WeightUnit unit, BigDecimal metricWeight, int reps) {
this.id = id; this.id = id;
this.gym = gym; this.gym = gym;
this.exercise = exercise; this.exercise = exercise;
this.videoFileId = videoFileId;
this.submitterName = submitterName; this.submitterName = submitterName;
this.rawWeight = rawWeight; this.rawWeight = rawWeight;
this.weightUnit = unit; this.weightUnit = unit;
this.metricWeight = metricWeight; this.metricWeight = metricWeight;
this.reps = reps; this.reps = reps;
this.status = Status.WAITING;
this.complete = false;
} }
public String getId() { public String getId() {
@ -105,12 +82,8 @@ public class ExerciseSubmission {
return exercise; return exercise;
} }
public Status getStatus() { public String getVideoFileId() {
return status; return videoFileId;
}
public void setStatus(Status status) {
this.status = status;
} }
public String getSubmitterName() { public String getSubmitterName() {
@ -132,12 +105,4 @@ public class ExerciseSubmission {
public int getReps() { public int getReps() {
return reps; return reps;
} }
public boolean isComplete() {
return complete;
}
public void setComplete(boolean complete) {
this.complete = complete;
}
} }

View File

@ -1,58 +0,0 @@
package nl.andrewlalis.gymboard_api.model.exercise;
import jakarta.persistence.*;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
/**
* Tracks the temporary file on disk that's stored while a user is preparing
* their submission. This file will be removed after the submission is
* processed.
*/
@Entity
@Table(name = "exercise_submission_temp_file")
public class ExerciseSubmissionTempFile {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@CreationTimestamp
private LocalDateTime createdAt;
@Column(nullable = false, updatable = false, length = 1024)
private String path;
/**
* The submission that this temporary file is for. This will initially be
* null, but will be set as soon as the submission is finalized.
*/
@OneToOne(fetch = FetchType.LAZY)
private ExerciseSubmission submission;
public ExerciseSubmissionTempFile() {}
public ExerciseSubmissionTempFile(String path) {
this.path = path;
}
public Long getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getPath() {
return path;
}
public ExerciseSubmission getSubmission() {
return submission;
}
public void setSubmission(ExerciseSubmission submission) {
this.submission = submission;
}
}

View File

@ -1,41 +0,0 @@
package nl.andrewlalis.gymboard_api.model.exercise;
import jakarta.persistence.*;
import nl.andrewlalis.gymboard_api.model.StoredFile;
/**
* An entity which links an {@link ExerciseSubmission} to a {@link nl.andrewlalis.gymboard_api.model.StoredFile}
* containing the video that was submitted along with the submission.
*/
@Entity
@Table(name = "exercise_submission_video_file")
public class ExerciseSubmissionVideoFile {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@OneToOne(optional = false, fetch = FetchType.LAZY)
private ExerciseSubmission submission;
@OneToOne(optional = false, fetch = FetchType.LAZY, orphanRemoval = true)
private StoredFile file;
public ExerciseSubmissionVideoFile() {}
public ExerciseSubmissionVideoFile(ExerciseSubmission submission, StoredFile file) {
this.submission = submission;
this.file = file;
}
public Long getId() {
return id;
}
public ExerciseSubmission getSubmission() {
return submission;
}
public StoredFile getFile() {
return file;
}
}

View File

@ -1,76 +0,0 @@
package nl.andrewlalis.gymboard_api.service;
import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.controller.dto.UploadedFileResponse;
import nl.andrewlalis.gymboard_api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository;
import nl.andrewlalis.gymboard_api.model.Gym;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* Service for handling large file uploads.
*/
@Service
public class UploadService {
public static final Path SUBMISSION_TEMP_FILE_DIR = Path.of("exercise_submission_temp_files");
private static final String[] ALLOWED_VIDEO_TYPES = {
"video/mp4"
};
private final ExerciseSubmissionTempFileRepository tempFileRepository;
private final GymRepository gymRepository;
public UploadService(ExerciseSubmissionTempFileRepository tempFileRepository, GymRepository gymRepository) {
this.tempFileRepository = tempFileRepository;
this.gymRepository = gymRepository;
}
/**
* Handles the upload of an exercise submission's video file by saving the
* file to a temporary location, and recording that location in the
* database for when the exercise submission is completed. We'll only do
* the computationally expensive video processing if a user successfully
* submits their submission; otherwise, the raw video is discarded after a
* while.
* @param gymId The gym's id.
* @param multipartFile The uploaded file.
* @return A response containing the uploaded file's id, to be included in
* the user's submission.
*/
@Transactional
public UploadedFileResponse handleSubmissionUpload(CompoundGymId gymId, MultipartFile multipartFile) {
Gym gym = gymRepository.findByCompoundId(gymId)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
// TODO: Check that user is allowed to upload.
boolean fileTypeAcceptable = false;
for (String allowedType : ALLOWED_VIDEO_TYPES) {
if (allowedType.equalsIgnoreCase(multipartFile.getContentType())) {
fileTypeAcceptable = true;
break;
}
}
if (!fileTypeAcceptable) {
throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid content type.");
}
try {
if (!Files.exists(SUBMISSION_TEMP_FILE_DIR)) {
Files.createDirectory(SUBMISSION_TEMP_FILE_DIR);
}
Path tempFilePath = Files.createTempFile(SUBMISSION_TEMP_FILE_DIR, null, null);
multipartFile.transferTo(tempFilePath);
ExerciseSubmissionTempFile tempFileEntity = tempFileRepository.save(new ExerciseSubmissionTempFile(tempFilePath.toString()));
return new UploadedFileResponse(tempFileEntity.getId());
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "File upload failed.", e);
}
}
}

View File

@ -0,0 +1,47 @@
package nl.andrewlalis.gymboard_api.service.cdn_client;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Path;
import java.time.Duration;
public class CdnClient {
private final HttpClient httpClient;
private final String baseUrl;
private final ObjectMapper objectMapper;
public final UploadsClient uploads;
public CdnClient(String baseUrl) {
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(3))
.followRedirects(HttpClient.Redirect.NORMAL)
.build();
this.baseUrl = baseUrl;
this.objectMapper = new ObjectMapper();
this.uploads = new UploadsClient(this);
}
public <T> T get(String urlPath, Class<T> responseType) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.GET()
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
return objectMapper.readValue(response.body(), responseType);
}
public <T> T postFile(String urlPath, Path filePath, String contentType, Class<T> responseType) throws IOException, InterruptedException {
HttpRequest req = HttpRequest.newBuilder(URI.create(baseUrl + urlPath))
.POST(HttpRequest.BodyPublishers.ofFile(filePath))
.header("Content-Type", contentType)
.header("X-Gymboard-Filename", filePath.getFileName().toString())
.build();
HttpResponse<String> response = httpClient.send(req, HttpResponse.BodyHandlers.ofString());
return objectMapper.readValue(response.body(), responseType);
}
}

View File

@ -0,0 +1,16 @@
package nl.andrewlalis.gymboard_api.service.cdn_client;
import java.nio.file.Path;
public record UploadsClient(CdnClient client) {
public record FileUploadResponse(String id) {}
public record VideoProcessingTaskStatusResponse(String status) {}
public FileUploadResponse uploadVideo(Path filePath, String contentType) throws Exception {
return client.postFile("/uploads/video", filePath, contentType, FileUploadResponse.class);
}
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) throws Exception {
return client.get("/uploads/video/" + id + "/status", VideoProcessingTaskStatusResponse.class);
}
}

View File

@ -1,19 +1,14 @@
package nl.andrewlalis.gymboard_api.service.submission; package nl.andrewlalis.gymboard_api.service.submission;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionResponse;
import nl.andrewlalis.gymboard_api.dao.GymRepository; import nl.andrewlalis.gymboard_api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionVideoFileRepository;
import nl.andrewlalis.gymboard_api.model.Gym; import nl.andrewlalis.gymboard_api.model.Gym;
import nl.andrewlalis.gymboard_api.model.exercise.Exercise; import nl.andrewlalis.gymboard_api.model.exercise.Exercise;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile;
import nl.andrewlalis.gymboard_api.util.ULID; import nl.andrewlalis.gymboard_api.util.ULID;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -22,7 +17,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException; import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
/** /**
@ -36,21 +30,15 @@ public class ExerciseSubmissionService {
private final GymRepository gymRepository; private final GymRepository gymRepository;
private final ExerciseRepository exerciseRepository; private final ExerciseRepository exerciseRepository;
private final ExerciseSubmissionRepository exerciseSubmissionRepository; private final ExerciseSubmissionRepository exerciseSubmissionRepository;
private final ExerciseSubmissionTempFileRepository tempFileRepository;
private final ExerciseSubmissionVideoFileRepository submissionVideoFileRepository;
private final ULID ulid; private final ULID ulid;
public ExerciseSubmissionService(GymRepository gymRepository, public ExerciseSubmissionService(GymRepository gymRepository,
ExerciseRepository exerciseRepository, ExerciseRepository exerciseRepository,
ExerciseSubmissionRepository exerciseSubmissionRepository, ExerciseSubmissionRepository exerciseSubmissionRepository,
ExerciseSubmissionTempFileRepository tempFileRepository,
ExerciseSubmissionVideoFileRepository submissionVideoFileRepository,
ULID ulid) { ULID ulid) {
this.gymRepository = gymRepository; this.gymRepository = gymRepository;
this.exerciseRepository = exerciseRepository; this.exerciseRepository = exerciseRepository;
this.exerciseSubmissionRepository = exerciseSubmissionRepository; this.exerciseSubmissionRepository = exerciseSubmissionRepository;
this.tempFileRepository = tempFileRepository;
this.submissionVideoFileRepository = submissionVideoFileRepository;
this.ulid = ulid; this.ulid = ulid;
} }
@ -61,33 +49,11 @@ public class ExerciseSubmissionService {
return new ExerciseSubmissionResponse(submission); return new ExerciseSubmissionResponse(submission);
} }
@Transactional(readOnly = true)
public void streamVideo(String submissionId, HttpServletResponse response) {
ExerciseSubmissionVideoFile videoFile = submissionVideoFileRepository.findByCompletedSubmissionId(submissionId)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
response.setContentType(videoFile.getFile().getMimeType());
response.setContentLengthLong(videoFile.getFile().getSize());
try {
response.getOutputStream().write(videoFile.getFile().getContent());
} catch (IOException e) {
log.error("Failed to write submission video file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
/** /**
* Handles the creation of a new exercise submission. This involves a few steps: * Handles the creation of a new exercise submission.
* <ol>
* <li>Pre-fetch all of the referenced data, like exercise and video file.</li>
* <li>Check that the submission is legitimate.</li>
* <li>Save the submission. (With the WAITING status initially.)</li>
* <li>Sometime soon, {@link SubmissionProcessingService#processWaitingSubmissions()} will pick up the submission for processing.</li>
* </ol>
* Once the asynchronous submission processing is complete, the submission
* status will change to COMPLETE.
* @param id The gym id. * @param id The gym id.
* @param payload The submission data. * @param payload The submission data.
* @return The saved submission, which will be in the PROCESSING state at first. * @return The saved submission.
*/ */
@Transactional @Transactional
public ExerciseSubmissionResponse createSubmission(CompoundGymId id, ExerciseSubmissionPayload payload) { public ExerciseSubmissionResponse createSubmission(CompoundGymId id, ExerciseSubmissionPayload payload) {
@ -95,10 +61,8 @@ public class ExerciseSubmissionService {
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
Exercise exercise = exerciseRepository.findById(payload.exerciseShortName()) Exercise exercise = exerciseRepository.findById(payload.exerciseShortName())
.orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid exercise.")); .orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid exercise."));
ExerciseSubmissionTempFile tempFile = tempFileRepository.findById(payload.videoId())
.orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid video id."));
validateSubmission(payload, exercise, tempFile); // TODO: Validate the submission data.
// Create the submission. // Create the submission.
BigDecimal rawWeight = BigDecimal.valueOf(payload.weight()); BigDecimal rawWeight = BigDecimal.valueOf(payload.weight());
@ -107,26 +71,17 @@ public class ExerciseSubmissionService {
if (unit == ExerciseSubmission.WeightUnit.LBS) { if (unit == ExerciseSubmission.WeightUnit.LBS) {
metricWeight = metricWeight.multiply(new BigDecimal("0.45359237")); metricWeight = metricWeight.multiply(new BigDecimal("0.45359237"));
} }
ExerciseSubmission submission = exerciseSubmissionRepository.saveAndFlush(new ExerciseSubmission( ExerciseSubmission submission = exerciseSubmissionRepository.saveAndFlush(new ExerciseSubmission(
ulid.nextULID(), ulid.nextULID(),
gym, gym,
exercise, exercise,
payload.videoFileId(),
payload.name(), payload.name(),
rawWeight, rawWeight,
unit, unit,
metricWeight, metricWeight,
payload.reps() payload.reps()
)); ));
// Then link it to the temporary video file so the async task can find it.
tempFile.setSubmission(submission);
tempFileRepository.save(tempFile);
// The submission will be picked up eventually to be processed.
return new ExerciseSubmissionResponse(submission); return new ExerciseSubmissionResponse(submission);
} }
private void validateSubmission(ExerciseSubmissionPayload payload, Exercise exercise, ExerciseSubmissionTempFile tempFile) {
// TODO: Implement this validation.
}
} }

View File

@ -1,237 +0,0 @@
package nl.andrewlalis.gymboard_api.service.submission;
import nl.andrewlalis.gymboard_api.dao.StoredFileRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionTempFileRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseSubmissionVideoFileRepository;
import nl.andrewlalis.gymboard_api.model.StoredFile;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionTempFile;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmissionVideoFile;
import nl.andrewlalis.gymboard_api.service.CommandFailedException;
import nl.andrewlalis.gymboard_api.service.UploadService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
/**
* This service is responsible for the logic of processing new exercise
* submissions and tasks immediately related to that.
*/
@Service
public class SubmissionProcessingService {
private static final Logger log = LoggerFactory.getLogger(SubmissionProcessingService.class);
private final ExerciseSubmissionRepository exerciseSubmissionRepository;
private final Executor taskExecutor;
private final ExerciseSubmissionTempFileRepository tempFileRepository;
private final ExerciseSubmissionVideoFileRepository videoFileRepository;
private final StoredFileRepository fileRepository;
public SubmissionProcessingService(ExerciseSubmissionRepository exerciseSubmissionRepository,
Executor taskExecutor,
ExerciseSubmissionTempFileRepository tempFileRepository,
ExerciseSubmissionVideoFileRepository videoFileRepository,
StoredFileRepository fileRepository) {
this.exerciseSubmissionRepository = exerciseSubmissionRepository;
this.taskExecutor = taskExecutor;
this.tempFileRepository = tempFileRepository;
this.videoFileRepository = videoFileRepository;
this.fileRepository = fileRepository;
}
/**
* Simple scheduled task that periodically checks for new submissions
* that are waiting to be processed, and queues tasks to do so.
*/
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void processWaitingSubmissions() {
List<ExerciseSubmission> waitingSubmissions = exerciseSubmissionRepository.findAllByStatus(ExerciseSubmission.Status.WAITING);
for (var submission : waitingSubmissions) {
taskExecutor.execute(() -> processSubmission(submission.getId()));
}
}
/**
* Asynchronous task that's started after a submission is submitted, which
* handles video processing and anything else that might need to be done
* before the submission can be marked as COMPLETED.
* <p>
* Note: This method is intentionally NOT transactional, since it may
* have a long duration, and we want real-time status updates.
* </p>
* @param submissionId The submission's id.
*/
private void processSubmission(String submissionId) {
log.info("Starting processing of submission {}.", submissionId);
// First try and fetch the submission.
Optional<ExerciseSubmission> optionalSubmission = exerciseSubmissionRepository.findById(submissionId);
if (optionalSubmission.isEmpty()) {
log.warn("Submission id {} is not associated with a submission.", submissionId);
return;
}
ExerciseSubmission submission = optionalSubmission.get();
if (submission.getStatus() != ExerciseSubmission.Status.WAITING) {
log.warn("Submission {} cannot be processed because its status {} is not WAITING.", submission.getId(), submission.getStatus());
return;
}
// Set the status to processing.
submission.setStatus(ExerciseSubmission.Status.PROCESSING);
exerciseSubmissionRepository.saveAndFlush(submission);
// Then try and fetch the temporary video file associated with it.
Optional<ExerciseSubmissionTempFile> optionalTempFile = tempFileRepository.findBySubmission(submission);
if (optionalTempFile.isEmpty()) {
log.warn("Submission {} failed because the temporary video file couldn't be found.", submission.getId());
submission.setStatus(ExerciseSubmission.Status.FAILED);
exerciseSubmissionRepository.save(submission);
return;
}
ExerciseSubmissionTempFile tempFile = optionalTempFile.get();
Path tempFilePath = Path.of(tempFile.getPath());
if (!Files.exists(tempFilePath) || !Files.isReadable(tempFilePath)) {
log.error("Submission {} failed because the temporary video file {} isn't readable.", submission.getId(), tempFilePath);
submission.setStatus(ExerciseSubmission.Status.FAILED);
exerciseSubmissionRepository.saveAndFlush(submission);
return;
}
// Now we can try to process the video file into a compressed format that can be stored in the DB.
Path dir = UploadService.SUBMISSION_TEMP_FILE_DIR;
String tempFileName = tempFilePath.getFileName().toString();
String tempFileBaseName = tempFileName.substring(0, tempFileName.length() - ".tmp".length());
Path outFilePath = dir.resolve(tempFileBaseName + "-out.mp4");
StoredFile file;
try {
processVideo(dir, tempFilePath, outFilePath);
file = fileRepository.save(new StoredFile(
"compressed.mp4",
"video/mp4",
Files.size(outFilePath),
Files.readAllBytes(outFilePath)
));
} catch (Exception e) {
log.error("""
Video processing failed for submission {}:
Input file: {}
Output file: {}
Exception message: {}""",
submission.getId(),
tempFilePath,
outFilePath,
e.getMessage()
);
submission.setStatus(ExerciseSubmission.Status.FAILED);
exerciseSubmissionRepository.saveAndFlush(submission);
return;
}
// After we've saved the processed file, we can link it to the submission, and set the submission's status.
videoFileRepository.save(new ExerciseSubmissionVideoFile(
submission,
file
));
submission.setStatus(ExerciseSubmission.Status.COMPLETED);
submission.setComplete(true);
exerciseSubmissionRepository.save(submission);
// And delete the temporary files.
try {
Files.delete(tempFilePath);
Files.delete(outFilePath);
tempFileRepository.delete(tempFile);
} catch (IOException e) {
log.error("Couldn't delete temporary files after submission completed.", e);
}
log.info("Processing of submission {} complete.", submission.getId());
}
/**
* Uses the `ffmpeg` system command to process a raw input video and produce
* a compressed, reduced-size output video that's ready for usage in the
* application.
* @param dir The working directory.
* @param inFile The input file to read from.
* @param outFile The output file to write to. MUST have a ".mp4" extension.
* @throws IOException If a filesystem error occurs.
* @throws CommandFailedException If the ffmpeg command fails.
* @throws InterruptedException If the ffmpeg command is interrupted.
*/
private void processVideo(Path dir, Path inFile, Path outFile) throws IOException, InterruptedException {
Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log");
Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log");
final String[] command = {
"ffmpeg", "-i", inFile.getFileName().toString(),
"-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264",
"-crf", "28",
outFile.getFileName().toString()
};
long startSize = Files.size(inFile);
Instant startTime = Instant.now();
Process ffmpegProcess = new ProcessBuilder()
.command(command)
.redirectOutput(tmpStdout.toFile())
.redirectError(tmpStderr.toFile())
.directory(dir.toFile())
.start();
int result = ffmpegProcess.waitFor();
if (result != 0) throw new CommandFailedException(command, result, tmpStdout, tmpStderr);
long endSize = Files.size(outFile);
Duration dur = Duration.between(startTime, Instant.now());
double reductionFactor = startSize / (double) endSize;
String reductionFactorStr = String.format("%.3f%%", reductionFactor * 100);
log.info("Processed video from {} bytes to {} bytes in {} seconds, {} reduction.", startSize, endSize, dur.getSeconds(), reductionFactorStr);
// Delete the logs if everything was successful.
Files.deleteIfExists(tmpStdout);
Files.deleteIfExists(tmpStderr);
}
@Scheduled(fixedRate = 1, timeUnit = TimeUnit.MINUTES)
public void removeOldUploadedFiles() {
// First remove any temp files older than 10 minutes.
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(10);
var tempFiles = tempFileRepository.findAllByCreatedAtBefore(cutoff);
for (var file : tempFiles) {
try {
Files.deleteIfExists(Path.of(file.getPath()));
tempFileRepository.delete(file);
log.info("Removed temporary submission file {} at {}.", file.getId(), file.getPath());
} catch (IOException e) {
log.error(String.format("Could not delete submission temp file %d at %s.", file.getId(), file.getPath()), e);
}
}
// Then remove any files in the directory which don't correspond to a valid file in the db.
if (Files.notExists(UploadService.SUBMISSION_TEMP_FILE_DIR)) return;
try (var s = Files.list(UploadService.SUBMISSION_TEMP_FILE_DIR)) {
for (var path : s.toList()) {
if (!tempFileRepository.existsByPath(path.toString())) {
try {
Files.delete(path);
} catch (IOException e) {
log.error("Couldn't delete orphan temp file: " + path, e);
}
}
}
} catch (IOException e) {
log.error("Couldn't get list of temp files.", e);
}
}
}

View File

@ -1,4 +1,4 @@
package nl.andrewlalis.gymboard_api.model; package nl.andrewlalis.gymboard_api.util;
import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId; import nl.andrewlalis.gymboard_api.controller.dto.CompoundGymId;
import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload; import nl.andrewlalis.gymboard_api.controller.dto.ExerciseSubmissionPayload;
@ -9,20 +9,24 @@ import nl.andrewlalis.gymboard_api.dao.GymRepository;
import nl.andrewlalis.gymboard_api.dao.auth.RoleRepository; import nl.andrewlalis.gymboard_api.dao.auth.RoleRepository;
import nl.andrewlalis.gymboard_api.dao.auth.UserRepository; import nl.andrewlalis.gymboard_api.dao.auth.UserRepository;
import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository; import nl.andrewlalis.gymboard_api.dao.exercise.ExerciseRepository;
import nl.andrewlalis.gymboard_api.model.City;
import nl.andrewlalis.gymboard_api.model.Country;
import nl.andrewlalis.gymboard_api.model.GeoPoint;
import nl.andrewlalis.gymboard_api.model.Gym;
import nl.andrewlalis.gymboard_api.model.auth.Role; import nl.andrewlalis.gymboard_api.model.auth.Role;
import nl.andrewlalis.gymboard_api.model.auth.User; import nl.andrewlalis.gymboard_api.model.auth.User;
import nl.andrewlalis.gymboard_api.model.exercise.Exercise; import nl.andrewlalis.gymboard_api.model.exercise.Exercise;
import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission; import nl.andrewlalis.gymboard_api.model.exercise.ExerciseSubmission;
import nl.andrewlalis.gymboard_api.service.UploadService;
import nl.andrewlalis.gymboard_api.service.auth.UserService; import nl.andrewlalis.gymboard_api.service.auth.UserService;
import nl.andrewlalis.gymboard_api.service.cdn_client.CdnClient;
import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService; import nl.andrewlalis.gymboard_api.service.submission.ExerciseSubmissionService;
import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord; import org.apache.commons.csv.CSVRecord;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener; import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
@ -31,7 +35,8 @@ import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.function.Consumer; import java.util.HashSet;
import java.util.Set;
/** /**
* Simple component that loads sample data that's useful when testing the application. * Simple component that loads sample data that's useful when testing the application.
@ -44,25 +49,25 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
private final GymRepository gymRepository; private final GymRepository gymRepository;
private final ExerciseRepository exerciseRepository; private final ExerciseRepository exerciseRepository;
private final ExerciseSubmissionService submissionService; private final ExerciseSubmissionService submissionService;
private final UploadService uploadService;
private final RoleRepository roleRepository; private final RoleRepository roleRepository;
private final UserRepository userRepository; private final UserRepository userRepository;
private final UserService userService; private final UserService userService;
@Value("${app.cdn-origin}")
private String cdnOrigin;
public SampleDataLoader( public SampleDataLoader(
CountryRepository countryRepository, CountryRepository countryRepository,
CityRepository cityRepository, CityRepository cityRepository,
GymRepository gymRepository, GymRepository gymRepository,
ExerciseRepository exerciseRepository, ExerciseRepository exerciseRepository,
ExerciseSubmissionService submissionService, ExerciseSubmissionService submissionService,
UploadService uploadService,
RoleRepository roleRepository, UserRepository userRepository, UserService userService) { RoleRepository roleRepository, UserRepository userRepository, UserService userService) {
this.countryRepository = countryRepository; this.countryRepository = countryRepository;
this.cityRepository = cityRepository; this.cityRepository = cityRepository;
this.gymRepository = gymRepository; this.gymRepository = gymRepository;
this.exerciseRepository = exerciseRepository; this.exerciseRepository = exerciseRepository;
this.submissionService = submissionService; this.submissionService = submissionService;
this.uploadService = uploadService;
this.roleRepository = roleRepository; this.roleRepository = roleRepository;
this.userRepository = userRepository; this.userRepository = userRepository;
this.userService = userService; this.userService = userService;
@ -77,13 +82,13 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
try { try {
generateSampleData(); generateSampleData();
Files.writeString(markerFile, "Yes"); Files.writeString(markerFile, "Yes");
} catch (IOException e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
@Transactional @Transactional
protected void generateSampleData() throws IOException { protected void generateSampleData() throws Exception {
loadCsv("exercises", record -> { loadCsv("exercises", record -> {
exerciseRepository.save(new Exercise(record.get(0), record.get(1))); exerciseRepository.save(new Exercise(record.get(0), record.get(1)));
}); });
@ -108,6 +113,13 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
record.get(7) record.get(7)
)); ));
}); });
// Loading sample submissions involves sending content to the Gymboard CDN service.
// We upload a video for each submission, and wait until all uploads are processed before continuing.
final CdnClient cdnClient = new CdnClient(cdnOrigin);
final Set<String> videoIds = new HashSet<>();
loadCsv("submissions", record -> { loadCsv("submissions", record -> {
var exercise = exerciseRepository.findById(record.get(0)).orElseThrow(); var exercise = exerciseRepository.findById(record.get(0)).orElseThrow();
BigDecimal weight = new BigDecimal(record.get(1)); BigDecimal weight = new BigDecimal(record.get(1));
@ -117,26 +129,34 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
CompoundGymId gymId = CompoundGymId.parse(record.get(5)); CompoundGymId gymId = CompoundGymId.parse(record.get(5));
String videoFilename = record.get(6); String videoFilename = record.get(6);
try { // Upload the video to the CDN, and wait until it's done processing.
var uploadResp = uploadService.handleSubmissionUpload(gymId, new MockMultipartFile( log.info("Uploading video {} to CDN...", videoFilename);
videoFilename, var video = cdnClient.uploads.uploadVideo(Path.of("sample_data", videoFilename), "video/mp4");
videoFilename,
"video/mp4",
Files.readAllBytes(Path.of("sample_data", videoFilename))
));
submissionService.createSubmission(gymId, new ExerciseSubmissionPayload( submissionService.createSubmission(gymId, new ExerciseSubmissionPayload(
name, name,
exercise.getShortName(), exercise.getShortName(),
weight.floatValue(), weight.floatValue(),
unit.name(), unit.name(),
reps, reps,
uploadResp.id() video.id()
)); ));
} catch (IOException e) { videoIds.add(video.id());
e.printStackTrace();
}
}); });
int count = videoIds.size();
while (!videoIds.isEmpty()) {
log.info("Waiting for {} / {} videos to finish processing...", videoIds.size(), count);
Set<String> removalSet = new HashSet<>();
for (var videoId : videoIds) {
String status = cdnClient.uploads.getVideoProcessingStatus(videoId).status();
if (status.equalsIgnoreCase("COMPLETED") || status.equalsIgnoreCase("FAILED")) {
removalSet.add(videoId);
}
}
videoIds.removeAll(removalSet);
Thread.sleep(1000);
}
loadCsv("users", record -> { loadCsv("users", record -> {
String email = record.get(0); String email = record.get(0);
String password = record.get(1); String password = record.get(1);
@ -156,12 +176,21 @@ public class SampleDataLoader implements ApplicationListener<ContextRefreshedEve
}); });
} }
private void loadCsv(String csvName, Consumer<CSVRecord> recordConsumer) throws IOException { @FunctionalInterface
interface ThrowableConsumer<T> {
void accept(T item) throws Exception;
}
private void loadCsv(String csvName, ThrowableConsumer<CSVRecord> recordConsumer) throws IOException {
String path = "sample_data/" + csvName + ".csv"; String path = "sample_data/" + csvName + ".csv";
log.info("Loading data from {}...", path); log.info("Loading data from {}...", path);
var reader = new FileReader(path); var reader = new FileReader(path);
for (var record : CSVFormat.DEFAULT.parse(reader)) { for (var record : CSVFormat.DEFAULT.parse(reader)) {
try {
recordConsumer.accept(record); recordConsumer.accept(record);
} catch (Exception e) {
e.printStackTrace();
}
} }
} }
} }

View File

@ -16,3 +16,4 @@ spring.mail.properties.mail.smtp.timeout=10000
app.auth.private-key-location=./private_key.der app.auth.private-key-location=./private_key.der
app.web-origin=http://localhost:9000 app.web-origin=http://localhost:9000
app.cdn-origin=http://localhost:8082

View File

@ -31,3 +31,5 @@ build/
### VS Code ### ### VS Code ###
.vscode/ .vscode/
cdn-files/

View File

@ -1,5 +1,6 @@
package nl.andrewlalis.gymboardcdn; package nl.andrewlalis.gymboardcdn;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@ -14,6 +15,8 @@ import org.springframework.web.cors.UrlBasedCorsConfigurationSource;
public class Config { public class Config {
@Value("${app.web-origin}") @Value("${app.web-origin}")
private String webOrigin; private String webOrigin;
@Value("${app.api-origin}")
private String apiOrigin;
/** /**
* Defines the CORS configuration for this API, which is to say that we * Defines the CORS configuration for this API, which is to say that we
@ -27,11 +30,16 @@ public class Config {
final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); final UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
final CorsConfiguration config = new CorsConfiguration(); final CorsConfiguration config = new CorsConfiguration();
config.setAllowCredentials(true); config.setAllowCredentials(true);
// Don't do this in production, use a proper list of allowed origins
config.addAllowedOriginPattern(webOrigin); config.addAllowedOriginPattern(webOrigin);
config.addAllowedOriginPattern(apiOrigin);
config.addAllowedHeader("*"); config.addAllowedHeader("*");
config.addAllowedMethod("*"); config.addAllowedMethod("*");
source.registerCorsConfiguration("/**", config); source.registerCorsConfiguration("/**", config);
return source; return source;
} }
@Bean
public ULID ulid() {
return new ULID();
}
} }

View File

@ -0,0 +1,9 @@
package nl.andrewlalis.gymboardcdn.api;
public record FileMetadataResponse(
String filename,
String mimeType,
long size,
String uploadedAt,
boolean availableForDownload
) {}

View File

@ -1,5 +1,5 @@
package nl.andrewlalis.gymboardcdn.api; package nl.andrewlalis.gymboardcdn.api;
public record FileUploadResponse( public record FileUploadResponse(
String identifier String id
) {} ) {}

View File

@ -1,9 +1,12 @@
package nl.andrewlalis.gymboardcdn.api; package nl.andrewlalis.gymboardcdn.api;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.UploadService; import nl.andrewlalis.gymboardcdn.service.UploadService;
import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController @RestController
public class UploadController { public class UploadController {
@ -13,13 +16,23 @@ public class UploadController {
this.uploadService = uploadService; this.uploadService = uploadService;
} }
@PostMapping(path = "/uploads/video", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) @PostMapping(path = "/uploads/video", consumes = {"video/mp4"})
public FileUploadResponse uploadVideo(@RequestParam MultipartFile file) { public FileUploadResponse uploadVideo(HttpServletRequest request) {
return uploadService.processableVideoUpload(file); return uploadService.processableVideoUpload(request);
} }
@GetMapping(path = "/uploads/video/{identifier}/status") @GetMapping(path = "/uploads/video/{id}/status")
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String identifier) { public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) {
return uploadService.getVideoProcessingStatus(identifier); return uploadService.getVideoProcessingStatus(id);
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
uploadService.streamFile(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
return uploadService.getFileMetadata(id);
} }
} }

View File

@ -1,6 +1,9 @@
package nl.andrewlalis.gymboardcdn.model; package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.*; import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
import org.hibernate.annotations.CreationTimestamp; import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -8,9 +11,12 @@ import java.time.LocalDateTime;
@Entity @Entity
@Table(name = "stored_file") @Table(name = "stored_file")
public class StoredFile { public class StoredFile {
/**
* ULID-based unique file identifier.
*/
@Id @Id
@GeneratedValue(strategy = GenerationType.IDENTITY) @Column(nullable = false, updatable = false, length = 26)
private Long id; private String id;
@CreationTimestamp @CreationTimestamp
private LocalDateTime createdAt; private LocalDateTime createdAt;
@ -27,13 +33,6 @@ public class StoredFile {
@Column(nullable = false, updatable = false) @Column(nullable = false, updatable = false)
private String name; private String name;
/**
* The internal id that's used to find this file wherever it's placed on
* our service's storage. It is universally unique.
*/
@Column(nullable = false, updatable = false, unique = true)
private String identifier;
/** /**
* The type of the file. * The type of the file.
*/ */
@ -48,15 +47,15 @@ public class StoredFile {
public StoredFile() {} public StoredFile() {}
public StoredFile(String name, String identifier, String mimeType, long size, LocalDateTime uploadedAt) { public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) {
this.id = id;
this.name = name; this.name = name;
this.identifier = identifier;
this.mimeType = mimeType; this.mimeType = mimeType;
this.size = size; this.size = size;
this.uploadedAt = uploadedAt; this.uploadedAt = uploadedAt;
} }
public Long getId() { public String getId() {
return id; return id;
} }
@ -68,10 +67,6 @@ public class StoredFile {
return name; return name;
} }
public String getIdentifier() {
return identifier;
}
public String getMimeType() { public String getMimeType() {
return mimeType; return mimeType;
} }

View File

@ -3,10 +3,6 @@ package nl.andrewlalis.gymboardcdn.model;
import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository @Repository
public interface StoredFileRepository extends JpaRepository<StoredFile, Long> { public interface StoredFileRepository extends JpaRepository<StoredFile, String> {
Optional<StoredFile> findByIdentifier(String identifier);
boolean existsByIdentifier(String identifier);
} }

View File

@ -46,7 +46,7 @@ public class VideoProcessingTask {
* The identifier that will be used to identify the final video, if it * The identifier that will be used to identify the final video, if it
* is processed successfully. * is processed successfully.
*/ */
@Column(nullable = false) @Column(nullable = false, updatable = false, length = 26)
private String videoIdentifier; private String videoIdentifier;
public VideoProcessingTask() {} public VideoProcessingTask() {}

View File

@ -1,7 +1,7 @@
package nl.andrewlalis.gymboardcdn.service; package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; import nl.andrewlalis.gymboardcdn.util.ULID;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -13,7 +13,6 @@ import java.io.InputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.Random;
/** /**
* The service that manages storing and retrieving files from a base filesystem. * The service that manages storing and retrieving files from a base filesystem.
@ -28,52 +27,29 @@ public class FileService {
@Value("${app.files.temp-dir}") @Value("${app.files.temp-dir}")
private String tempDir; private String tempDir;
private final StoredFileRepository storedFileRepository; private final ULID ulid;
private final VideoProcessingTaskRepository videoProcessingTaskRepository;
public FileService(StoredFileRepository storedFileRepository, VideoProcessingTaskRepository videoProcessingTaskRepository) { public FileService(ULID ulid) {
this.storedFileRepository = storedFileRepository; this.ulid = ulid;
this.videoProcessingTaskRepository = videoProcessingTaskRepository;
} }
public Path getStorageDirForTime(LocalDateTime time) throws IOException { public Path getStoragePathForFile(StoredFile file) throws IOException {
Path dir = getStorageDir() LocalDateTime time = file.getUploadedAt();
Path dir = Path.of(storageDir)
.resolve(Integer.toString(time.getYear())) .resolve(Integer.toString(time.getYear()))
.resolve(Integer.toString(time.getMonthValue())) .resolve(Integer.toString(time.getMonthValue()))
.resolve(Integer.toString(time.getDayOfMonth())); .resolve(Integer.toString(time.getDayOfMonth()));
if (Files.notExists(dir)) Files.createDirectories(dir); if (Files.notExists(dir)) Files.createDirectories(dir);
return dir; return dir.resolve(file.getId());
} }
public String createNewFileIdentifier() { public String createNewFileIdentifier() {
String ident = generateRandomIdentifier(); return ulid.nextULID();
int attempts = 0;
while (storedFileRepository.existsByIdentifier(ident) || videoProcessingTaskRepository.existsByVideoIdentifier(ident)) {
ident = generateRandomIdentifier();
attempts++;
if (attempts > 10) {
log.warn("Took more than 10 attempts to generate a unique file identifier.");
}
if (attempts > 100) {
log.error("Couldn't generate a unique file identifier after 100 attempts. Quitting!");
throw new RuntimeException("Couldn't generate a unique file identifier.");
}
}
return ident;
} }
private String generateRandomIdentifier() { public Path saveToTempFile(InputStream in, String filename) throws IOException {
StringBuilder sb = new StringBuilder(9);
String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
Random rand = new Random();
for (int i = 0; i < 9; i++) sb.append(alphabet.charAt(rand.nextInt(alphabet.length())));
return sb.toString();
}
public Path saveToTempFile(MultipartFile file) throws IOException {
Path tempDir = getTempDir(); Path tempDir = getTempDir();
String suffix = null; String suffix = null;
String filename = file.getOriginalFilename();
if (filename != null) { if (filename != null) {
int idx = filename.lastIndexOf('.'); int idx = filename.lastIndexOf('.');
if (idx >= 0) { if (idx >= 0) {
@ -81,12 +57,10 @@ public class FileService {
} }
} }
Path tempFile = Files.createTempFile(tempDir, null, suffix); Path tempFile = Files.createTempFile(tempDir, null, suffix);
file.transferTo(tempFile); try (var out = Files.newOutputStream(tempFile)) {
return tempFile; in.transferTo(out);
} }
return tempFile;
public Path saveToStorage(String filename, InputStream in) throws IOException {
throw new RuntimeException("Not implemented!");
} }
private Path getStorageDir() throws IOException { private Path getStorageDir() throws IOException {

View File

@ -1,7 +1,11 @@
package nl.andrewlalis.gymboardcdn.service; package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse; import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
@ -14,7 +18,9 @@ import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException; import org.springframework.web.server.ResponseStatusException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.format.DateTimeFormatter;
@Service @Service
public class UploadService { public class UploadService {
@ -32,11 +38,20 @@ public class UploadService {
this.fileService = fileService; this.fileService = fileService;
} }
/**
* Handles uploading of a processable video file that will be processed
* before being stored in the system.
* @param request The request from which we can read the file.
* @return A response that contains an identifier that can be used to check
* the status of the video processing, and eventually fetch the video.
*/
@Transactional @Transactional
public FileUploadResponse processableVideoUpload(MultipartFile file) { public FileUploadResponse processableVideoUpload(HttpServletRequest request) {
Path tempFile; Path tempFile;
String filename = request.getHeader("X-Gymboard-Filename");
if (filename == null) filename = "unnamed.mp4";
try { try {
tempFile = fileService.saveToTempFile(file); tempFile = fileService.saveToTempFile(request.getInputStream(), filename);
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to save video upload to temp file.", e); log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
@ -44,17 +59,64 @@ public class UploadService {
String identifier = fileService.createNewFileIdentifier(); String identifier = fileService.createNewFileIdentifier();
videoTaskRepository.save(new VideoProcessingTask( videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.WAITING, VideoProcessingTask.Status.WAITING,
file.getOriginalFilename(), filename,
tempFile.toString(), tempFile.toString(),
identifier identifier
)); ));
return new FileUploadResponse(identifier); return new FileUploadResponse(identifier);
} }
/**
* Gets the status of a video processing task.
* @param id The video identifier.
* @return The status of the video processing task.
*/
@Transactional(readOnly = true) @Transactional(readOnly = true)
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String identifier) { public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String id) {
VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(identifier) VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(task.getStatus().name()); return new VideoProcessingTaskStatusResponse(task.getStatus().name());
} }
/**
* Streams the contents of a stored file to a client via the Http response.
* @param id The file's unique identifier.
* @param response The response to stream the content to.
*/
@Transactional(readOnly = true)
public void streamFile(String id, HttpServletResponse response) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = fileService.getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
@Transactional(readOnly = true)
public FileMetadataResponse getFileMetadata(String id) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
try {
Path filePath = fileService.getStoragePathForFile(file);
boolean exists = Files.exists(filePath);
return new FileMetadataResponse(
file.getName(),
file.getMimeType(),
file.getSize(),
file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
exists
);
} catch (IOException e) {
log.error("Couldn't get path to stored file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
} }

View File

@ -14,7 +14,6 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDateTime;
import java.util.List; import java.util.List;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -77,21 +76,21 @@ public class VideoProcessingService {
} }
// And finally, copy the output to the final location. // And finally, copy the output to the final location.
LocalDateTime uploadedAt = task.getCreatedAt();
try { try {
Path finalFilePath = fileService.getStorageDirForTime(uploadedAt) StoredFile storedFile = new StoredFile(
.resolve(task.getVideoIdentifier()); task.getVideoIdentifier(),
task.getFilename(),
"video/mp4",
Files.size(ffmpegOutputFile),
task.getCreatedAt()
);
Path finalFilePath = fileService.getStoragePathForFile(storedFile);
Files.move(ffmpegOutputFile, finalFilePath); Files.move(ffmpegOutputFile, finalFilePath);
Files.deleteIfExists(tempFile); Files.deleteIfExists(tempFile);
Files.deleteIfExists(ffmpegOutputFile); Files.deleteIfExists(ffmpegOutputFile);
storedFileRepository.saveAndFlush(new StoredFile( storedFileRepository.saveAndFlush(storedFile);
task.getFilename(),
task.getVideoIdentifier(),
"video/mp4",
Files.size(ffmpegOutputFile),
uploadedAt
));
updateTask(task, VideoProcessingTask.Status.COMPLETED); updateTask(task, VideoProcessingTask.Status.COMPLETED);
log.info("Finished processing video {}.", task.getVideoIdentifier());
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to copy processed video to final storage location.", e); log.error("Failed to copy processed video to final storage location.", e);
updateTask(task, VideoProcessingTask.Status.FAILED); updateTask(task, VideoProcessingTask.Status.FAILED);
@ -113,7 +112,8 @@ public class VideoProcessingService {
Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log"); Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log");
Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log"); Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log");
final String[] command = { final String[] command = {
"ffmpeg", "-i", inFile.getFileName().toString(), "ffmpeg",
"-i", inFile.getFileName().toString(),
"-vf", "scale=640x480:flags=lanczos", "-vf", "scale=640x480:flags=lanczos",
"-vcodec", "libx264", "-vcodec", "libx264",
"-crf", "28", "-crf", "28",

View File

@ -0,0 +1,456 @@
package nl.andrewlalis.gymboardcdn.util;
/*
* sulky-modules - several general-purpose modules.
* Copyright (C) 2007-2019 Joern Huxhorn
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* Copyright 2007-2019 Joern Huxhorn
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Serializable;
import java.security.SecureRandom;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
/*
* https://github.com/ulid/spec
*/
@SuppressWarnings("PMD.ShortClassName")
public class ULID
{
private static final char[] ENCODING_CHARS = {
'0','1','2','3','4','5','6','7','8','9',
'A','B','C','D','E','F','G','H','J','K',
'M','N','P','Q','R','S','T','V','W','X',
'Y','Z',
};
private static final byte[] DECODING_CHARS = {
// 0
-1, -1, -1, -1, -1, -1, -1, -1,
// 8
-1, -1, -1, -1, -1, -1, -1, -1,
// 16
-1, -1, -1, -1, -1, -1, -1, -1,
// 24
-1, -1, -1, -1, -1, -1, -1, -1,
// 32
-1, -1, -1, -1, -1, -1, -1, -1,
// 40
-1, -1, -1, -1, -1, -1, -1, -1,
// 48
0, 1, 2, 3, 4, 5, 6, 7,
// 56
8, 9, -1, -1, -1, -1, -1, -1,
// 64
-1, 10, 11, 12, 13, 14, 15, 16,
// 72
17, 1, 18, 19, 1, 20, 21, 0,
// 80
22, 23, 24, 25, 26, -1, 27, 28,
// 88
29, 30, 31, -1, -1, -1, -1, -1,
// 96
-1, 10, 11, 12, 13, 14, 15, 16,
// 104
17, 1, 18, 19, 1, 20, 21, 0,
// 112
22, 23, 24, 25, 26, -1, 27, 28,
// 120
29, 30, 31,
};
private static final int MASK = 0x1F;
private static final int MASK_BITS = 5;
private static final long TIMESTAMP_OVERFLOW_MASK = 0xFFFF_0000_0000_0000L;
private static final long TIMESTAMP_MSB_MASK = 0xFFFF_FFFF_FFFF_0000L;
private static final long RANDOM_MSB_MASK = 0xFFFFL;
private final Random random;
public ULID()
{
this(new SecureRandom());
}
public ULID(Random random)
{
Objects.requireNonNull(random, "random must not be null!");
this.random = random;
}
public void appendULID(StringBuilder stringBuilder)
{
Objects.requireNonNull(stringBuilder, "stringBuilder must not be null!");
internalAppendULID(stringBuilder, System.currentTimeMillis(), random);
}
public String nextULID()
{
return nextULID(System.currentTimeMillis());
}
public String nextULID(long timestamp)
{
return internalUIDString(timestamp, random);
}
public Value nextValue()
{
return nextValue(System.currentTimeMillis());
}
public Value nextValue(long timestamp)
{
return internalNextValue(timestamp, random);
}
/**
* Returns the next monotonic value. If an overflow happened while incrementing
* the random part of the given previous ULID value then the returned value will
* have a zero random part.
*
* @param previousUlid the previous ULID value.
* @return the next monotonic value.
*/
public Value nextMonotonicValue(Value previousUlid)
{
return nextMonotonicValue(previousUlid, System.currentTimeMillis());
}
/**
* Returns the next monotonic value. If an overflow happened while incrementing
* the random part of the given previous ULID value then the returned value will
* have a zero random part.
*
* @param previousUlid the previous ULID value.
* @param timestamp the timestamp of the next ULID value.
* @return the next monotonic value.
*/
public Value nextMonotonicValue(Value previousUlid, long timestamp)
{
Objects.requireNonNull(previousUlid, "previousUlid must not be null!");
if(previousUlid.timestamp() == timestamp)
{
return previousUlid.increment();
}
return nextValue(timestamp);
}
/**
* Returns the next monotonic value or empty if an overflow happened while incrementing
* the random part of the given previous ULID value.
*
* @param previousUlid the previous ULID value.
* @return the next monotonic value or empty if an overflow happened.
*/
public Optional<Value> nextStrictlyMonotonicValue(Value previousUlid)
{
return nextStrictlyMonotonicValue(previousUlid, System.currentTimeMillis());
}
/**
* Returns the next monotonic value or empty if an overflow happened while incrementing
* the random part of the given previous ULID value.
*
* @param previousUlid the previous ULID value.
* @param timestamp the timestamp of the next ULID value.
* @return the next monotonic value or empty if an overflow happened.
*/
public Optional<Value> nextStrictlyMonotonicValue(Value previousUlid, long timestamp)
{
Value result = nextMonotonicValue(previousUlid, timestamp);
if(result.compareTo(previousUlid) < 1)
{
return Optional.empty();
}
return Optional.of(result);
}
public static Value parseULID(String ulidString)
{
Objects.requireNonNull(ulidString, "ulidString must not be null!");
if(ulidString.length() != 26)
{
throw new IllegalArgumentException("ulidString must be exactly 26 chars long.");
}
String timeString = ulidString.substring(0, 10);
long time = internalParseCrockford(timeString);
if ((time & TIMESTAMP_OVERFLOW_MASK) != 0)
{
throw new IllegalArgumentException("ulidString must not exceed '7ZZZZZZZZZZZZZZZZZZZZZZZZZ'!");
}
String part1String = ulidString.substring(10, 18);
String part2String = ulidString.substring(18);
long part1 = internalParseCrockford(part1String);
long part2 = internalParseCrockford(part2String);
long most = (time << 16) | (part1 >>> 24);
long least = part2 | (part1 << 40);
return new Value(most, least);
}
public static Value fromBytes(byte[] data)
{
Objects.requireNonNull(data, "data must not be null!");
if(data.length != 16)
{
throw new IllegalArgumentException("data must be 16 bytes in length!");
}
long mostSignificantBits = 0;
long leastSignificantBits = 0;
for (int i=0; i<8; i++)
{
mostSignificantBits = (mostSignificantBits << 8) | (data[i] & 0xff);
}
for (int i=8; i<16; i++)
{
leastSignificantBits = (leastSignificantBits << 8) | (data[i] & 0xff);
}
return new Value(mostSignificantBits, leastSignificantBits);
}
public static class Value
implements Comparable<Value>, Serializable
{
private static final long serialVersionUID = -3563159514112487717L;
/*
* The most significant 64 bits of this ULID.
*/
private final long mostSignificantBits;
/*
* The least significant 64 bits of this ULID.
*/
private final long leastSignificantBits;
public Value(long mostSignificantBits, long leastSignificantBits)
{
this.mostSignificantBits = mostSignificantBits;
this.leastSignificantBits = leastSignificantBits;
}
/**
* Returns the most significant 64 bits of this ULID's 128 bit value.
*
* @return The most significant 64 bits of this ULID's 128 bit value
*/
public long getMostSignificantBits() {
return mostSignificantBits;
}
/**
* Returns the least significant 64 bits of this ULID's 128 bit value.
*
* @return The least significant 64 bits of this ULID's 128 bit value
*/
public long getLeastSignificantBits() {
return leastSignificantBits;
}
public long timestamp()
{
return mostSignificantBits >>> 16;
}
public byte[] toBytes()
{
byte[] result=new byte[16];
for (int i=0; i<8; i++)
{
result[i] = (byte)((mostSignificantBits >> ((7-i)*8)) & 0xFF);
}
for (int i=8; i<16; i++)
{
result[i] = (byte)((leastSignificantBits >> ((15-i)*8)) & 0xFF);
}
return result;
}
public Value increment()
{
long lsb = leastSignificantBits;
if(lsb != 0xFFFF_FFFF_FFFF_FFFFL)
{
return new Value(mostSignificantBits, lsb+1);
}
long msb = mostSignificantBits;
if((msb & RANDOM_MSB_MASK) != RANDOM_MSB_MASK)
{
return new Value(msb + 1, 0);
}
return new Value(msb & TIMESTAMP_MSB_MASK, 0);
}
@Override
public int hashCode() {
long hilo = mostSignificantBits ^ leastSignificantBits;
return ((int)(hilo >> 32)) ^ (int) hilo;
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Value value = (Value) o;
return mostSignificantBits == value.mostSignificantBits
&& leastSignificantBits == value.leastSignificantBits;
}
@Override
public int compareTo(Value val)
{
// The ordering is intentionally set up so that the ULIDs
// can simply be numerically compared as two numbers
return (this.mostSignificantBits < val.mostSignificantBits ? -1 :
(this.mostSignificantBits > val.mostSignificantBits ? 1 :
(this.leastSignificantBits < val.leastSignificantBits ? -1 :
(this.leastSignificantBits > val.leastSignificantBits ? 1 :
0))));
}
@Override
public String toString()
{
char[] buffer = new char[26];
internalWriteCrockford(buffer, timestamp(), 10, 0);
long value = ((mostSignificantBits & 0xFFFFL) << 24);
long interim = (leastSignificantBits >>> 40);
value = value | interim;
internalWriteCrockford(buffer, value, 8, 10);
internalWriteCrockford(buffer, leastSignificantBits, 8, 18);
return new String(buffer);
}
}
/*
* http://crockford.com/wrmg/base32.html
*/
static void internalAppendCrockford(StringBuilder builder, long value, int count)
{
for(int i = count-1; i >= 0; i--)
{
int index = (int)((value >>> (i * MASK_BITS)) & MASK);
builder.append(ENCODING_CHARS[index]);
}
}
static long internalParseCrockford(String input)
{
Objects.requireNonNull(input, "input must not be null!");
int length = input.length();
if(length > 12)
{
throw new IllegalArgumentException("input length must not exceed 12 but was "+length+"!");
}
long result = 0;
for(int i=0;i<length;i++)
{
char current = input.charAt(i);
byte value = -1;
if(current < DECODING_CHARS.length)
{
value = DECODING_CHARS[current];
}
if(value < 0)
{
throw new IllegalArgumentException("Illegal character '"+current+"'!");
}
result |= ((long)value) << ((length - 1 - i)*MASK_BITS);
}
return result;
}
/*
* http://crockford.com/wrmg/base32.html
*/
static void internalWriteCrockford(char[] buffer, long value, int count, int offset)
{
for(int i = 0; i < count; i++)
{
int index = (int)((value >>> ((count - i - 1) * MASK_BITS)) & MASK);
buffer[offset+i] = ENCODING_CHARS[index];
}
}
static String internalUIDString(long timestamp, Random random)
{
checkTimestamp(timestamp);
char[] buffer = new char[26];
internalWriteCrockford(buffer, timestamp, 10, 0);
// could use nextBytes(byte[] bytes) instead
internalWriteCrockford(buffer, random.nextLong(), 8, 10);
internalWriteCrockford(buffer, random.nextLong(), 8, 18);
return new String(buffer);
}
static void internalAppendULID(StringBuilder builder, long timestamp, Random random)
{
checkTimestamp(timestamp);
internalAppendCrockford(builder, timestamp, 10);
// could use nextBytes(byte[] bytes) instead
internalAppendCrockford(builder, random.nextLong(), 8);
internalAppendCrockford(builder, random.nextLong(), 8);
}
static Value internalNextValue(long timestamp, Random random)
{
checkTimestamp(timestamp);
// could use nextBytes(byte[] bytes) instead
long mostSignificantBits = random.nextLong();
long leastSignificantBits = random.nextLong();
mostSignificantBits &= 0xFFFF;
mostSignificantBits |= (timestamp << 16);
return new Value(mostSignificantBits, leastSignificantBits);
}
private static void checkTimestamp(long timestamp)
{
if((timestamp & TIMESTAMP_OVERFLOW_MASK) != 0)
{
throw new IllegalArgumentException("ULID does not support timestamps after +10889-08-02T05:31:50.655Z!");
}
}
}

View File

@ -7,5 +7,6 @@ spring.jpa.hibernate.ddl-auto=update
server.port=8082 server.port=8082
app.web-origin=http://localhost:9000 app.web-origin=http://localhost:9000
app.api-origin=http://localhost:8080
app.files.storage-dir=./cdn-files/ app.files.storage-dir=./cdn-files/
app.files.temp-dir=./cdn-files/tmp/ app.files.temp-dir=./cdn-files/tmp/

View File

@ -12,5 +12,6 @@ spring.jpa.hibernate.ddl-auto=update
server.port=8082 server.port=8082
app.web-origin=http://localhost:9000 app.web-origin=http://localhost:9000
app.api-origin=http://localhost:8080
app.files.storage-dir=./test-cdn-files/ app.files.storage-dir=./test-cdn-files/
app.files.temp-dir=./test-cdn-files/tmp/ app.files.temp-dir=./test-cdn-files/tmp/