From 7d9c210278459e7a7649541ebde2777c868f0596 Mon Sep 17 00:00:00 2001 From: Andrew Lalis Date: Thu, 2 Feb 2023 17:03:41 +0100 Subject: [PATCH] Added Video Processing logic. --- gymboard-cdn/README.md | 2 + gymboard-cdn/pom.xml | 4 + .../nl/andrewlalis/gymboardcdn/Config.java | 2 + .../andrewlalis/gymboardcdn/FileService.java | 57 ------- .../gymboardcdn/UploadController.java | 15 -- .../gymboardcdn/UploadService.java | 8 - .../gymboardcdn/api/FileUploadResponse.java | 5 + .../gymboardcdn/api/UploadController.java | 25 +++ .../VideoProcessingTaskStatusResponse.java | 5 + .../gymboardcdn/model/StoredFile.java | 86 ++++++++++ .../model/StoredFileRepository.java | 12 ++ .../model/VideoProcessingTask.java | 88 ++++++++++ .../model/VideoProcessingTaskRepository.java | 16 ++ .../service/CommandFailedException.java | 43 +++++ .../gymboardcdn/service/FileService.java | 107 +++++++++++++ .../gymboardcdn/service/UploadService.java | 60 +++++++ .../service/VideoProcessingService.java | 151 ++++++++++++++++++ .../application-development.properties | 4 + 18 files changed, 610 insertions(+), 80 deletions(-) delete mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/FileService.java delete mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadController.java delete mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadService.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/VideoProcessingTaskStatusResponse.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTaskRepository.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/CommandFailedException.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java diff --git a/gymboard-cdn/README.md b/gymboard-cdn/README.md index 6845f65..5b4d896 100644 --- a/gymboard-cdn/README.md +++ b/gymboard-cdn/README.md @@ -1,2 +1,4 @@ # Gymboard CDN A content delivery and management system for Gymboard, which exposes endpoints for uploading and fetching content. + +This service stores file content in a directory defined by the `app.files.storage-dir` configuration property. Within the storage directory, files are stored like so: `////`. diff --git a/gymboard-cdn/pom.xml b/gymboard-cdn/pom.xml index 8677895..cc4ad41 100644 --- a/gymboard-cdn/pom.xml +++ b/gymboard-cdn/pom.xml @@ -21,6 +21,10 @@ org.springframework.boot spring-boot-starter-web + + org.springframework.boot + spring-boot-starter-data-jpa + org.postgresql diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java index 2809cac..eb8a190 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java @@ -4,11 +4,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.annotation.Order; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.CorsConfigurationSource; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; @Configuration +@EnableScheduling public class Config { @Value("${app.web-origin}") private String webOrigin; diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/FileService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/FileService.java deleted file mode 100644 index 88bd847..0000000 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/FileService.java +++ /dev/null @@ -1,57 +0,0 @@ -package nl.andrewlalis.gymboardcdn; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; - -/** - * The service that manages storing and retrieving files from a base filesystem. - */ -@Service -public class FileService { - @Value("${app.files.storage-dir}") - private String storageDir; - - @Value("${app.files.temp-dir}") - private String tempDir; - - public Path saveToTempFile(MultipartFile file) throws IOException { - Path tempDir = getTempDir(); - String suffix = null; - String filename = file.getOriginalFilename(); - if (filename != null) { - int idx = filename.lastIndexOf('.'); - if (idx >= 0) { - suffix = filename.substring(idx); - } - } - Path tempFile = Files.createTempFile(tempDir, null, suffix); - file.transferTo(tempFile); - return tempFile; - } - - public Path saveToStorage(String filename, InputStream in) throws IOException { - throw new RuntimeException("Not implemented!"); - } - - private Path getStorageDir() throws IOException { - Path dir = Path.of(storageDir); - if (Files.notExists(dir)) { - Files.createDirectories(dir); - } - return dir; - } - - private Path getTempDir() throws IOException { - Path dir = Path.of(tempDir); - if (Files.notExists(dir)) { - Files.createDirectories(dir); - } - return dir; - } -} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadController.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadController.java deleted file mode 100644 index 95d4362..0000000 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadController.java +++ /dev/null @@ -1,15 +0,0 @@ -package nl.andrewlalis.gymboardcdn; - -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; - -@RestController -public class UploadController { - @PostMapping(path = "/uploads", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - public void uploadContent(@RequestParam MultipartFile file) { - - } -} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadService.java deleted file mode 100644 index 63f3f69..0000000 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/UploadService.java +++ /dev/null @@ -1,8 +0,0 @@ -package nl.andrewlalis.gymboardcdn; - -import org.springframework.stereotype.Service; - -@Service -public class UploadService { - -} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java new file mode 100644 index 0000000..d120fb4 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileUploadResponse.java @@ -0,0 +1,5 @@ +package nl.andrewlalis.gymboardcdn.api; + +public record FileUploadResponse( + String identifier +) {} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java new file mode 100644 index 0000000..daee8ac --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java @@ -0,0 +1,25 @@ +package nl.andrewlalis.gymboardcdn.api; + +import nl.andrewlalis.gymboardcdn.service.UploadService; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +@RestController +public class UploadController { + private final UploadService uploadService; + + public UploadController(UploadService uploadService) { + this.uploadService = uploadService; + } + + @PostMapping(path = "/uploads/video", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public FileUploadResponse uploadVideo(@RequestParam MultipartFile file) { + return uploadService.processableVideoUpload(file); + } + + @GetMapping(path = "/uploads/video/{identifier}/status") + public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String identifier) { + return uploadService.getVideoProcessingStatus(identifier); + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/VideoProcessingTaskStatusResponse.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/VideoProcessingTaskStatusResponse.java new file mode 100644 index 0000000..49670d4 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/VideoProcessingTaskStatusResponse.java @@ -0,0 +1,5 @@ +package nl.andrewlalis.gymboardcdn.api; + +public record VideoProcessingTaskStatusResponse( + String status +) {} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java new file mode 100644 index 0000000..b19fc62 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java @@ -0,0 +1,86 @@ +package nl.andrewlalis.gymboardcdn.model; + +import jakarta.persistence.*; +import org.hibernate.annotations.CreationTimestamp; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "stored_file") +public class StoredFile { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @CreationTimestamp + private LocalDateTime createdAt; + + /** + * The timestamp at which the file was originally uploaded. + */ + @Column(nullable = false, updatable = false) + private LocalDateTime uploadedAt; + + /** + * The original filename. + */ + @Column(nullable = false, updatable = false) + private String name; + + /** + * The internal id that's used to find this file wherever it's placed on + * our service's storage. It is universally unique. + */ + @Column(nullable = false, updatable = false, unique = true) + private String identifier; + + /** + * The type of the file. + */ + @Column(updatable = false) + private String mimeType; + + /** + * The file's size on the disk. + */ + @Column(nullable = false, updatable = false) + private long size; + + public StoredFile() {} + + public StoredFile(String name, String identifier, String mimeType, long size, LocalDateTime uploadedAt) { + this.name = name; + this.identifier = identifier; + this.mimeType = mimeType; + this.size = size; + this.uploadedAt = uploadedAt; + } + + public Long getId() { + return id; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public String getName() { + return name; + } + + public String getIdentifier() { + return identifier; + } + + public String getMimeType() { + return mimeType; + } + + public long getSize() { + return size; + } + + public LocalDateTime getUploadedAt() { + return uploadedAt; + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java new file mode 100644 index 0000000..10a8d11 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java @@ -0,0 +1,12 @@ +package nl.andrewlalis.gymboardcdn.model; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface StoredFileRepository extends JpaRepository { + Optional findByIdentifier(String identifier); + boolean existsByIdentifier(String identifier); +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java new file mode 100644 index 0000000..d8fc7ea --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java @@ -0,0 +1,88 @@ +package nl.andrewlalis.gymboardcdn.model; + +import jakarta.persistence.*; +import org.hibernate.annotations.CreationTimestamp; + +import java.time.LocalDateTime; + +/** + * An entity to keep track of a task for processing a raw video into a better + * format for Gymboard to serve. + */ +@Entity +@Table(name = "task_video_processing") +public class VideoProcessingTask { + public enum Status { + WAITING, + IN_PROGRESS, + COMPLETED, + FAILED + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @CreationTimestamp + private LocalDateTime createdAt; + + @Enumerated(EnumType.STRING) + @Column(nullable = false) + private Status status; + + /** + * The original filename. + */ + @Column(nullable = false) + private String filename; + + /** + * The path to the temporary file that we'll use as input. + */ + @Column(nullable = false) + private String tempFilePath; + + /** + * The identifier that will be used to identify the final video, if it + * is processed successfully. + */ + @Column(nullable = false) + private String videoIdentifier; + + public VideoProcessingTask() {} + + public VideoProcessingTask(Status status, String filename, String tempFilePath, String videoIdentifier) { + this.status = status; + this.filename = filename; + this.tempFilePath = tempFilePath; + this.videoIdentifier = videoIdentifier; + } + + public Long getId() { + return id; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public String getFilename() { + return filename; + } + + public Status getStatus() { + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + + public String getTempFilePath() { + return tempFilePath; + } + + public String getVideoIdentifier() { + return videoIdentifier; + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTaskRepository.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTaskRepository.java new file mode 100644 index 0000000..f0361e2 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTaskRepository.java @@ -0,0 +1,16 @@ +package nl.andrewlalis.gymboardcdn.model; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; +import java.util.Optional; + +@Repository +public interface VideoProcessingTaskRepository extends JpaRepository { + Optional findByVideoIdentifier(String identifier); + + boolean existsByVideoIdentifier(String identifier); + + List findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status status); +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/CommandFailedException.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/CommandFailedException.java new file mode 100644 index 0000000..9ef064b --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/CommandFailedException.java @@ -0,0 +1,43 @@ +package nl.andrewlalis.gymboardcdn.service; + +import java.io.IOException; +import java.nio.file.Path; + +public class CommandFailedException extends IOException { + private final Path stdoutFile; + private final Path stderrFile; + private final int exitCode; + private final String[] command; + + public CommandFailedException(String[] command, int exitCode, Path stdoutFile, Path stderrFile) { + super( + String.format( + "Command \"%s\" exited with code %d. stdout available at %s and stderr available at %s.", + String.join(" ", command), + exitCode, + stdoutFile.toAbsolutePath(), + stderrFile.toAbsolutePath() + ) + ); + this.command = command; + this.exitCode = exitCode; + this.stdoutFile = stdoutFile; + this.stderrFile = stderrFile; + } + + public Path getStdoutFile() { + return stdoutFile; + } + + public Path getStderrFile() { + return stderrFile; + } + + public int getExitCode() { + return exitCode; + } + + public String[] getCommand() { + return command; + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java new file mode 100644 index 0000000..7bbc614 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java @@ -0,0 +1,107 @@ +package nl.andrewlalis.gymboardcdn.service; + +import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; +import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Random; + +/** + * The service that manages storing and retrieving files from a base filesystem. + */ +@Service +public class FileService { + private static final Logger log = LoggerFactory.getLogger(FileService.class); + + @Value("${app.files.storage-dir}") + private String storageDir; + + @Value("${app.files.temp-dir}") + private String tempDir; + + private final StoredFileRepository storedFileRepository; + private final VideoProcessingTaskRepository videoProcessingTaskRepository; + + public FileService(StoredFileRepository storedFileRepository, VideoProcessingTaskRepository videoProcessingTaskRepository) { + this.storedFileRepository = storedFileRepository; + this.videoProcessingTaskRepository = videoProcessingTaskRepository; + } + + public Path getStorageDirForTime(LocalDateTime time) throws IOException { + Path dir = getStorageDir() + .resolve(Integer.toString(time.getYear())) + .resolve(Integer.toString(time.getMonthValue())) + .resolve(Integer.toString(time.getDayOfMonth())); + if (Files.notExists(dir)) Files.createDirectories(dir); + return dir; + } + + public String createNewFileIdentifier() { + String ident = generateRandomIdentifier(); + int attempts = 0; + while (storedFileRepository.existsByIdentifier(ident) || videoProcessingTaskRepository.existsByVideoIdentifier(ident)) { + ident = generateRandomIdentifier(); + attempts++; + if (attempts > 10) { + log.warn("Took more than 10 attempts to generate a unique file identifier."); + } + if (attempts > 100) { + log.error("Couldn't generate a unique file identifier after 100 attempts. Quitting!"); + throw new RuntimeException("Couldn't generate a unique file identifier."); + } + } + return ident; + } + + private String generateRandomIdentifier() { + StringBuilder sb = new StringBuilder(9); + String alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + Random rand = new Random(); + for (int i = 0; i < 9; i++) sb.append(alphabet.charAt(rand.nextInt(alphabet.length()))); + return sb.toString(); + } + + public Path saveToTempFile(MultipartFile file) throws IOException { + Path tempDir = getTempDir(); + String suffix = null; + String filename = file.getOriginalFilename(); + if (filename != null) { + int idx = filename.lastIndexOf('.'); + if (idx >= 0) { + suffix = filename.substring(idx); + } + } + Path tempFile = Files.createTempFile(tempDir, null, suffix); + file.transferTo(tempFile); + return tempFile; + } + + public Path saveToStorage(String filename, InputStream in) throws IOException { + throw new RuntimeException("Not implemented!"); + } + + private Path getStorageDir() throws IOException { + Path dir = Path.of(storageDir); + if (Files.notExists(dir)) { + Files.createDirectories(dir); + } + return dir; + } + + private Path getTempDir() throws IOException { + Path dir = Path.of(tempDir); + if (Files.notExists(dir)) { + Files.createDirectories(dir); + } + return dir; + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java new file mode 100644 index 0000000..784379d --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java @@ -0,0 +1,60 @@ +package nl.andrewlalis.gymboardcdn.service; + +import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; +import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse; +import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; +import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; +import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; + +import java.io.IOException; +import java.nio.file.Path; + +@Service +public class UploadService { + private static final Logger log = LoggerFactory.getLogger(UploadService.class); + + private final StoredFileRepository storedFileRepository; + private final VideoProcessingTaskRepository videoTaskRepository; + private final FileService fileService; + + public UploadService(StoredFileRepository storedFileRepository, + VideoProcessingTaskRepository videoTaskRepository, + FileService fileService) { + this.storedFileRepository = storedFileRepository; + this.videoTaskRepository = videoTaskRepository; + this.fileService = fileService; + } + + @Transactional + public FileUploadResponse processableVideoUpload(MultipartFile file) { + Path tempFile; + try { + tempFile = fileService.saveToTempFile(file); + } catch (IOException e) { + log.error("Failed to save video upload to temp file.", e); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); + } + String identifier = fileService.createNewFileIdentifier(); + videoTaskRepository.save(new VideoProcessingTask( + VideoProcessingTask.Status.WAITING, + file.getOriginalFilename(), + tempFile.toString(), + identifier + )); + return new FileUploadResponse(identifier); + } + + @Transactional(readOnly = true) + public VideoProcessingTaskStatusResponse getVideoProcessingStatus(String identifier) { + VideoProcessingTask task = videoTaskRepository.findByVideoIdentifier(identifier) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); + return new VideoProcessingTaskStatusResponse(task.getStatus().name()); + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java new file mode 100644 index 0000000..d3e5799 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java @@ -0,0 +1,151 @@ +package nl.andrewlalis.gymboardcdn.service; + +import nl.andrewlalis.gymboardcdn.model.StoredFile; +import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; +import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; +import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +@Service +public class VideoProcessingService { + private static final Logger log = LoggerFactory.getLogger(VideoProcessingService.class); + + private final Executor taskExecutor; + private final VideoProcessingTaskRepository taskRepo; + private final StoredFileRepository storedFileRepository; + private final FileService fileService; + + public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, StoredFileRepository storedFileRepository, FileService fileService) { + this.taskExecutor = taskExecutor; + this.taskRepo = taskRepo; + this.storedFileRepository = storedFileRepository; + this.fileService = fileService; + } + + @Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS) + public void startWaitingTasks() { + List waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING); + for (var task : waitingTasks) { + log.info("Queueing processing of video {}.", task.getVideoIdentifier()); + updateTask(task, VideoProcessingTask.Status.IN_PROGRESS); + taskExecutor.execute(() -> processVideo(task)); + } + } + + private void processVideo(VideoProcessingTask task) { + log.info("Started processing video {}.", task.getVideoIdentifier()); + + Path tempFile = Path.of(task.getTempFilePath()); + if (Files.notExists(tempFile) || !Files.isReadable(tempFile)) { + log.error("Temp file {} doesn't exist or isn't readable.", tempFile); + updateTask(task, VideoProcessingTask.Status.FAILED); + return; + } + + // Then begin running the actual FFMPEG processing. + Path tempDir = tempFile.getParent(); + Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier()); + try { + processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile); + } catch (Exception e) { + e.printStackTrace(); + log.error(""" + Video processing failed for video {}: + Input file: {} + Output file: {} + Exception message: {}""", + task.getVideoIdentifier(), + tempFile, + ffmpegOutputFile, + e.getMessage() + ); + updateTask(task, VideoProcessingTask.Status.FAILED); + return; + } + + // And finally, copy the output to the final location. + LocalDateTime uploadedAt = task.getCreatedAt(); + try { + Path finalFilePath = fileService.getStorageDirForTime(uploadedAt) + .resolve(task.getVideoIdentifier()); + Files.move(ffmpegOutputFile, finalFilePath); + Files.deleteIfExists(tempFile); + Files.deleteIfExists(ffmpegOutputFile); + storedFileRepository.saveAndFlush(new StoredFile( + task.getFilename(), + task.getVideoIdentifier(), + "video/mp4", + Files.size(ffmpegOutputFile), + uploadedAt + )); + updateTask(task, VideoProcessingTask.Status.COMPLETED); + } catch (IOException e) { + log.error("Failed to copy processed video to final storage location.", e); + updateTask(task, VideoProcessingTask.Status.FAILED); + } + } + + /** + * Uses the `ffmpeg` system command to process a raw input video and produce + * a compressed, reduced-size output video that's ready for usage in the + * application. + * @param dir The working directory. + * @param inFile The input file to read from. + * @param outFile The output file to write to. MUST have a ".mp4" extension. + * @throws IOException If a filesystem error occurs. + * @throws CommandFailedException If the ffmpeg command fails. + * @throws InterruptedException If the ffmpeg command is interrupted. + */ + private void processVideoWithFFMPEG(Path dir, Path inFile, Path outFile) throws IOException, InterruptedException { + Path tmpStdout = Files.createTempFile(dir, "stdout-", ".log"); + Path tmpStderr = Files.createTempFile(dir, "stderr-", ".log"); + final String[] command = { + "ffmpeg", "-i", inFile.getFileName().toString(), + "-vf", "scale=640x480:flags=lanczos", + "-vcodec", "libx264", + "-crf", "28", + "-f", "mp4", + outFile.getFileName().toString() + }; + + long startSize = Files.size(inFile); + Instant startTime = Instant.now(); + + Process ffmpegProcess = new ProcessBuilder() + .command(command) + .redirectOutput(tmpStdout.toFile()) + .redirectError(tmpStderr.toFile()) + .directory(dir.toFile()) + .start(); + int result = ffmpegProcess.waitFor(); + if (result != 0) throw new CommandFailedException(command, result, tmpStdout, tmpStderr); + + long endSize = Files.size(outFile); + Duration dur = Duration.between(startTime, Instant.now()); + double reductionFactor = startSize / (double) endSize; + String reductionFactorStr = String.format("%.3f%%", reductionFactor * 100); + log.info("Processed video from {} bytes to {} bytes in {} seconds, {} reduction.", startSize, endSize, dur.getSeconds(), reductionFactorStr); + + // Delete the logs if everything was successful. + Files.deleteIfExists(tmpStdout); + Files.deleteIfExists(tmpStderr); + } + + private void updateTask(VideoProcessingTask task, VideoProcessingTask.Status status) { + task.setStatus(status); + taskRepo.saveAndFlush(task); + } +} diff --git a/gymboard-cdn/src/main/resources/application-development.properties b/gymboard-cdn/src/main/resources/application-development.properties index 14244fa..1f85786 100644 --- a/gymboard-cdn/src/main/resources/application-development.properties +++ b/gymboard-cdn/src/main/resources/application-development.properties @@ -1,3 +1,7 @@ +spring.datasource.username=gymboard-cdn-dev +spring.datasource.password=testpass +spring.datasource.url=jdbc:postgresql://localhost:5433/gymboard-cdn-dev + server.port=8082 app.web-origin=http://localhost:9000