andrew
/
Gymboard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more work.

This commit is contained in:
Andrew Lalis 2023-04-03 17:28:24 +02:00
parent 91d4624489
commit c00697b3d1
9 changed files with 292 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/Config.java
View File

@ -1,5 +1,6 @@
package nl.andrewlalis.gymboardcdn; package nl.andrewlalis.gymboardcdn;
import nl.andrewlalis.gymboardcdn.service.FileStorageService;
import nl.andrewlalis.gymboardcdn.util.ULID; import nl.andrewlalis.gymboardcdn.util.ULID;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
@ -36,4 +37,9 @@ public class Config {
public ULID ulid() { public ULID ulid() {
return new ULID(); return new ULID();
} }
@Bean
public FileStorageService fileStorageService() {
return new FileStorageService(ulid(), "cdn-files");
}
} }

28
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java
View File

@ -1,26 +1,40 @@
package nl.andrewlalis.gymboardcdn.api; package nl.andrewlalis.gymboardcdn.api;
import jakarta.servlet.http.HttpServletResponse; import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.FileService; import nl.andrewlalis.gymboardcdn.model.FullFileMetadata;
import nl.andrewlalis.gymboardcdn.service.FileStorageService;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
/**
* Controller for general-purpose file access.
*/
@RestController @RestController
public class FileController { public class FileController {
private final FileService fileService; private final FileStorageService fileStorageService;
public FileController(FileService fileService) { public FileController(FileStorageService fileStorageService) {
this.fileService = fileService; this.fileStorageService = fileStorageService;
} }
@GetMapping(path = "/files/{id}") @GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) { public void getFile(@PathVariable String id, HttpServletResponse response) {
fileService.streamFile(id, response); fileStorageService.streamToHttpResponse(id, response);
} }
@GetMapping(path = "/files/{id}/metadata") @GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) { public FullFileMetadata getFileMetadata(@PathVariable String id) {
return fileService.getFileMetadata(id); try {
var data = fileStorageService.getMetadata(id);
if (data == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND);
return data;
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Couldn't read file metadata.", e);
}
} }
} }

8
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FullFileMetadata.java Normal file
View File

@ -0,0 +1,8 @@
package nl.andrewlalis.gymboardcdn.model;
public record FullFileMetadata(
String filename,
String mimeType,
long size,
String createdAt
) {}

36
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/VideoProcessingTask.java
View File

@ -30,32 +30,14 @@ public class VideoProcessingTask {
@Column(nullable = false) @Column(nullable = false)
private Status status; private Status status;
/**
* The original filename.
*/
@Column(nullable = false)
private String filename;
/**
* The path to the temporary file that we'll use as input.
*/
@Column(nullable = false)
private String tempFilePath;
/**
* The identifier that will be used to identify the final video, if it
* is processed successfully.
*/
@Column(nullable = false, updatable = false, length = 26) @Column(nullable = false, updatable = false, length = 26)
private String videoIdentifier; private String rawUploadFileId;
public VideoProcessingTask() {} public VideoProcessingTask() {}
public VideoProcessingTask(Status status, String filename, String tempFilePath, String videoIdentifier) { public VideoProcessingTask(Status status, String rawUploadFileId) {
this.status = status; this.status = status;
this.filename = filename; this.rawUploadFileId = rawUploadFileId;
this.tempFilePath = tempFilePath;
this.videoIdentifier = videoIdentifier;
} }
public Long getId() { public Long getId() {
@ -66,10 +48,6 @@ public class VideoProcessingTask {
return createdAt; return createdAt;
} }
public String getFilename() {
return filename;
}
public Status getStatus() { public Status getStatus() {
return status; return status;
} }
@ -78,11 +56,7 @@ public class VideoProcessingTask {
this.status = status; this.status = status;
} }
public String getTempFilePath() { public String getRawUploadFileId() {
return tempFilePath; return rawUploadFileId;
}
public String getVideoIdentifier() {
return videoIdentifier;
} }
} }

189
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java
View File

@ -1,189 +0,0 @@
package nl.andrewlalis.gymboardcdn.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* The service that manages storing and retrieving files from a base filesystem.
*/
@Service
public class FileService {
private static final Logger log = LoggerFactory.getLogger(FileService.class);
@Value("${app.files.storage-dir}")
private String storageDir;
@Value("${app.files.temp-dir}")
private String tempDir;
private final ULID ulid;
private final ObjectMapper objectMapper = new ObjectMapper();
public FileService(ULID ulid) {
this.ulid = ulid;
}
public Path getStoragePathForFile(String rawId) {
ULID.Value id = ULID.parseULID(rawId);
LocalDateTime time = dateFromULID(id);
Path dir = Path.of(storageDir)
.resolve(Integer.toString(time.getYear()))
.resolve(Integer.toString(time.getMonthValue()))
.resolve(Integer.toString(time.getDayOfMonth()));
return dir.resolve(rawId);
}
public String createNewFileIdentifier() {
return ulid.nextULID();
}
public Path saveToTempFile(InputStream in, String filename) throws IOException {
Path tempDir = getTempDir();
String suffix = null;
if (filename != null) {
int idx = filename.lastIndexOf('.');
if (idx >= 0) {
suffix = filename.substring(idx);
}
}
Path tempFile = Files.createTempFile(tempDir, null, suffix);
try (var out = Files.newOutputStream(tempFile)) {
in.transferTo(out);
}
return tempFile;
}
private static LocalDateTime dateFromULID(ULID.Value value) {
return Instant.ofEpochMilli(value.timestamp())
.atOffset(ZoneOffset.UTC)
.toLocalDateTime();
}
private Path getStorageDir() throws IOException {
Path dir = Path.of(storageDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
private Path getTempDir() throws IOException {
Path dir = Path.of(tempDir);
if (Files.notExists(dir)) {
Files.createDirectories(dir);
}
return dir;
}
public String saveFile(InputStream in, String filename, String mimeType) throws IOException {
ULID.Value id = ulid.nextValue();
FileMetadata meta = new FileMetadata();
meta.filename = filename;
meta.mimeType = mimeType;
byte[] metaBytes = objectMapper.writeValueAsBytes(meta);
Path filePath = getStoragePathForFile(id.toString());
Files.createDirectories(filePath.getParent());
try (var out = Files.newOutputStream(filePath)) {
// Write metadata first.
ByteBuffer metaBuffer = ByteBuffer.allocate(2 + metaBytes.length);
metaBuffer.putShort((short) metaBytes.length);
metaBuffer.put(metaBytes);
out.write(metaBuffer.array());
// Now write real data.
byte[] buffer = new byte[8192];
int readCount;
while ((readCount = in.read(buffer)) > 0) {
out.write(buffer, 0, readCount);
}
}
return id.toString();
}
public FileMetadata readMetadata(InputStream in) throws IOException {
ByteBuffer b = ByteBuffer.allocate(2);
int bytesRead = in.read(b.array());
if (bytesRead != 2) throw new IOException("Missing metadata length.");
short length = b.getShort();
b = ByteBuffer.allocate(length);
bytesRead = in.read(b.array());
if (bytesRead != length) throw new IOException("Metadata body does not equal length header.");
return objectMapper.readValue(b.array(), FileMetadata.class);
}
/**
* Streams the contents of a stored file to a client via the Http response.
* @param rawId The file's unique identifier.
* @param response The response to stream the content to.
*/
public void streamFile(String rawId, HttpServletResponse response) {
Path filePath = getStoragePathForFile(rawId);
if (!Files.exists(filePath)) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
response.setContentType(metadata.mimeType);
response.setContentLengthLong(Files.size(filePath));
}
ULID.Value id = ULID.parseULID(rawId);
Instant timestamp = Instant.ofEpochMilli(id.timestamp());
LocalDateTime ldt = LocalDateTime.ofInstant(timestamp, ZoneOffset.UTC);
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
public FileMetadataResponse getFileMetadata(String id) {
Path filePath = getStoragePathForFile(id);
if (!Files.exists(filePath)) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
return new FileMetadataResponse(
metadata.filename,
metadata.mimeType,
Files.size(filePath),
Files.getLastModifiedTime(filePath)
.toInstant().atOffset(ZoneOffset.UTC)
.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
true
);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
}
}
}

200
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileStorageService.java Normal file
View File

@ -0,0 +1,200 @@
package nl.andrewlalis.gymboardcdn.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
import nl.andrewlalis.gymboardcdn.model.FullFileMetadata;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.springframework.http.HttpStatus;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* This service acts as a low-level driver for interacting with the storage
* system. This includes reading and writing files and their metadata.
* <p>
* Files are stored in a top-level directory, then in 3 sub-directories
* according to their creation date. So if a file is created on 2023-04-02,
* then it will be stored in BASE_DIR/2023/04/02/. All files are uniquely
* identified by a ULID; a monotonic, time-sorted id.
* </p>
* <p>
* Each file has a 1Kb (1024 bytes) metadata block baked into it when it's
* first saved. This metadata block stores a JSON-serialized set of metadata
* properties about the file.
* </p>
*/
public class FileStorageService {
private static final int HEADER_SIZE = 1024;
private final ULID ulid;
private final ObjectMapper objectMapper = new ObjectMapper();
private final String baseStorageDir;
public FileStorageService(ULID ulid, String baseStorageDir) {
this.ulid = ulid;
this.baseStorageDir = baseStorageDir;
}
/**
* Saves a new file to the storage.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(InputStream in, FileMetadata metadata, long maxSize) throws IOException {
ULID.Value id = ulid.nextValue();
return save(id, in, metadata, maxSize);
}
/**
* Saves a new file to the storage using a specific file id.
* @param id The file id to save to.
* @param in The input stream to the file contents.
* @param metadata The file's metadata.
* @param maxSize The maximum allowable filesize to download. If the given
* input stream has more content than this size, an exception
* is thrown.
* @return The file's id.
* @throws IOException If an error occurs.
*/
public String save(ULID.Value id, InputStream in, FileMetadata metadata, long maxSize) throws IOException {
Path filePath = getStoragePathForFile(id.toString());
Files.createDirectories(filePath.getParent());
try (var out = Files.newOutputStream(filePath)) {
writeMetadata(out, metadata);
byte[] buffer = new byte[8192];
int bytesRead;
long totalBytesWritten = 0;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
totalBytesWritten += bytesRead;
if (maxSize > 0 && totalBytesWritten > maxSize) {
out.close();
Files.delete(filePath);
throw new IOException("File too large.");
}
}
}
return id.toString();
}
/**
* Gets metadata for a file identified by the given id.
* @param rawId The file's id.
* @return The metadata for the file, or null if no file is found.
* @throws IOException If an error occurs while reading metadata.
*/
public FullFileMetadata getMetadata(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) return null;
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
LocalDateTime date = dateFromULID(ULID.parseULID(rawId));
return new FullFileMetadata(
metadata.filename,
metadata.mimeType,
Files.size(filePath) - HEADER_SIZE,
date.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
);
}
}
/**
* Streams a stored file to an HTTP response. A NOT_FOUND response is sent
* if the file doesn't exist. Responses include a cache-control header to
* allow clients to cache the response for a long time, as stored files are
* considered to be immutable, unless rarely deleted.
* @param rawId The file's id.
* @param response The HTTP response to write to.
*/
public void streamToHttpResponse(String rawId, HttpServletResponse response) {
Path filePath = getStoragePathForFile(rawId);
if (Files.notExists(filePath)) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return;
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
response.setContentType(metadata.mimeType);
response.setContentLengthLong(Files.size(filePath) - HEADER_SIZE);
response.addHeader("Cache-Control", "max-age=604800, immutable");
var out = response.getOutputStream();
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = in.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
}
}
/**
* Gets the path to the physical location of the file with the given id.
* Note that this method makes no guarantee as to whether the file exists.
* @param rawId The id of the file.
* @return The path to the location where the file is stored.
*/
public Path getStoragePathForFile(String rawId) {
ULID.Value id = ULID.parseULID(rawId);
LocalDateTime time = dateFromULID(id);
Path dir = Path.of(baseStorageDir)
.resolve(String.format("%04d", time.getYear()))
.resolve(String.format("%02d", time.getMonthValue()))
.resolve(String.format("%02d", time.getDayOfMonth()));
return dir.resolve(rawId);
}
/**
* Deletes the file with a given id, if it exists.
* @param rawId The file's id.
* @throws IOException If an error occurs.
*/
public void delete(String rawId) throws IOException {
Path filePath = getStoragePathForFile(rawId);
Files.deleteIfExists(filePath);
}
private static LocalDateTime dateFromULID(ULID.Value value) {
return Instant.ofEpochMilli(value.timestamp())
.atOffset(ZoneOffset.UTC)
.toLocalDateTime();
}
private FileMetadata readMetadata(InputStream in) throws IOException {
ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
int readCount = in.read(buffer.array(), 0, HEADER_SIZE);
if (readCount != HEADER_SIZE) throw new IOException("Invalid header.");
short metadataBytesLength = buffer.getShort();
byte[] metadataBytes = new byte[metadataBytesLength];
buffer.get(metadataBytes);
return objectMapper.readValue(metadataBytes, FileMetadata.class);
}
private void writeMetadata(OutputStream out, FileMetadata metadata) throws IOException {
ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE);
byte[] metadataBytes = objectMapper.writeValueAsBytes(metadata);
if (metadataBytes.length > HEADER_SIZE - 2) {
throw new IOException("Metadata is too large.");
}
headerBuffer.putShort((short) metadataBytes.length);
headerBuffer.put(metadataBytes);
out.write(headerBuffer.array());
}
}

27
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java
View File

@ -3,7 +3,7 @@ package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse; import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.FileMetadata;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -14,7 +14,6 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException; import org.springframework.web.server.ResponseStatusException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path;
@Service @Service
public class UploadService { public class UploadService {
@ -23,11 +22,11 @@ public class UploadService {
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final VideoProcessingTaskRepository videoTaskRepository; private final VideoProcessingTaskRepository videoTaskRepository;
private final FileService fileService; private final FileStorageService fileStorageService;
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileService fileService) { public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileStorageService fileStorageService) {
this.videoTaskRepository = videoTaskRepository; this.videoTaskRepository = videoTaskRepository;
this.fileService = fileService; this.fileStorageService = fileStorageService;
} }
/** /**
@ -47,23 +46,23 @@ public class UploadService {
if (contentLength > MAX_UPLOAD_SIZE_BYTES) { if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE); throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
} }
Path tempFile; FileMetadata metadata = new FileMetadata();
String filename = request.getHeader("X-Gymboard-Filename"); metadata.mimeType = request.getContentType();
if (filename == null) filename = "unnamed.mp4"; metadata.filename = request.getHeader("X-Gymboard-Filename");
if (metadata.filename == null) metadata.filename = "unnamed.mp4";
String fileId;
try { try {
tempFile = fileService.saveToTempFile(request.getInputStream(), filename); fileId = fileStorageService.save(request.getInputStream(), metadata, contentLength);
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to save video upload to temp file.", e); log.error("Failed to save video upload to temp file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
} }
String identifier = fileService.createNewFileIdentifier();
videoTaskRepository.save(new VideoProcessingTask( videoTaskRepository.save(new VideoProcessingTask(
VideoProcessingTask.Status.WAITING, VideoProcessingTask.Status.WAITING,
filename, fileId,
tempFile.toString(), "bleh"
identifier
)); ));
return new FileUploadResponse(identifier); return new FileUploadResponse("bleh");
} }
/** /**

31
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/VideoProcessingService.java
View File

@ -1,7 +1,5 @@
package nl.andrewlalis.gymboardcdn.service; package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -25,21 +23,19 @@ public class VideoProcessingService {
private final Executor taskExecutor; private final Executor taskExecutor;
private final VideoProcessingTaskRepository taskRepo; private final VideoProcessingTaskRepository taskRepo;
private final StoredFileRepository storedFileRepository; private final FileStorageService fileStorageService;
private final FileService fileService;
public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, StoredFileRepository storedFileRepository, FileService fileService) { public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, FileStorageService fileStorageService) {
this.taskExecutor = taskExecutor; this.taskExecutor = taskExecutor;
this.taskRepo = taskRepo; this.taskRepo = taskRepo;
this.storedFileRepository = storedFileRepository; this.fileStorageService = fileStorageService;
this.fileService = fileService;
} }
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS) @Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
public void startWaitingTasks() { public void startWaitingTasks() {
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING); List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
for (var task : waitingTasks) { for (var task : waitingTasks) {
log.info("Queueing processing of video {}.", task.getVideoIdentifier()); log.info("Queueing processing of task {}.", task.getId());
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS); updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
taskExecutor.execute(() -> processVideo(task)); taskExecutor.execute(() -> processVideo(task));
} }
@ -51,33 +47,34 @@ public class VideoProcessingService {
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff); List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
for (var task : oldTasks) { for (var task : oldTasks) {
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) { if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
log.info("Deleting completed task for video {}.", task.getVideoIdentifier()); log.info("Deleting completed task {}.", task.getId());
taskRepo.delete(task); taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) { } else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
log.info("Deleting failed task for video {}.", task.getVideoIdentifier()); log.info("Deleting failed task {}.", task.getId());
taskRepo.delete(task); taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) { } else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
log.info("Task for video {} was in progress for too long; deleting.", task.getVideoIdentifier()); log.info("Task {} was in progress for too long; deleting.", task.getId());
taskRepo.delete(task); taskRepo.delete(task);
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) { } else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
log.info("Task for video {} was waiting for too long; deleting.", task.getVideoIdentifier()); log.info("Task {} was waiting for too long; deleting.", task.getId());
taskRepo.delete(task); taskRepo.delete(task);
} }
} }
} }
private void processVideo(VideoProcessingTask task) { private void processVideo(VideoProcessingTask task) {
log.info("Started processing video {}.", task.getVideoIdentifier()); log.info("Started processing task {}.", task.getId());
Path tempFile = Path.of(task.getTempFilePath()); Path tempFilePath = fileStorageService.getStoragePathForFile(task.getRawUploadFileId());
if (Files.notExists(tempFile) || !Files.isReadable(tempFile)) { if (Files.notExists(tempFilePath) || !Files.isReadable(tempFilePath)) {
log.error("Temp file {} doesn't exist or isn't readable.", tempFile); log.error("Temp file {} doesn't exist or isn't readable.", tempFilePath);
updateTask(task, VideoProcessingTask.Status.FAILED); updateTask(task, VideoProcessingTask.Status.FAILED);
return; return;
} }
// Then begin running the actual FFMPEG processing. // Then begin running the actual FFMPEG processing.
Path tempDir = tempFile.getParent(); Path tempDir = tempFilePath.getParent();
Files.createTempFile()
Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier()); Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier());
try { try {
processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile); processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile);

49
gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java
View File

@ -26,29 +26,30 @@ public class UploadServiceTest {
*/ */
@Test @Test
public void processableVideoUploadSuccess() throws IOException { public void processableVideoUploadSuccess() throws IOException {
VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class); // TODO: Refactor all of this!
when(videoTaskRepository.save(any(VideoProcessingTask.class))) // VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
.then(returnsFirstArg()); // when(videoTaskRepository.save(any(VideoProcessingTask.class)))
FileService fileService = Mockito.mock(FileService.class); // .then(returnsFirstArg());
when(fileService.saveToTempFile(any(InputStream.class), any(String.class))) // FileService fileService = Mockito.mock(FileService.class);
.thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4")); // when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
// .thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
when(fileService.createNewFileIdentifier()).thenReturn("abc"); //
// when(fileService.createNewFileIdentifier()).thenReturn("abc");
UploadService uploadService = new UploadService( //
videoTaskRepository, // UploadService uploadService = new UploadService(
fileService // videoTaskRepository,
); // fileService
HttpServletRequest mockRequest = mock(HttpServletRequest.class); // );
when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4"); // HttpServletRequest mockRequest = mock(HttpServletRequest.class);
when(mockRequest.getHeader("Content-Length")).thenReturn("123"); // when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
ServletInputStream mockRequestInputStream = mock(ServletInputStream.class); // when(mockRequest.getHeader("Content-Length")).thenReturn("123");
when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream); // ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
var expectedResponse = new FileUploadResponse("abc"); // when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
var response = uploadService.processableVideoUpload(mockRequest); // var expectedResponse = new FileUploadResponse("abc");
assertEquals(expectedResponse, response); // var response = uploadService.processableVideoUpload(mockRequest);
verify(fileService, times(1)).saveToTempFile(any(), any()); // assertEquals(expectedResponse, response);
verify(videoTaskRepository, times(1)).save(any()); // verify(fileService, times(1)).saveToTempFile(any(), any());
verify(fileService, times(1)).createNewFileIdentifier(); // verify(videoTaskRepository, times(1)).save(any());
// verify(fileService, times(1)).createNewFileIdentifier();
} }
} }