more work.
This commit is contained in:
parent
91d4624489
commit
c00697b3d1
|
|
@ -1,5 +1,6 @@
|
|||
package nl.andrewlalis.gymboardcdn;
|
||||
|
||||
import nl.andrewlalis.gymboardcdn.service.FileStorageService;
|
||||
import nl.andrewlalis.gymboardcdn.util.ULID;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
|
@ -36,4 +37,9 @@ public class Config {
|
|||
public ULID ulid() {
|
||||
return new ULID();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public FileStorageService fileStorageService() {
|
||||
return new FileStorageService(ulid(), "cdn-files");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,40 @@
|
|||
package nl.andrewlalis.gymboardcdn.api;
|
||||
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import nl.andrewlalis.gymboardcdn.service.FileService;
|
||||
import nl.andrewlalis.gymboardcdn.model.FullFileMetadata;
|
||||
import nl.andrewlalis.gymboardcdn.service.FileStorageService;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Controller for general-purpose file access.
|
||||
*/
|
||||
@RestController
|
||||
public class FileController {
|
||||
private final FileService fileService;
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
public FileController(FileService fileService) {
|
||||
this.fileService = fileService;
|
||||
public FileController(FileStorageService fileStorageService) {
|
||||
this.fileStorageService = fileStorageService;
|
||||
}
|
||||
|
||||
@GetMapping(path = "/files/{id}")
|
||||
public void getFile(@PathVariable String id, HttpServletResponse response) {
|
||||
fileService.streamFile(id, response);
|
||||
fileStorageService.streamToHttpResponse(id, response);
|
||||
}
|
||||
|
||||
@GetMapping(path = "/files/{id}/metadata")
|
||||
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
|
||||
return fileService.getFileMetadata(id);
|
||||
public FullFileMetadata getFileMetadata(@PathVariable String id) {
|
||||
try {
|
||||
var data = fileStorageService.getMetadata(id);
|
||||
if (data == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND);
|
||||
return data;
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Couldn't read file metadata.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
package nl.andrewlalis.gymboardcdn.model;
|
||||
|
||||
public record FullFileMetadata(
|
||||
String filename,
|
||||
String mimeType,
|
||||
long size,
|
||||
String createdAt
|
||||
) {}
|
||||
|
|
@ -30,32 +30,14 @@ public class VideoProcessingTask {
|
|||
@Column(nullable = false)
|
||||
private Status status;
|
||||
|
||||
/**
|
||||
* The original filename.
|
||||
*/
|
||||
@Column(nullable = false)
|
||||
private String filename;
|
||||
|
||||
/**
|
||||
* The path to the temporary file that we'll use as input.
|
||||
*/
|
||||
@Column(nullable = false)
|
||||
private String tempFilePath;
|
||||
|
||||
/**
|
||||
* The identifier that will be used to identify the final video, if it
|
||||
* is processed successfully.
|
||||
*/
|
||||
@Column(nullable = false, updatable = false, length = 26)
|
||||
private String videoIdentifier;
|
||||
private String rawUploadFileId;
|
||||
|
||||
public VideoProcessingTask() {}
|
||||
|
||||
public VideoProcessingTask(Status status, String filename, String tempFilePath, String videoIdentifier) {
|
||||
public VideoProcessingTask(Status status, String rawUploadFileId) {
|
||||
this.status = status;
|
||||
this.filename = filename;
|
||||
this.tempFilePath = tempFilePath;
|
||||
this.videoIdentifier = videoIdentifier;
|
||||
this.rawUploadFileId = rawUploadFileId;
|
||||
}
|
||||
|
||||
public Long getId() {
|
||||
|
|
@ -66,10 +48,6 @@ public class VideoProcessingTask {
|
|||
return createdAt;
|
||||
}
|
||||
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
|
||||
public Status getStatus() {
|
||||
return status;
|
||||
}
|
||||
|
|
@ -78,11 +56,7 @@ public class VideoProcessingTask {
|
|||
this.status = status;
|
||||
}
|
||||
|
||||
public String getTempFilePath() {
|
||||
return tempFilePath;
|
||||
}
|
||||
|
||||
public String getVideoIdentifier() {
|
||||
return videoIdentifier;
|
||||
public String getRawUploadFileId() {
|
||||
return rawUploadFileId;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,189 +0,0 @@
|
|||
package nl.andrewlalis.gymboardcdn.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
|
||||
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
|
||||
import nl.andrewlalis.gymboardcdn.util.ULID;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* The service that manages storing and retrieving files from a base filesystem.
|
||||
*/
|
||||
@Service
|
||||
public class FileService {
|
||||
private static final Logger log = LoggerFactory.getLogger(FileService.class);
|
||||
|
||||
@Value("${app.files.storage-dir}")
|
||||
private String storageDir;
|
||||
|
||||
@Value("${app.files.temp-dir}")
|
||||
private String tempDir;
|
||||
|
||||
private final ULID ulid;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
public FileService(ULID ulid) {
|
||||
this.ulid = ulid;
|
||||
}
|
||||
|
||||
public Path getStoragePathForFile(String rawId) {
|
||||
ULID.Value id = ULID.parseULID(rawId);
|
||||
LocalDateTime time = dateFromULID(id);
|
||||
Path dir = Path.of(storageDir)
|
||||
.resolve(Integer.toString(time.getYear()))
|
||||
.resolve(Integer.toString(time.getMonthValue()))
|
||||
.resolve(Integer.toString(time.getDayOfMonth()));
|
||||
return dir.resolve(rawId);
|
||||
}
|
||||
|
||||
public String createNewFileIdentifier() {
|
||||
return ulid.nextULID();
|
||||
}
|
||||
|
||||
public Path saveToTempFile(InputStream in, String filename) throws IOException {
|
||||
Path tempDir = getTempDir();
|
||||
String suffix = null;
|
||||
if (filename != null) {
|
||||
int idx = filename.lastIndexOf('.');
|
||||
if (idx >= 0) {
|
||||
suffix = filename.substring(idx);
|
||||
}
|
||||
}
|
||||
Path tempFile = Files.createTempFile(tempDir, null, suffix);
|
||||
try (var out = Files.newOutputStream(tempFile)) {
|
||||
in.transferTo(out);
|
||||
}
|
||||
return tempFile;
|
||||
}
|
||||
|
||||
private static LocalDateTime dateFromULID(ULID.Value value) {
|
||||
return Instant.ofEpochMilli(value.timestamp())
|
||||
.atOffset(ZoneOffset.UTC)
|
||||
.toLocalDateTime();
|
||||
}
|
||||
|
||||
private Path getStorageDir() throws IOException {
|
||||
Path dir = Path.of(storageDir);
|
||||
if (Files.notExists(dir)) {
|
||||
Files.createDirectories(dir);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
private Path getTempDir() throws IOException {
|
||||
Path dir = Path.of(tempDir);
|
||||
if (Files.notExists(dir)) {
|
||||
Files.createDirectories(dir);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
public String saveFile(InputStream in, String filename, String mimeType) throws IOException {
|
||||
ULID.Value id = ulid.nextValue();
|
||||
|
||||
FileMetadata meta = new FileMetadata();
|
||||
meta.filename = filename;
|
||||
meta.mimeType = mimeType;
|
||||
byte[] metaBytes = objectMapper.writeValueAsBytes(meta);
|
||||
|
||||
Path filePath = getStoragePathForFile(id.toString());
|
||||
Files.createDirectories(filePath.getParent());
|
||||
try (var out = Files.newOutputStream(filePath)) {
|
||||
// Write metadata first.
|
||||
ByteBuffer metaBuffer = ByteBuffer.allocate(2 + metaBytes.length);
|
||||
metaBuffer.putShort((short) metaBytes.length);
|
||||
metaBuffer.put(metaBytes);
|
||||
out.write(metaBuffer.array());
|
||||
|
||||
// Now write real data.
|
||||
byte[] buffer = new byte[8192];
|
||||
int readCount;
|
||||
while ((readCount = in.read(buffer)) > 0) {
|
||||
out.write(buffer, 0, readCount);
|
||||
}
|
||||
}
|
||||
return id.toString();
|
||||
}
|
||||
|
||||
public FileMetadata readMetadata(InputStream in) throws IOException {
|
||||
ByteBuffer b = ByteBuffer.allocate(2);
|
||||
int bytesRead = in.read(b.array());
|
||||
if (bytesRead != 2) throw new IOException("Missing metadata length.");
|
||||
short length = b.getShort();
|
||||
b = ByteBuffer.allocate(length);
|
||||
bytesRead = in.read(b.array());
|
||||
if (bytesRead != length) throw new IOException("Metadata body does not equal length header.");
|
||||
return objectMapper.readValue(b.array(), FileMetadata.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams the contents of a stored file to a client via the Http response.
|
||||
* @param rawId The file's unique identifier.
|
||||
* @param response The response to stream the content to.
|
||||
*/
|
||||
public void streamFile(String rawId, HttpServletResponse response) {
|
||||
Path filePath = getStoragePathForFile(rawId);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
try (var in = Files.newInputStream(filePath)) {
|
||||
FileMetadata metadata = readMetadata(in);
|
||||
response.setContentType(metadata.mimeType);
|
||||
response.setContentLengthLong(Files.size(filePath));
|
||||
}
|
||||
|
||||
ULID.Value id = ULID.parseULID(rawId);
|
||||
Instant timestamp = Instant.ofEpochMilli(id.timestamp());
|
||||
LocalDateTime ldt = LocalDateTime.ofInstant(timestamp, ZoneOffset.UTC);
|
||||
response.setContentType(file.getMimeType());
|
||||
response.setContentLengthLong(file.getSize());
|
||||
try {
|
||||
Path filePath = getStoragePathForFile(file);
|
||||
try (var in = Files.newInputStream(filePath)) {
|
||||
in.transferTo(response.getOutputStream());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to write file to response.", e);
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
public FileMetadataResponse getFileMetadata(String id) {
|
||||
Path filePath = getStoragePathForFile(id);
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
|
||||
}
|
||||
try (var in = Files.newInputStream(filePath)) {
|
||||
FileMetadata metadata = readMetadata(in);
|
||||
return new FileMetadataResponse(
|
||||
metadata.filename,
|
||||
metadata.mimeType,
|
||||
Files.size(filePath),
|
||||
Files.getLastModifiedTime(filePath)
|
||||
.toInstant().atOffset(ZoneOffset.UTC)
|
||||
.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
|
||||
true
|
||||
);
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
package nl.andrewlalis.gymboardcdn.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
|
||||
import nl.andrewlalis.gymboardcdn.model.FullFileMetadata;
|
||||
import nl.andrewlalis.gymboardcdn.util.ULID;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* This service acts as a low-level driver for interacting with the storage
|
||||
* system. This includes reading and writing files and their metadata.
|
||||
* <p>
|
||||
* Files are stored in a top-level directory, then in 3 sub-directories
|
||||
* according to their creation date. So if a file is created on 2023-04-02,
|
||||
* then it will be stored in BASE_DIR/2023/04/02/. All files are uniquely
|
||||
* identified by a ULID; a monotonic, time-sorted id.
|
||||
* </p>
|
||||
* <p>
|
||||
* Each file has a 1Kb (1024 bytes) metadata block baked into it when it's
|
||||
* first saved. This metadata block stores a JSON-serialized set of metadata
|
||||
* properties about the file.
|
||||
* </p>
|
||||
*/
|
||||
public class FileStorageService {
|
||||
private static final int HEADER_SIZE = 1024;
|
||||
|
||||
private final ULID ulid;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final String baseStorageDir;
|
||||
|
||||
public FileStorageService(ULID ulid, String baseStorageDir) {
|
||||
this.ulid = ulid;
|
||||
this.baseStorageDir = baseStorageDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a new file to the storage.
|
||||
* @param in The input stream to the file contents.
|
||||
* @param metadata The file's metadata.
|
||||
* @param maxSize The maximum allowable filesize to download. If the given
|
||||
* input stream has more content than this size, an exception
|
||||
* is thrown.
|
||||
* @return The file's id.
|
||||
* @throws IOException If an error occurs.
|
||||
*/
|
||||
public String save(InputStream in, FileMetadata metadata, long maxSize) throws IOException {
|
||||
ULID.Value id = ulid.nextValue();
|
||||
return save(id, in, metadata, maxSize);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a new file to the storage using a specific file id.
|
||||
* @param id The file id to save to.
|
||||
* @param in The input stream to the file contents.
|
||||
* @param metadata The file's metadata.
|
||||
* @param maxSize The maximum allowable filesize to download. If the given
|
||||
* input stream has more content than this size, an exception
|
||||
* is thrown.
|
||||
* @return The file's id.
|
||||
* @throws IOException If an error occurs.
|
||||
*/
|
||||
public String save(ULID.Value id, InputStream in, FileMetadata metadata, long maxSize) throws IOException {
|
||||
Path filePath = getStoragePathForFile(id.toString());
|
||||
Files.createDirectories(filePath.getParent());
|
||||
try (var out = Files.newOutputStream(filePath)) {
|
||||
writeMetadata(out, metadata);
|
||||
byte[] buffer = new byte[8192];
|
||||
int bytesRead;
|
||||
long totalBytesWritten = 0;
|
||||
while ((bytesRead = in.read(buffer)) > 0) {
|
||||
out.write(buffer, 0, bytesRead);
|
||||
totalBytesWritten += bytesRead;
|
||||
if (maxSize > 0 && totalBytesWritten > maxSize) {
|
||||
out.close();
|
||||
Files.delete(filePath);
|
||||
throw new IOException("File too large.");
|
||||
}
|
||||
}
|
||||
}
|
||||
return id.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets metadata for a file identified by the given id.
|
||||
* @param rawId The file's id.
|
||||
* @return The metadata for the file, or null if no file is found.
|
||||
* @throws IOException If an error occurs while reading metadata.
|
||||
*/
|
||||
public FullFileMetadata getMetadata(String rawId) throws IOException {
|
||||
Path filePath = getStoragePathForFile(rawId);
|
||||
if (Files.notExists(filePath)) return null;
|
||||
try (var in = Files.newInputStream(filePath)) {
|
||||
FileMetadata metadata = readMetadata(in);
|
||||
LocalDateTime date = dateFromULID(ULID.parseULID(rawId));
|
||||
return new FullFileMetadata(
|
||||
metadata.filename,
|
||||
metadata.mimeType,
|
||||
Files.size(filePath) - HEADER_SIZE,
|
||||
date.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams a stored file to an HTTP response. A NOT_FOUND response is sent
|
||||
* if the file doesn't exist. Responses include a cache-control header to
|
||||
* allow clients to cache the response for a long time, as stored files are
|
||||
* considered to be immutable, unless rarely deleted.
|
||||
* @param rawId The file's id.
|
||||
* @param response The HTTP response to write to.
|
||||
*/
|
||||
public void streamToHttpResponse(String rawId, HttpServletResponse response) {
|
||||
Path filePath = getStoragePathForFile(rawId);
|
||||
if (Files.notExists(filePath)) {
|
||||
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
|
||||
return;
|
||||
}
|
||||
|
||||
try (var in = Files.newInputStream(filePath)) {
|
||||
FileMetadata metadata = readMetadata(in);
|
||||
response.setContentType(metadata.mimeType);
|
||||
response.setContentLengthLong(Files.size(filePath) - HEADER_SIZE);
|
||||
response.addHeader("Cache-Control", "max-age=604800, immutable");
|
||||
var out = response.getOutputStream();
|
||||
byte[] buffer = new byte[8192];
|
||||
int bytesRead;
|
||||
while ((bytesRead = in.read(buffer)) > 0) {
|
||||
out.write(buffer, 0, bytesRead);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the path to the physical location of the file with the given id.
|
||||
* Note that this method makes no guarantee as to whether the file exists.
|
||||
* @param rawId The id of the file.
|
||||
* @return The path to the location where the file is stored.
|
||||
*/
|
||||
public Path getStoragePathForFile(String rawId) {
|
||||
ULID.Value id = ULID.parseULID(rawId);
|
||||
LocalDateTime time = dateFromULID(id);
|
||||
Path dir = Path.of(baseStorageDir)
|
||||
.resolve(String.format("%04d", time.getYear()))
|
||||
.resolve(String.format("%02d", time.getMonthValue()))
|
||||
.resolve(String.format("%02d", time.getDayOfMonth()));
|
||||
return dir.resolve(rawId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the file with a given id, if it exists.
|
||||
* @param rawId The file's id.
|
||||
* @throws IOException If an error occurs.
|
||||
*/
|
||||
public void delete(String rawId) throws IOException {
|
||||
Path filePath = getStoragePathForFile(rawId);
|
||||
Files.deleteIfExists(filePath);
|
||||
}
|
||||
|
||||
private static LocalDateTime dateFromULID(ULID.Value value) {
|
||||
return Instant.ofEpochMilli(value.timestamp())
|
||||
.atOffset(ZoneOffset.UTC)
|
||||
.toLocalDateTime();
|
||||
}
|
||||
|
||||
private FileMetadata readMetadata(InputStream in) throws IOException {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
|
||||
int readCount = in.read(buffer.array(), 0, HEADER_SIZE);
|
||||
if (readCount != HEADER_SIZE) throw new IOException("Invalid header.");
|
||||
short metadataBytesLength = buffer.getShort();
|
||||
byte[] metadataBytes = new byte[metadataBytesLength];
|
||||
buffer.get(metadataBytes);
|
||||
return objectMapper.readValue(metadataBytes, FileMetadata.class);
|
||||
}
|
||||
|
||||
private void writeMetadata(OutputStream out, FileMetadata metadata) throws IOException {
|
||||
ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_SIZE);
|
||||
byte[] metadataBytes = objectMapper.writeValueAsBytes(metadata);
|
||||
if (metadataBytes.length > HEADER_SIZE - 2) {
|
||||
throw new IOException("Metadata is too large.");
|
||||
}
|
||||
headerBuffer.putShort((short) metadataBytes.length);
|
||||
headerBuffer.put(metadataBytes);
|
||||
out.write(headerBuffer.array());
|
||||
}
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ package nl.andrewlalis.gymboardcdn.service;
|
|||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
|
||||
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
|
||||
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
|
||||
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
|
||||
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
|
||||
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -14,7 +14,6 @@ import org.springframework.transaction.annotation.Transactional;
|
|||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
@Service
|
||||
public class UploadService {
|
||||
|
|
@ -23,11 +22,11 @@ public class UploadService {
|
|||
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
|
||||
|
||||
private final VideoProcessingTaskRepository videoTaskRepository;
|
||||
private final FileService fileService;
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileService fileService) {
|
||||
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileStorageService fileStorageService) {
|
||||
this.videoTaskRepository = videoTaskRepository;
|
||||
this.fileService = fileService;
|
||||
this.fileStorageService = fileStorageService;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -47,23 +46,23 @@ public class UploadService {
|
|||
if (contentLength > MAX_UPLOAD_SIZE_BYTES) {
|
||||
throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE);
|
||||
}
|
||||
Path tempFile;
|
||||
String filename = request.getHeader("X-Gymboard-Filename");
|
||||
if (filename == null) filename = "unnamed.mp4";
|
||||
FileMetadata metadata = new FileMetadata();
|
||||
metadata.mimeType = request.getContentType();
|
||||
metadata.filename = request.getHeader("X-Gymboard-Filename");
|
||||
if (metadata.filename == null) metadata.filename = "unnamed.mp4";
|
||||
String fileId;
|
||||
try {
|
||||
tempFile = fileService.saveToTempFile(request.getInputStream(), filename);
|
||||
fileId = fileStorageService.save(request.getInputStream(), metadata, contentLength);
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to save video upload to temp file.", e);
|
||||
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
String identifier = fileService.createNewFileIdentifier();
|
||||
videoTaskRepository.save(new VideoProcessingTask(
|
||||
VideoProcessingTask.Status.WAITING,
|
||||
filename,
|
||||
tempFile.toString(),
|
||||
identifier
|
||||
fileId,
|
||||
"bleh"
|
||||
));
|
||||
return new FileUploadResponse(identifier);
|
||||
return new FileUploadResponse("bleh");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
package nl.andrewlalis.gymboardcdn.service;
|
||||
|
||||
import nl.andrewlalis.gymboardcdn.model.StoredFile;
|
||||
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
|
||||
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
|
||||
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -25,21 +23,19 @@ public class VideoProcessingService {
|
|||
|
||||
private final Executor taskExecutor;
|
||||
private final VideoProcessingTaskRepository taskRepo;
|
||||
private final StoredFileRepository storedFileRepository;
|
||||
private final FileService fileService;
|
||||
private final FileStorageService fileStorageService;
|
||||
|
||||
public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, StoredFileRepository storedFileRepository, FileService fileService) {
|
||||
public VideoProcessingService(Executor taskExecutor, VideoProcessingTaskRepository taskRepo, FileStorageService fileStorageService) {
|
||||
this.taskExecutor = taskExecutor;
|
||||
this.taskRepo = taskRepo;
|
||||
this.storedFileRepository = storedFileRepository;
|
||||
this.fileService = fileService;
|
||||
this.fileStorageService = fileStorageService;
|
||||
}
|
||||
|
||||
@Scheduled(fixedDelay = 5, timeUnit = TimeUnit.SECONDS)
|
||||
public void startWaitingTasks() {
|
||||
List<VideoProcessingTask> waitingTasks = taskRepo.findAllByStatusOrderByCreatedAtDesc(VideoProcessingTask.Status.WAITING);
|
||||
for (var task : waitingTasks) {
|
||||
log.info("Queueing processing of video {}.", task.getVideoIdentifier());
|
||||
log.info("Queueing processing of task {}.", task.getId());
|
||||
updateTask(task, VideoProcessingTask.Status.IN_PROGRESS);
|
||||
taskExecutor.execute(() -> processVideo(task));
|
||||
}
|
||||
|
|
@ -51,33 +47,34 @@ public class VideoProcessingService {
|
|||
List<VideoProcessingTask> oldTasks = taskRepo.findAllByCreatedAtBefore(cutoff);
|
||||
for (var task : oldTasks) {
|
||||
if (task.getStatus() == VideoProcessingTask.Status.COMPLETED) {
|
||||
log.info("Deleting completed task for video {}.", task.getVideoIdentifier());
|
||||
log.info("Deleting completed task {}.", task.getId());
|
||||
taskRepo.delete(task);
|
||||
} else if (task.getStatus() == VideoProcessingTask.Status.FAILED) {
|
||||
log.info("Deleting failed task for video {}.", task.getVideoIdentifier());
|
||||
log.info("Deleting failed task {}.", task.getId());
|
||||
taskRepo.delete(task);
|
||||
} else if (task.getStatus() == VideoProcessingTask.Status.IN_PROGRESS) {
|
||||
log.info("Task for video {} was in progress for too long; deleting.", task.getVideoIdentifier());
|
||||
log.info("Task {} was in progress for too long; deleting.", task.getId());
|
||||
taskRepo.delete(task);
|
||||
} else if (task.getStatus() == VideoProcessingTask.Status.WAITING) {
|
||||
log.info("Task for video {} was waiting for too long; deleting.", task.getVideoIdentifier());
|
||||
log.info("Task {} was waiting for too long; deleting.", task.getId());
|
||||
taskRepo.delete(task);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void processVideo(VideoProcessingTask task) {
|
||||
log.info("Started processing video {}.", task.getVideoIdentifier());
|
||||
log.info("Started processing task {}.", task.getId());
|
||||
|
||||
Path tempFile = Path.of(task.getTempFilePath());
|
||||
if (Files.notExists(tempFile) || !Files.isReadable(tempFile)) {
|
||||
log.error("Temp file {} doesn't exist or isn't readable.", tempFile);
|
||||
Path tempFilePath = fileStorageService.getStoragePathForFile(task.getRawUploadFileId());
|
||||
if (Files.notExists(tempFilePath) || !Files.isReadable(tempFilePath)) {
|
||||
log.error("Temp file {} doesn't exist or isn't readable.", tempFilePath);
|
||||
updateTask(task, VideoProcessingTask.Status.FAILED);
|
||||
return;
|
||||
}
|
||||
|
||||
// Then begin running the actual FFMPEG processing.
|
||||
Path tempDir = tempFile.getParent();
|
||||
Path tempDir = tempFilePath.getParent();
|
||||
Files.createTempFile()
|
||||
Path ffmpegOutputFile = tempDir.resolve(task.getVideoIdentifier());
|
||||
try {
|
||||
processVideoWithFFMPEG(tempDir, tempFile, ffmpegOutputFile);
|
||||
|
|
|
|||
|
|
@ -26,29 +26,30 @@ public class UploadServiceTest {
|
|||
*/
|
||||
@Test
|
||||
public void processableVideoUploadSuccess() throws IOException {
|
||||
VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
|
||||
when(videoTaskRepository.save(any(VideoProcessingTask.class)))
|
||||
.then(returnsFirstArg());
|
||||
FileService fileService = Mockito.mock(FileService.class);
|
||||
when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
|
||||
.thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
|
||||
|
||||
when(fileService.createNewFileIdentifier()).thenReturn("abc");
|
||||
|
||||
UploadService uploadService = new UploadService(
|
||||
videoTaskRepository,
|
||||
fileService
|
||||
);
|
||||
HttpServletRequest mockRequest = mock(HttpServletRequest.class);
|
||||
when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
|
||||
when(mockRequest.getHeader("Content-Length")).thenReturn("123");
|
||||
ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
|
||||
when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
|
||||
var expectedResponse = new FileUploadResponse("abc");
|
||||
var response = uploadService.processableVideoUpload(mockRequest);
|
||||
assertEquals(expectedResponse, response);
|
||||
verify(fileService, times(1)).saveToTempFile(any(), any());
|
||||
verify(videoTaskRepository, times(1)).save(any());
|
||||
verify(fileService, times(1)).createNewFileIdentifier();
|
||||
// TODO: Refactor all of this!
|
||||
// VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
|
||||
// when(videoTaskRepository.save(any(VideoProcessingTask.class)))
|
||||
// .then(returnsFirstArg());
|
||||
// FileService fileService = Mockito.mock(FileService.class);
|
||||
// when(fileService.saveToTempFile(any(InputStream.class), any(String.class)))
|
||||
// .thenReturn(Path.of("test-cdn-files", "tmp", "bleh.mp4"));
|
||||
//
|
||||
// when(fileService.createNewFileIdentifier()).thenReturn("abc");
|
||||
//
|
||||
// UploadService uploadService = new UploadService(
|
||||
// videoTaskRepository,
|
||||
// fileService
|
||||
// );
|
||||
// HttpServletRequest mockRequest = mock(HttpServletRequest.class);
|
||||
// when(mockRequest.getHeader("X-Filename")).thenReturn("testing.mp4");
|
||||
// when(mockRequest.getHeader("Content-Length")).thenReturn("123");
|
||||
// ServletInputStream mockRequestInputStream = mock(ServletInputStream.class);
|
||||
// when(mockRequest.getInputStream()).thenReturn(mockRequestInputStream);
|
||||
// var expectedResponse = new FileUploadResponse("abc");
|
||||
// var response = uploadService.processableVideoUpload(mockRequest);
|
||||
// assertEquals(expectedResponse, response);
|
||||
// verify(fileService, times(1)).saveToTempFile(any(), any());
|
||||
// verify(videoTaskRepository, times(1)).save(any());
|
||||
// verify(fileService, times(1)).createNewFileIdentifier();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue