From 91d46244897c47065064afa931c117b90214aa69 Mon Sep 17 00:00:00 2001 From: Andrew Lalis Date: Mon, 3 Apr 2023 13:21:02 +0200 Subject: [PATCH] Start work on no-db implementation, maybe redo it completely? --- .../gymboardcdn/api/FileController.java | 26 +++ .../gymboardcdn/api/UploadController.java | 11 -- .../gymboardcdn/model/FileMetadata.java | 6 + .../gymboardcdn/model/StoredFile.java | 81 ---------- .../model/StoredFileRepository.java | 8 - .../gymboardcdn/service/FileService.java | 149 +++++++++++++----- .../gymboardcdn/service/UploadService.java | 53 +------ .../service/UploadServiceTest.java | 3 - 8 files changed, 144 insertions(+), 193 deletions(-) create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java create mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FileMetadata.java delete mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java delete mode 100644 gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java new file mode 100644 index 0000000..fc89172 --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java @@ -0,0 +1,26 @@ +package nl.andrewlalis.gymboardcdn.api; + +import jakarta.servlet.http.HttpServletResponse; +import nl.andrewlalis.gymboardcdn.service.FileService; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +@RestController +public class FileController { + private final FileService fileService; + + public FileController(FileService fileService) { + this.fileService = fileService; + } + + @GetMapping(path = "/files/{id}") + public void getFile(@PathVariable String id, HttpServletResponse response) { + fileService.streamFile(id, response); + } + + @GetMapping(path = "/files/{id}/metadata") + public FileMetadataResponse getFileMetadata(@PathVariable String id) { + return fileService.getFileMetadata(id); + } +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java index fe8ede2..eb5ade9 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java @@ -1,7 +1,6 @@ package nl.andrewlalis.gymboardcdn.api; import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import nl.andrewlalis.gymboardcdn.service.UploadService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -25,14 +24,4 @@ public class UploadController { public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) { return uploadService.getVideoProcessingStatus(id); } - - @GetMapping(path = "/files/{id}") - public void getFile(@PathVariable String id, HttpServletResponse response) { - uploadService.streamFile(id, response); - } - - @GetMapping(path = "/files/{id}/metadata") - public FileMetadataResponse getFileMetadata(@PathVariable String id) { - return uploadService.getFileMetadata(id); - } } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FileMetadata.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FileMetadata.java new file mode 100644 index 0000000..fdef3ff --- /dev/null +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FileMetadata.java @@ -0,0 +1,6 @@ +package nl.andrewlalis.gymboardcdn.model; + +public class FileMetadata { + public String filename; + public String mimeType; +} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java deleted file mode 100644 index 8b87382..0000000 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java +++ /dev/null @@ -1,81 +0,0 @@ -package nl.andrewlalis.gymboardcdn.model; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import org.hibernate.annotations.CreationTimestamp; - -import java.time.LocalDateTime; - -@Entity -@Table(name = "stored_file") -public class StoredFile { - /** - * ULID-based unique file identifier. - */ - @Id - @Column(nullable = false, updatable = false, length = 26) - private String id; - - @CreationTimestamp - private LocalDateTime createdAt; - - /** - * The timestamp at which the file was originally uploaded. - */ - @Column(nullable = false, updatable = false) - private LocalDateTime uploadedAt; - - /** - * The original filename. - */ - @Column(nullable = false, updatable = false) - private String name; - - /** - * The type of the file. - */ - @Column(updatable = false) - private String mimeType; - - /** - * The file's size on the disk. - */ - @Column(nullable = false, updatable = false) - private long size; - - public StoredFile() {} - - public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) { - this.id = id; - this.name = name; - this.mimeType = mimeType; - this.size = size; - this.uploadedAt = uploadedAt; - } - - public String getId() { - return id; - } - - public LocalDateTime getCreatedAt() { - return createdAt; - } - - public String getName() { - return name; - } - - public String getMimeType() { - return mimeType; - } - - public long getSize() { - return size; - } - - public LocalDateTime getUploadedAt() { - return uploadedAt; - } -} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java deleted file mode 100644 index d4da008..0000000 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java +++ /dev/null @@ -1,8 +0,0 @@ -package nl.andrewlalis.gymboardcdn.model; - -import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.stereotype.Repository; - -@Repository -public interface StoredFileRepository extends JpaRepository { -} diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java index 0f11a71..40b03ed 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java @@ -1,25 +1,26 @@ package nl.andrewlalis.gymboardcdn.service; -import nl.andrewlalis.gymboardcdn.model.StoredFile; -import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletResponse; +import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse; +import nl.andrewlalis.gymboardcdn.model.FileMetadata; import nl.andrewlalis.gymboardcdn.util.ULID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.server.ResponseStatusException; import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Instant; import java.time.LocalDateTime; -import java.util.concurrent.TimeUnit; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; /** * The service that manages storing and retrieving files from a base filesystem. @@ -34,22 +35,22 @@ public class FileService { @Value("${app.files.temp-dir}") private String tempDir; - private final StoredFileRepository storedFileRepository; private final ULID ulid; - public FileService(StoredFileRepository storedFileRepository, ULID ulid) { - this.storedFileRepository = storedFileRepository; + private final ObjectMapper objectMapper = new ObjectMapper(); + + public FileService(ULID ulid) { this.ulid = ulid; } - public Path getStoragePathForFile(StoredFile file) throws IOException { - LocalDateTime time = file.getUploadedAt(); + public Path getStoragePathForFile(String rawId) { + ULID.Value id = ULID.parseULID(rawId); + LocalDateTime time = dateFromULID(id); Path dir = Path.of(storageDir) .resolve(Integer.toString(time.getYear())) .resolve(Integer.toString(time.getMonthValue())) .resolve(Integer.toString(time.getDayOfMonth())); - if (Files.notExists(dir)) Files.createDirectories(dir); - return dir.resolve(file.getId()); + return dir.resolve(rawId); } public String createNewFileIdentifier() { @@ -72,6 +73,12 @@ public class FileService { return tempFile; } + private static LocalDateTime dateFromULID(ULID.Value value) { + return Instant.ofEpochMilli(value.timestamp()) + .atOffset(ZoneOffset.UTC) + .toLocalDateTime(); + } + private Path getStorageDir() throws IOException { Path dir = Path.of(storageDir); if (Files.notExists(dir)) { @@ -88,29 +95,95 @@ public class FileService { return dir; } - /** - * Scheduled task that removes any StoredFile entities for which no more - * physical file exists. - */ - @Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES) - @Transactional - public void removeOrphanedFiles() { - Pageable pageable = PageRequest.of(0, 100, Sort.by(Sort.Direction.DESC, "createdAt")); - Page page = storedFileRepository.findAll(pageable); - while (!page.isEmpty()) { - for (var storedFile : page) { - try { - Path filePath = getStoragePathForFile(storedFile); - if (Files.notExists(filePath)) { - log.warn("Removing stored file {} because it no longer exists on the disk.", storedFile.getId()); - storedFileRepository.delete(storedFile); - } - } catch (IOException e) { - log.error("Couldn't get storage path for stored file.", e); - } + public String saveFile(InputStream in, String filename, String mimeType) throws IOException { + ULID.Value id = ulid.nextValue(); + + FileMetadata meta = new FileMetadata(); + meta.filename = filename; + meta.mimeType = mimeType; + byte[] metaBytes = objectMapper.writeValueAsBytes(meta); + + Path filePath = getStoragePathForFile(id.toString()); + Files.createDirectories(filePath.getParent()); + try (var out = Files.newOutputStream(filePath)) { + // Write metadata first. + ByteBuffer metaBuffer = ByteBuffer.allocate(2 + metaBytes.length); + metaBuffer.putShort((short) metaBytes.length); + metaBuffer.put(metaBytes); + out.write(metaBuffer.array()); + + // Now write real data. + byte[] buffer = new byte[8192]; + int readCount; + while ((readCount = in.read(buffer)) > 0) { + out.write(buffer, 0, readCount); } - pageable = pageable.next(); - page = storedFileRepository.findAll(pageable); + } + return id.toString(); + } + + public FileMetadata readMetadata(InputStream in) throws IOException { + ByteBuffer b = ByteBuffer.allocate(2); + int bytesRead = in.read(b.array()); + if (bytesRead != 2) throw new IOException("Missing metadata length."); + short length = b.getShort(); + b = ByteBuffer.allocate(length); + bytesRead = in.read(b.array()); + if (bytesRead != length) throw new IOException("Metadata body does not equal length header."); + return objectMapper.readValue(b.array(), FileMetadata.class); + } + + /** + * Streams the contents of a stored file to a client via the Http response. + * @param rawId The file's unique identifier. + * @param response The response to stream the content to. + */ + public void streamFile(String rawId, HttpServletResponse response) { + Path filePath = getStoragePathForFile(rawId); + if (!Files.exists(filePath)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + + try (var in = Files.newInputStream(filePath)) { + FileMetadata metadata = readMetadata(in); + response.setContentType(metadata.mimeType); + response.setContentLengthLong(Files.size(filePath)); + } + + ULID.Value id = ULID.parseULID(rawId); + Instant timestamp = Instant.ofEpochMilli(id.timestamp()); + LocalDateTime ldt = LocalDateTime.ofInstant(timestamp, ZoneOffset.UTC); + response.setContentType(file.getMimeType()); + response.setContentLengthLong(file.getSize()); + try { + Path filePath = getStoragePathForFile(file); + try (var in = Files.newInputStream(filePath)) { + in.transferTo(response.getOutputStream()); + } + } catch (IOException e) { + log.error("Failed to write file to response.", e); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + public FileMetadataResponse getFileMetadata(String id) { + Path filePath = getStoragePathForFile(id); + if (!Files.exists(filePath)) { + throw new ResponseStatusException(HttpStatus.NOT_FOUND); + } + try (var in = Files.newInputStream(filePath)) { + FileMetadata metadata = readMetadata(in); + return new FileMetadataResponse( + metadata.filename, + metadata.mimeType, + Files.size(filePath), + Files.getLastModifiedTime(filePath) + .toInstant().atOffset(ZoneOffset.UTC) + .toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), + true + ); + } catch (IOException e) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e); } } } diff --git a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java index 52eac9d..ce4fb29 100644 --- a/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java +++ b/gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java @@ -1,11 +1,8 @@ package nl.andrewlalis.gymboardcdn.service; import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse; import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse; -import nl.andrewlalis.gymboardcdn.model.StoredFile; import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; @@ -17,9 +14,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.server.ResponseStatusException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.time.format.DateTimeFormatter; @Service public class UploadService { @@ -27,14 +22,10 @@ public class UploadService { private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb - private final StoredFileRepository storedFileRepository; private final VideoProcessingTaskRepository videoTaskRepository; private final FileService fileService; - public UploadService(StoredFileRepository storedFileRepository, - VideoProcessingTaskRepository videoTaskRepository, - FileService fileService) { - this.storedFileRepository = storedFileRepository; + public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileService fileService) { this.videoTaskRepository = videoTaskRepository; this.fileService = fileService; } @@ -86,46 +77,4 @@ public class UploadService { .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); return new VideoProcessingTaskStatusResponse(task.getStatus().name()); } - - /** - * Streams the contents of a stored file to a client via the Http response. - * @param id The file's unique identifier. - * @param response The response to stream the content to. - */ - @Transactional(readOnly = true) - public void streamFile(String id, HttpServletResponse response) { - StoredFile file = storedFileRepository.findById(id) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); - response.setContentType(file.getMimeType()); - response.setContentLengthLong(file.getSize()); - try { - Path filePath = fileService.getStoragePathForFile(file); - try (var in = Files.newInputStream(filePath)) { - in.transferTo(response.getOutputStream()); - } - } catch (IOException e) { - log.error("Failed to write file to response.", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); - } - } - - @Transactional(readOnly = true) - public FileMetadataResponse getFileMetadata(String id) { - StoredFile file = storedFileRepository.findById(id) - .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND)); - try { - Path filePath = fileService.getStoragePathForFile(file); - boolean exists = Files.exists(filePath); - return new FileMetadataResponse( - file.getName(), - file.getMimeType(), - file.getSize(), - file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), - exists - ); - } catch (IOException e) { - log.error("Couldn't get path to stored file.", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR); - } - } } diff --git a/gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java b/gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java index d458290..3d093f0 100644 --- a/gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java +++ b/gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java @@ -3,7 +3,6 @@ package nl.andrewlalis.gymboardcdn.service; import jakarta.servlet.ServletInputStream; import jakarta.servlet.http.HttpServletRequest; import nl.andrewlalis.gymboardcdn.api.FileUploadResponse; -import nl.andrewlalis.gymboardcdn.model.StoredFileRepository; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask; import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository; import org.junit.jupiter.api.Test; @@ -27,7 +26,6 @@ public class UploadServiceTest { */ @Test public void processableVideoUploadSuccess() throws IOException { - StoredFileRepository storedFileRepository = Mockito.mock(StoredFileRepository.class); VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class); when(videoTaskRepository.save(any(VideoProcessingTask.class))) .then(returnsFirstArg()); @@ -38,7 +36,6 @@ public class UploadServiceTest { when(fileService.createNewFileIdentifier()).thenReturn("abc"); UploadService uploadService = new UploadService( - storedFileRepository, videoTaskRepository, fileService );