andrew
/
Gymboard
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start work on no-db implementation, maybe redo it completely?

This commit is contained in:
Andrew Lalis 2023-04-03 13:21:02 +02:00
parent b347cd609e
commit 91d4624489
8 changed files with 144 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/FileController.java Normal file
View File

@ -0,0 +1,26 @@
package nl.andrewlalis.gymboardcdn.api;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.FileService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class FileController {
private final FileService fileService;
public FileController(FileService fileService) {
this.fileService = fileService;
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
fileService.streamFile(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
return fileService.getFileMetadata(id);
}
}

11
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/api/UploadController.java
View File

@ -1,7 +1,6 @@
package nl.andrewlalis.gymboardcdn.api;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.service.UploadService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@ -25,14 +24,4 @@ public class UploadController {
public VideoProcessingTaskStatusResponse getVideoProcessingStatus(@PathVariable String id) {
return uploadService.getVideoProcessingStatus(id);
}
@GetMapping(path = "/files/{id}")
public void getFile(@PathVariable String id, HttpServletResponse response) {
uploadService.streamFile(id, response);
}
@GetMapping(path = "/files/{id}/metadata")
public FileMetadataResponse getFileMetadata(@PathVariable String id) {
return uploadService.getFileMetadata(id);
}
}

6
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/FileMetadata.java Normal file
View File

@ -0,0 +1,6 @@
package nl.andrewlalis.gymboardcdn.model;
public class FileMetadata {
public String filename;
public String mimeType;
}

81
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFile.java
View File

@ -1,81 +0,0 @@
package nl.andrewlalis.gymboardcdn.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
import org.hibernate.annotations.CreationTimestamp;
import java.time.LocalDateTime;
@Entity
@Table(name = "stored_file")
public class StoredFile {
/**
* ULID-based unique file identifier.
*/
@Id
@Column(nullable = false, updatable = false, length = 26)
private String id;
@CreationTimestamp
private LocalDateTime createdAt;
/**
* The timestamp at which the file was originally uploaded.
*/
@Column(nullable = false, updatable = false)
private LocalDateTime uploadedAt;
/**
* The original filename.
*/
@Column(nullable = false, updatable = false)
private String name;
/**
* The type of the file.
*/
@Column(updatable = false)
private String mimeType;
/**
* The file's size on the disk.
*/
@Column(nullable = false, updatable = false)
private long size;
public StoredFile() {}
public StoredFile(String id, String name, String mimeType, long size, LocalDateTime uploadedAt) {
this.id = id;
this.name = name;
this.mimeType = mimeType;
this.size = size;
this.uploadedAt = uploadedAt;
}
public String getId() {
return id;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public String getName() {
return name;
}
public String getMimeType() {
return mimeType;
}
public long getSize() {
return size;
}
public LocalDateTime getUploadedAt() {
return uploadedAt;
}
}

8
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/model/StoredFileRepository.java
View File

@ -1,8 +0,0 @@
package nl.andrewlalis.gymboardcdn.model;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface StoredFileRepository extends JpaRepository<StoredFile, String> {
}

137
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/FileService.java
View File

@ -1,25 +1,26 @@
package nl.andrewlalis.gymboardcdn.service;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.model.FileMetadata;
import nl.andrewlalis.gymboardcdn.util.ULID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.concurrent.TimeUnit;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* The service that manages storing and retrieving files from a base filesystem.
@ -34,22 +35,22 @@ public class FileService {
@Value("${app.files.temp-dir}")
private String tempDir;
private final StoredFileRepository storedFileRepository;
private final ULID ulid;
public FileService(StoredFileRepository storedFileRepository, ULID ulid) {
this.storedFileRepository = storedFileRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
public FileService(ULID ulid) {
this.ulid = ulid;
}
public Path getStoragePathForFile(StoredFile file) throws IOException {
LocalDateTime time = file.getUploadedAt();
public Path getStoragePathForFile(String rawId) {
ULID.Value id = ULID.parseULID(rawId);
LocalDateTime time = dateFromULID(id);
Path dir = Path.of(storageDir)
.resolve(Integer.toString(time.getYear()))
.resolve(Integer.toString(time.getMonthValue()))
.resolve(Integer.toString(time.getDayOfMonth()));
if (Files.notExists(dir)) Files.createDirectories(dir);
return dir.resolve(file.getId());
return dir.resolve(rawId);
}
public String createNewFileIdentifier() {
@ -72,6 +73,12 @@ public class FileService {
return tempFile;
}
private static LocalDateTime dateFromULID(ULID.Value value) {
return Instant.ofEpochMilli(value.timestamp())
.atOffset(ZoneOffset.UTC)
.toLocalDateTime();
}
private Path getStorageDir() throws IOException {
Path dir = Path.of(storageDir);
if (Files.notExists(dir)) {
@ -88,29 +95,95 @@ public class FileService {
return dir;
}
public String saveFile(InputStream in, String filename, String mimeType) throws IOException {
ULID.Value id = ulid.nextValue();
FileMetadata meta = new FileMetadata();
meta.filename = filename;
meta.mimeType = mimeType;
byte[] metaBytes = objectMapper.writeValueAsBytes(meta);
Path filePath = getStoragePathForFile(id.toString());
Files.createDirectories(filePath.getParent());
try (var out = Files.newOutputStream(filePath)) {
// Write metadata first.
ByteBuffer metaBuffer = ByteBuffer.allocate(2 + metaBytes.length);
metaBuffer.putShort((short) metaBytes.length);
metaBuffer.put(metaBytes);
out.write(metaBuffer.array());
// Now write real data.
byte[] buffer = new byte[8192];
int readCount;
while ((readCount = in.read(buffer)) > 0) {
out.write(buffer, 0, readCount);
}
}
return id.toString();
}
public FileMetadata readMetadata(InputStream in) throws IOException {
ByteBuffer b = ByteBuffer.allocate(2);
int bytesRead = in.read(b.array());
if (bytesRead != 2) throw new IOException("Missing metadata length.");
short length = b.getShort();
b = ByteBuffer.allocate(length);
bytesRead = in.read(b.array());
if (bytesRead != length) throw new IOException("Metadata body does not equal length header.");
return objectMapper.readValue(b.array(), FileMetadata.class);
}
/**
* Scheduled task that removes any StoredFile entities for which no more
* physical file exists.
* Streams the contents of a stored file to a client via the Http response.
* @param rawId The file's unique identifier.
* @param response The response to stream the content to.
*/
@Scheduled(fixedDelay = 1, timeUnit = TimeUnit.MINUTES)
@Transactional
public void removeOrphanedFiles() {
Pageable pageable = PageRequest.of(0, 100, Sort.by(Sort.Direction.DESC, "createdAt"));
Page<StoredFile> page = storedFileRepository.findAll(pageable);
while (!page.isEmpty()) {
for (var storedFile : page) {
public void streamFile(String rawId, HttpServletResponse response) {
Path filePath = getStoragePathForFile(rawId);
if (!Files.exists(filePath)) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
response.setContentType(metadata.mimeType);
response.setContentLengthLong(Files.size(filePath));
}
ULID.Value id = ULID.parseULID(rawId);
Instant timestamp = Instant.ofEpochMilli(id.timestamp());
LocalDateTime ldt = LocalDateTime.ofInstant(timestamp, ZoneOffset.UTC);
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = getStoragePathForFile(storedFile);
if (Files.notExists(filePath)) {
log.warn("Removing stored file {} because it no longer exists on the disk.", storedFile.getId());
storedFileRepository.delete(storedFile);
Path filePath = getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Couldn't get storage path for stored file.", e);
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
pageable = pageable.next();
page = storedFileRepository.findAll(pageable);
public FileMetadataResponse getFileMetadata(String id) {
Path filePath = getStoragePathForFile(id);
if (!Files.exists(filePath)) {
throw new ResponseStatusException(HttpStatus.NOT_FOUND);
}
try (var in = Files.newInputStream(filePath)) {
FileMetadata metadata = readMetadata(in);
return new FileMetadataResponse(
metadata.filename,
metadata.mimeType,
Files.size(filePath),
Files.getLastModifiedTime(filePath)
.toInstant().atOffset(ZoneOffset.UTC)
.toLocalDateTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
true
);
} catch (IOException e) {
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "IO error", e);
}
}
}

53
gymboard-cdn/src/main/java/nl/andrewlalis/gymboardcdn/service/UploadService.java
View File

@ -1,11 +1,8 @@
package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import nl.andrewlalis.gymboardcdn.api.FileMetadataResponse;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.api.VideoProcessingTaskStatusResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFile;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
@ -17,9 +14,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.server.ResponseStatusException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.format.DateTimeFormatter;
@Service
public class UploadService {
@ -27,14 +22,10 @@ public class UploadService {
private static final long MAX_UPLOAD_SIZE_BYTES = (1024 * 1024 * 1024); // 1 Gb
private final StoredFileRepository storedFileRepository;
private final VideoProcessingTaskRepository videoTaskRepository;
private final FileService fileService;
public UploadService(StoredFileRepository storedFileRepository,
VideoProcessingTaskRepository videoTaskRepository,
FileService fileService) {
this.storedFileRepository = storedFileRepository;
public UploadService(VideoProcessingTaskRepository videoTaskRepository, FileService fileService) {
this.videoTaskRepository = videoTaskRepository;
this.fileService = fileService;
}
@ -86,46 +77,4 @@ public class UploadService {
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
return new VideoProcessingTaskStatusResponse(task.getStatus().name());
}
/**
* Streams the contents of a stored file to a client via the Http response.
* @param id The file's unique identifier.
* @param response The response to stream the content to.
*/
@Transactional(readOnly = true)
public void streamFile(String id, HttpServletResponse response) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
response.setContentType(file.getMimeType());
response.setContentLengthLong(file.getSize());
try {
Path filePath = fileService.getStoragePathForFile(file);
try (var in = Files.newInputStream(filePath)) {
in.transferTo(response.getOutputStream());
}
} catch (IOException e) {
log.error("Failed to write file to response.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
@Transactional(readOnly = true)
public FileMetadataResponse getFileMetadata(String id) {
StoredFile file = storedFileRepository.findById(id)
.orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND));
try {
Path filePath = fileService.getStoragePathForFile(file);
boolean exists = Files.exists(filePath);
return new FileMetadataResponse(
file.getName(),
file.getMimeType(),
file.getSize(),
file.getUploadedAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
exists
);
} catch (IOException e) {
log.error("Couldn't get path to stored file.", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}

3
gymboard-cdn/src/test/java/nl/andrewlalis/gymboardcdn/service/UploadServiceTest.java
View File

@ -3,7 +3,6 @@ package nl.andrewlalis.gymboardcdn.service;
import jakarta.servlet.ServletInputStream;
import jakarta.servlet.http.HttpServletRequest;
import nl.andrewlalis.gymboardcdn.api.FileUploadResponse;
import nl.andrewlalis.gymboardcdn.model.StoredFileRepository;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTask;
import nl.andrewlalis.gymboardcdn.model.VideoProcessingTaskRepository;
import org.junit.jupiter.api.Test;
@ -27,7 +26,6 @@ public class UploadServiceTest {
*/
@Test
public void processableVideoUploadSuccess() throws IOException {
StoredFileRepository storedFileRepository = Mockito.mock(StoredFileRepository.class);
VideoProcessingTaskRepository videoTaskRepository = Mockito.mock(VideoProcessingTaskRepository.class);
when(videoTaskRepository.save(any(VideoProcessingTask.class)))
.then(returnsFirstArg());
@ -38,7 +36,6 @@ public class UploadServiceTest {
when(fileService.createNewFileIdentifier()).thenReturn("abc");
UploadService uploadService = new UploadService(
storedFileRepository,
videoTaskRepository,
fileService
);