gopro-ingester/source/ingest.d

154 lines
4.6 KiB
D
Raw Normal View History

2022-11-10 15:58:30 +00:00
module ingest;
import std.file;
import std.stdio;
import std.path;
import std.algorithm;
import std.string;
import std.typecons;
import filesizes;
import progress;
2022-11-11 10:40:35 +00:00
import utils;
2022-11-10 15:58:30 +00:00
2022-11-10 16:26:24 +00:00
const DEFAULT_BUFFER_SIZE = 1024 * 1024;
2022-11-10 15:58:30 +00:00
private struct IngestData {
DirEntry[] filesToCopy;
ulong totalFileSize() {
return filesToCopy.map!(f => f.size).sum;
}
size_t fileCount() {
return filesToCopy.length;
}
}
2022-11-10 16:26:24 +00:00
public struct IngestConfig {
string inputDir;
string outputDir;
size_t bufferSize = DEFAULT_BUFFER_SIZE;
bool force = false;
bool dryRun = false;
bool clean = false;
}
2022-11-10 15:58:30 +00:00
/**
* Determines if we should copy a given file from the media card to the local
* directory.
* Params:
* entry = The entry for the file at its source.
* targetFile = The place to copy the file to.
* force = Whether the force flag has been set.
* Returns: True if we should copy the file, or false otherwise.
*/
private bool shouldCopyFile(DirEntry entry, string targetFile, bool force) {
return entry.isFile() &&
2022-11-11 10:40:35 +00:00
(endsWithAny(entry.name, ".MP4", ".mp4", ".WAV", ".wav")) &&
2022-11-10 15:58:30 +00:00
(!exists(targetFile) || getSize(targetFile) != entry.size || force);
}
2022-11-10 16:26:24 +00:00
/**
* Searches for relevant files to ingest.
* Params:
* config = The ingest config.
* Returns: Data about what to ingest.
*/
private IngestData discoverIngestData(IngestConfig config) {
2022-11-10 15:58:30 +00:00
IngestData data;
2022-11-10 16:26:24 +00:00
foreach (DirEntry entry; dirEntries(config.inputDir, SpanMode.shallow)) {
string targetFile = buildPath(config.outputDir, baseName(entry.name));
if (shouldCopyFile(entry, targetFile, config.force)) {
2022-11-10 15:58:30 +00:00
data.filesToCopy ~= entry;
}
}
return data;
}
/**
* Copies files from a source to a target directory.
* Params:
2022-11-10 16:26:24 +00:00
* config = The configuration for the ingest operation.
2022-11-10 15:58:30 +00:00
* Returns: An exit code.
*/
2022-11-10 16:26:24 +00:00
public int copyFiles(IngestConfig config) {
IngestData ingestData = discoverIngestData(config);
2022-11-10 15:58:30 +00:00
if (ingestData.fileCount == 0) {
writeln("No new files to copy.");
return 0;
}
2022-11-10 16:26:24 +00:00
if (getAvailableDiskSpace(config.outputDir) < ingestData.totalFileSize) {
2022-11-10 15:58:30 +00:00
writefln!"Not enough disk space to copy all files: %s available, %s needed."(
2022-11-10 16:26:24 +00:00
formatFilesize(getAvailableDiskSpace(config.outputDir)),
2022-11-10 15:58:30 +00:00
formatFilesize(ingestData.totalFileSize)
);
return 1;
}
2022-11-10 16:26:24 +00:00
writefln!"Copying %d files (%s) to %s."(ingestData.fileCount, formatFilesize(ingestData.totalFileSize), config.outputDir);
if (config.dryRun) writeln("(Dry Run)");
2022-11-10 15:58:30 +00:00
2022-11-10 16:26:24 +00:00
if (!exists(config.outputDir) && !config.dryRun) mkdirRecurse(config.outputDir);
2022-11-11 08:14:46 +00:00
2022-11-10 16:26:24 +00:00
ubyte[] buffer = new ubyte[config.bufferSize];
2022-11-10 15:58:30 +00:00
foreach (DirEntry entry; ingestData.filesToCopy) {
string filename = baseName(entry.name);
2022-11-10 16:26:24 +00:00
string targetFile = buildPath(config.outputDir, filename);
2022-11-10 15:58:30 +00:00
string verb = exists(targetFile) ? "Overwriting" : "Copying";
2022-11-11 08:14:46 +00:00
Bar progressBar = new FillingSquaresBar();
progressBar.width = 40;
progressBar.max = entry.size;
string message = format!"%s %s (%s)"(verb, filename, formatFilesize(entry.size))
.leftJustify(40, ' ');
progressBar.message = { return message; };
progressBar.start();
2022-11-10 16:26:24 +00:00
if (!config.dryRun) {
2022-11-10 15:58:30 +00:00
File inputFile = File(entry.name, "rb");
File outputFile = File(targetFile, "wb");
foreach (ubyte[] localBuffer; inputFile.byChunk(buffer)) {
outputFile.rawWrite(localBuffer);
progressBar.next(localBuffer.length);
}
} else {
progressBar.next(getSize(entry.name));
}
2022-11-11 08:14:46 +00:00
progressBar.finish();
}
if (config.clean && !config.dryRun) {
writeln("Cleaning GoPro media card.");
string[] filesToRemove;
foreach (string filename; dirEntries(config.inputDir, SpanMode.shallow)) {
filesToRemove ~= filename;
}
Bar progressBar = new FillingSquaresBar();
progressBar.max = filesToRemove.length;
progressBar.width = 80;
progressBar.start();
foreach (string filename; filesToRemove) {
std.file.remove(filename);
progressBar.next();
}
progressBar.finish();
string trashDir = buildNormalizedPath(config.inputDir, "..", "..", ".Trash-1000");
if (exists(trashDir) && isDir(trashDir)) {
writefln!"Removing \"%s\"."(trashDir);
rmdirRecurse(trashDir);
}
2022-11-10 15:58:30 +00:00
}
return 0;
2022-11-11 10:40:35 +00:00
}
unittest {
import testutils;
// Test a dry run.
prepareCardTests("1");
IngestConfig c1;
c1.dryRun = true;
c1.inputDir = getTestCardDir("1");
c1.outputDir = "test-out";
assert(copyFiles(c1) == 0);
assertCardsUnchanged("1");
cleanupCardTests("1");
2022-11-10 15:58:30 +00:00
}