Added start of testing framework.

README.md

@@ -22,4 +22,15 @@ This tool comes with a variety of options, which you can view by running `gopro-
 - `--force` - Forcibly overwrite existing files.
 - `--dryRun` - Perform a dry-run (and don't actually copy anything).
 - `--bufferSize` - The size of the memory buffer for copying.
+- `--clean` - Delete copied items from the GoPro's media card afterwards.
 - `--help` - Shows help information.
+
+### System Compatibility
+Currently, this software is known to work with the following GoPro models:
+- Hero 9
+
+And the following operating systems:
+- Linux/Ubuntu
+- MacOS
+
+If you've got the time and hardware, I'd greatly appreciate if you could test this software on your own device, and make a PR to add it to the list here, or an issue if something doesn't work.

dub.json

@@ -4,7 +4,6 @@
 	],
 	"copyright": "Copyright © 2022, Andrew Lalis",
 	"dependencies": {
-		"dsh": "~>1.6.1",
 		"filesizes": "~>1.1.0",
 		"progress": "~>5.0.2"
 	},

dub.selections.json

@@ -1,7 +1,6 @@
 {
 	"fileVersion": 1,
 	"versions": {
-		"dsh": "1.6.1",
 		"filesizes": "1.1.0",
 		"progress": "5.0.2"
 	}

source/app.d

@@ -1,160 +1,84 @@
-import dsh;
-import std.typecons;
-import std.path;
-import std.string;
-import std.algorithm;
-import std.getopt;
-import filesizes;
-import progress;
-
-const DEFAULT_OUTPUT_DIR = "raw";
-const DEFAULT_MEDIA_DIR = "/media";
-const GOPRO_CONTENT_DIR = "DCIM/100GOPRO";
-const DEFAULT_BUFFER_SIZE = 1024 * 1024;
+import utils;
+import ingest;
 
 int main(string[] args) {
-	writeln(
-		"+---------------------------------+\n" ~
-		"|                                 |\n" ~
-		"|       GoPro Ingester            |\n" ~
-		"|         v1.0.0                  |\n" ~
-		"|         by Andrew Lalis         |\n" ~
-		"|                                 |\n" ~
-		"+---------------------------------+\n"
-	);
-
-    string mediaSearchDir = DEFAULT_MEDIA_DIR;
-    string outputDir = buildPath(getcwd(), DEFAULT_OUTPUT_DIR);
-	size_t bufferSize = DEFAULT_BUFFER_SIZE;
-    bool force = false;
-	bool dryRun = false;
-
-    auto helpInfo = getopt(
-        args,
-        "mediaDir|i",
-		format!"The base directory from which to search for the GoPro media. Defaults to \"%s\"."(mediaSearchDir),
-		&mediaSearchDir,
-        "outputDir|o",
-		format!"The directory to copy data to. Defaults to \"%s\". Will create the directory if it doesn't exist yet."(outputDir),
-		&outputDir,
-        "force|f",
-		format!"Whether to forcibly overwrite existing files. Defaults to %s."(force),
-		&force,
-		"dryRun|d",
-		format!"Whether to perform a dry-run (don't actually copy anything). Defaults to %s."(dryRun),
-		&dryRun,
-		"bufferSize|b",
-		format!"The size of the buffer for copying files, in bytes. Defaults to %s."(formatFilesize(DEFAULT_BUFFER_SIZE)),
-		&bufferSize
-    );
-
-    if (helpInfo.helpWanted) {
-        defaultGetoptPrinter("Ingestion tool for importing data from GoPro media cards.", helpInfo.options);
+	printBanner();
+    CliResult result = parseArgs(args);
+    if (result.type == CliResultType.NO_CONTENT) {
         return 0;
-    }
-
-    auto nullableGoProDir = getGoProDir(mediaSearchDir);
-    if (nullableGoProDir.isNull) {
-        error("Couldn't find GoPro directory.");
+    } else if (result.type == CliResultType.MISSING_MEDIA) {
         return 1;
-    }
-    string goProDir = nullableGoProDir.get();
-    writefln!"Found GoPro media at %s."(goProDir);
-    return copyFiles(goProDir, outputDir, bufferSize, force, dryRun);
-}
-
-/** 
- * Tries to find a GoPro's media directory.
- * Params:
- *   baseDir = The base directory to start the search from.
- * Returns: A nullable string that, if present, refers to the GoPro's media
- * directory.
- */
-Nullable!string getGoProDir(string baseDir) {
-    if (!exists(baseDir) || !isDir(baseDir)) return Nullable!string.init;
-    foreach (dir; std.file.dirEntries(baseDir, SpanMode.breadth)) {
-        string mediaPath = buildPath(dir.name, GOPRO_CONTENT_DIR);
-        if (exists(mediaPath) && isDir(mediaPath)) {
-            return nullable(mediaPath);
-        }
-    }
-    return Nullable!string.init;
-}
-
-/** 
- * Determines if we should copy a given file from the media card to the local
- * directory.
- * Params:
- *   entry = The entry for the file at its source.
- *   targetFile = The place to copy the file to.
- *   force = Whether the force flag has been set.
- * Returns: True if we should copy the file, or false otherwise.
- */
-bool shouldCopyFile(DirEntry entry, string targetFile, bool force) {
-    return entry.isFile() &&
-        (entry.name.endsWith(".MP4") || entry.name.endsWith(".WAV")) &&
-        (!exists(targetFile) || getSize(targetFile) != entry.size || force);
-}
-
-/** 
- * Copies files from a source to a target directory.
- * Params:
- *   sourceDir = The source directory.
- *   targetDir = The target directory.
- *   bufferSize = The buffer size to use when copying.
- *   force = Whether to overwrite existing files.
- *   dryRun = Whether to perform a dry-run.
- * Returns: An exit code.
- */
-int copyFiles(string sourceDir, string targetDir, size_t bufferSize, bool force, bool dryRun) {
-    if (!exists(targetDir) && !dryRun) mkdirRecurse(targetDir);
-    DirEntry[] filesToCopy;
-    ulong totalFileSize = 0;
-    foreach (DirEntry entry; dirEntries(sourceDir, SpanMode.shallow)) {
-        string targetFile = buildPath(targetDir, baseName(entry.name));
-        if (shouldCopyFile(entry, targetFile, force)) {
-            filesToCopy ~= entry;
-            totalFileSize += entry.size;
-        }
-    }
-
-    if (filesToCopy.length == 0) {
-        writeln("No new files to copy.");
-        return 0;
-    }
-
-	if (getAvailableDiskSpace(targetDir) < totalFileSize) {
-		writefln!"Not enough disk space to copy all files: %s available, %s needed."(
-			formatFilesize(getAvailableDiskSpace(targetDir)),
-			formatFilesize(totalFileSize)
-		);
-		return 1;
-	}
-
-    writefln!"Copying %d files (%s) to %s."(filesToCopy.length, formatFilesize(totalFileSize), targetDir);
-	if (dryRun) writeln("(Dry Run)");
-    Bar progressBar = new FillingSquaresBar();
-    progressBar.width = 80;
-    progressBar.max = totalFileSize;
-    progressBar.start();
-	ubyte[] buffer = new ubyte[bufferSize];
-    foreach (DirEntry entry; filesToCopy) {
-        string filename = baseName(entry.name);
-        string targetFile = buildPath(targetDir, filename);
-		string verb = exists(targetFile) ? "Overwriting" : "Copying";
-        progressBar.message = { return std.string.format!"%s %s (%s)"(verb, filename, formatFilesize(entry.size)); };
-		if (!dryRun) {
-			File inputFile = File(entry.name, "rb");
-			File outputFile = File(targetFile, "wb");
-			foreach (ubyte[] localBuffer; inputFile.byChunk(buffer)) {
-				outputFile.rawWrite(localBuffer);
-				progressBar.next(localBuffer.length);
-			}
     } else {
-			progressBar.next(getSize(entry.name));
+        return copyFiles(result.config);
     }
 }
-    progressBar.finish();
 
-	return 0;
+unittest {
+    // First some utilities to make the tests simpler.
+    import std.stdio;
+    import std.file;
+    import std.path;
+    import std.algorithm;
+    import std.string;
+    import utils;
+
+    struct DirView {
+        DirEntry[] entries;
+    }
+
+    DirView getDirView(string dir) {
+        DirView view;
+        foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow)) {
+            view.entries ~= entry;
+        }
+        view.entries.sort!((a, b) => a.name > b.name);
+        return view;
+    }
+
+    int callApp(string[] args ...) {
+        return main(["app"] ~ args);
+    }
+
+    string getBaseCardDir(string name) {
+        return buildPath("test", "media-cards", "card-" ~ name);
+    }
+
+    string getTestCardDir(string name) {
+        return buildPath("test", "media-cards", "card-test-" ~ name);
+    }
+
+    void assertCardsUnchanged(string[] cards ...) {
+        foreach (string card; cards) {
+            string baseDir = getBaseCardDir(card);
+            string testDir = getTestCardDir(card);
+            if (exists(testDir)) {
+                assert(getDirView(baseDir) == getDirView(testDir));
+            }
+        }
+    }
+
+    void prepareCardTests(string[] cards ...) {
+        foreach (string card; cards) {
+            copyDir(getBaseCardDir(card), getTestCardDir(card));
+        }
+    }
+
+    void cleanupCardTests(string[] cards ...) {
+        foreach (string card; cards) {
+            rmdirRecurse(getTestCardDir(card));
+        }
+    }
+
+    string[] allCards = ["1", "2", "3"];
+    prepareCardTests(allCards);
+    writeln(isDir(getBaseCardDir("1")));
+    assert(callApp("-h") == 0);
+    assertCardsUnchanged(allCards);
+    assert(callApp("--help") == 0);
+    assertCardsUnchanged(allCards);
+    assert(callApp("-f", "-h") == 0);
+    assertCardsUnchanged(allCards);
+    // Ingesting from an empty card shouldn't have any effect.
+    assert(callApp("-i", getTestCardDir("1")) == 0);
+    assertCardsUnchanged("1");
 }

source/ingest.d

@@ -0,0 +1,154 @@
+module ingest;
+
+import std.file;
+import std.stdio;
+import std.path;
+import std.algorithm;
+import std.string;
+import std.typecons;
+import filesizes;
+import progress;
+import utils;
+
+const DEFAULT_BUFFER_SIZE = 1024 * 1024;
+
+private struct IngestData {
+    DirEntry[] filesToCopy;
+
+    ulong totalFileSize() {
+        return filesToCopy.map!(f => f.size).sum;
+    }
+
+    size_t fileCount() {
+        return filesToCopy.length;
+    }
+}
+
+public struct IngestConfig {
+    string inputDir;
+    string outputDir;
+    size_t bufferSize = DEFAULT_BUFFER_SIZE;
+    bool force = false;
+    bool dryRun = false;
+    bool clean = false;
+}
+
+/** 
+ * Determines if we should copy a given file from the media card to the local
+ * directory.
+ * Params:
+ *   entry = The entry for the file at its source.
+ *   targetFile = The place to copy the file to.
+ *   force = Whether the force flag has been set.
+ * Returns: True if we should copy the file, or false otherwise.
+ */
+private bool shouldCopyFile(DirEntry entry, string targetFile, bool force) {
+    return entry.isFile() &&
+        (endsWithAny(entry.name, ".MP4", ".mp4", ".WAV", ".wav")) &&
+        (!exists(targetFile) || getSize(targetFile) != entry.size || force);
+}
+
+/** 
+ * Searches for relevant files to ingest.
+ * Params:
+ *   config = The ingest config.
+ * Returns: Data about what to ingest.
+ */
+private IngestData discoverIngestData(IngestConfig config) {
+    IngestData data;
+    foreach (DirEntry entry; dirEntries(config.inputDir, SpanMode.shallow)) {
+        string targetFile = buildPath(config.outputDir, baseName(entry.name));
+        if (shouldCopyFile(entry, targetFile, config.force)) {
+            data.filesToCopy ~= entry;
+        }
+    }
+    return data;
+}
+
+/** 
+ * Copies files from a source to a target directory.
+ * Params:
+ *   config = The configuration for the ingest operation.
+ * Returns: An exit code.
+ */
+public int copyFiles(IngestConfig config) {
+    IngestData ingestData = discoverIngestData(config);
+    if (ingestData.fileCount == 0) {
+        writeln("No new files to copy.");
+        return 0;
+    }
+	if (getAvailableDiskSpace(config.outputDir) < ingestData.totalFileSize) {
+		writefln!"Not enough disk space to copy all files: %s available, %s needed."(
+			formatFilesize(getAvailableDiskSpace(config.outputDir)),
+			formatFilesize(ingestData.totalFileSize)
+		);
+		return 1;
+	}
+    writefln!"Copying %d files (%s) to %s."(ingestData.fileCount, formatFilesize(ingestData.totalFileSize), config.outputDir);
+	if (config.dryRun) writeln("(Dry Run)");
+
+    if (!exists(config.outputDir) && !config.dryRun) mkdirRecurse(config.outputDir);
+    
+	ubyte[] buffer = new ubyte[config.bufferSize];
+    foreach (DirEntry entry; ingestData.filesToCopy) {
+        string filename = baseName(entry.name);
+        string targetFile = buildPath(config.outputDir, filename);
+		string verb = exists(targetFile) ? "Overwriting" : "Copying";
+        Bar progressBar = new FillingSquaresBar();
+        progressBar.width = 40;
+        progressBar.max = entry.size;
+        string message = format!"%s %s (%s)"(verb, filename, formatFilesize(entry.size))
+            .leftJustify(40, ' ');
+        progressBar.message = { return message; };
+        progressBar.start();
+		if (!config.dryRun) {
+			File inputFile = File(entry.name, "rb");
+			File outputFile = File(targetFile, "wb");
+			foreach (ubyte[] localBuffer; inputFile.byChunk(buffer)) {
+				outputFile.rawWrite(localBuffer);
+				progressBar.next(localBuffer.length);
+			}
+		} else {
+			progressBar.next(getSize(entry.name));
+		}
+        progressBar.finish();
+    }
+
+    if (config.clean && !config.dryRun) {
+        writeln("Cleaning GoPro media card.");
+        string[] filesToRemove;
+        foreach (string filename; dirEntries(config.inputDir, SpanMode.shallow)) {
+            filesToRemove ~= filename;
+        }
+        Bar progressBar = new FillingSquaresBar();
+        progressBar.max = filesToRemove.length;
+        progressBar.width = 80;
+        progressBar.start();
+        foreach (string filename; filesToRemove) {
+            std.file.remove(filename);
+            progressBar.next();
+        }
+        progressBar.finish();
+        string trashDir = buildNormalizedPath(config.inputDir, "..", "..", ".Trash-1000");
+        if (exists(trashDir) && isDir(trashDir)) {
+            writefln!"Removing \"%s\"."(trashDir);
+            rmdirRecurse(trashDir);
+        }
+    }
+
+	return 0;
+}
+
+unittest {
+    import testutils;
+
+    // Test a dry run.
+    prepareCardTests("1");
+    IngestConfig c1;
+    c1.dryRun = true;
+    c1.inputDir = getTestCardDir("1");
+    c1.outputDir = "test-out";
+    assert(copyFiles(c1) == 0);
+    assertCardsUnchanged("1");
+    cleanupCardTests("1");
+}

source/testutils.d

@@ -0,0 +1,51 @@
+module testutils;
+
+import std.stdio;
+import std.file;
+import std.path;
+import std.algorithm;
+import std.string;
+import utils;
+
+struct DirView {
+    DirEntry[] entries;
+}
+
+DirView getDirView(string dir) {
+    DirView view;
+    foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow)) {
+        view.entries ~= entry;
+    }
+    view.entries.sort!((a, b) => a.name > b.name);
+    return view;
+}
+
+string getBaseCardDir(string name) {
+    return buildPath("test", "media-cards", "card-" ~ name);
+}
+
+string getTestCardDir(string name) {
+    return buildPath("test", "media-cards", "card-test-" ~ name);
+}
+
+void assertCardsUnchanged(string[] cards ...) {
+    foreach (string card; cards) {
+        string baseDir = getBaseCardDir(card);
+        string testDir = getTestCardDir(card);
+        if (exists(testDir)) {
+            assert(getDirView(baseDir) == getDirView(testDir));
+        }
+    }
+}
+
+void prepareCardTests(string[] cards ...) {
+    foreach (string card; cards) {
+        copyDir(getBaseCardDir(card), getTestCardDir(card));
+    }
+}
+
+void cleanupCardTests(string[] cards ...) {
+    foreach (string card; cards) {
+        rmdirRecurse(getTestCardDir(card));
+    }
+}

source/utils.d

@@ -0,0 +1,147 @@
+module utils;
+
+import std.typecons;
+import ingest;
+
+const DEFAULT_MEDIA_DIR = "/media";
+const DEFAULT_OUTPUT_DIR = "raw";
+
+public enum CliResultType {
+    OK,
+    NO_CONTENT,
+    MISSING_MEDIA
+}
+
+public struct CliResult {
+    CliResultType type;
+    IngestConfig config;
+}
+
+public CliResult parseArgs(string[] args) {
+    import std.path;
+    import std.file;
+    import std.getopt;
+    import std.string;
+    import std.stdio;
+    import filesizes;
+    CliResult result;
+    IngestConfig config;
+    config.outputDir = buildPath(getcwd(), DEFAULT_OUTPUT_DIR);
+    string mediaSearchDir = DEFAULT_MEDIA_DIR;
+    auto helpInfo = getopt(
+        args,
+        "mediaDir|i",
+		format!"The base directory from which to search for the GoPro media. Defaults to \"%s\"."(mediaSearchDir),
+		&mediaSearchDir,
+        "outputDir|o",
+		format!"The directory to copy data to. Defaults to \"%s\". Will create the directory if it doesn't exist yet."(config.outputDir),
+		&config.outputDir,
+        "force|f",
+		format!"Whether to forcibly overwrite existing files. Defaults to %s."(config.force),
+		&config.force,
+		"dryRun|d",
+		format!"Whether to perform a dry-run (don't actually copy anything). Defaults to %s."(config.dryRun),
+		&config.dryRun,
+		"bufferSize|b",
+		format!"The size of the buffer for copying files, in bytes. Defaults to %s."(formatFilesize(config.bufferSize)),
+		&config.bufferSize,
+        "clean|c",
+        format!"Whether to remove files from the GoPro media card after copying. Defaults to %s."(config.clean),
+        &config.clean
+    );
+
+    if (helpInfo.helpWanted) {
+        defaultGetoptPrinter("Ingestion tool for importing data from GoPro media cards.", helpInfo.options);
+        result.type = CliResultType.NO_CONTENT;
+        return result;
+    } else {
+        auto nullableGoProDir = getGoProDir(mediaSearchDir);
+        if (nullableGoProDir.isNull) {
+            writeln("Couldn't find GoPro directory.");
+            result.type = CliResultType.MISSING_MEDIA;
+        } else {
+            config.inputDir = nullableGoProDir.get();
+            writefln!"Found GoPro media at %s."(config.inputDir);
+            result.type = CliResultType.OK;
+            result.config = config;
+        }
+    }
+    return result;
+}
+
+unittest {
+    assert(parseArgs(["app", "-h"]).type == CliResultType.NO_CONTENT);
+    // TODO: Expand tests.
+}
+
+public void printBanner() {
+    import std.stdio : writeln;
+    writeln(
+		"+---------------------------------+\n" ~
+		"|                                 |\n" ~
+		"|       GoPro Ingester            |\n" ~
+		"|         v1.0.0                  |\n" ~
+		"|         by Andrew Lalis         |\n" ~
+		"|                                 |\n" ~
+		"+---------------------------------+\n"
+	);
+}
+
+public bool endsWithAny(string s, string[] suffixes ...) {
+    import std.algorithm : endsWith;
+    foreach (string suffix; suffixes) {
+        if (s.endsWith(suffix)) return true;
+    }
+    return false;
+}
+
+unittest {
+    assert(endsWithAny("abc", "a", "b", "c"));
+    assert(!endsWithAny("abc", "d"));
+    assert(!endsWithAny("abc", "A", "B", "C"));
+}
+
+/** 
+ * Copies all files from the given source directory, to the given destination
+ * directory. Will create the destination directory if it doesn't exist yet.
+ * Overwrites any files that already exist in the destination directory.
+ * Params:
+ *   sourceDir = The source directory to copy from.
+ *   destDir = The destination directory to copy to.
+ */
+public void copyDir(string sourceDir, string destDir) {
+    import std.file;
+    if (!isDir(sourceDir)) return;
+    if (exists(destDir) && !isDir(destDir)) return;
+    if (!exists(destDir)) mkdirRecurse(destDir);
+    import std.path : buildPath, baseName;
+    foreach (DirEntry entry; dirEntries(sourceDir, SpanMode.shallow)) {
+        string destPath = buildPath(destDir, baseName(entry.name));
+        if (entry.isDir) {
+            copyDir(entry.name, destPath);
+        } else if (entry.isFile) {
+            copy(entry.name, destPath);
+        }
+    }
+}
+
+/** 
+ * Tries to find a GoPro's media directory.
+ * Params:
+ *   baseDir = The base directory to start the search from.
+ * Returns: A nullable string that, if present, refers to the GoPro's media
+ * directory.
+ */
+private Nullable!string getGoProDir(string baseDir) {
+    import std.file;
+    import std.path;
+    if (!exists(baseDir) || !isDir(baseDir)) return Nullable!string.init;
+    foreach (dir; std.file.dirEntries(baseDir, SpanMode.breadth)) {
+        // We know that a GoPro contains DCIM/100GOPRO in it.
+        string mediaPath = buildPath(dir.name, "DCIM", "100GOPRO");
+        if (exists(mediaPath) && isDir(mediaPath)) {
+            return nullable(mediaPath);
+        }
+    }
+    return Nullable!string.init;
+}

test/media-cards/card-1/DCIM/100GOPRO/tmp.txt

@@ -0,0 +1 @@
+Hello

test/media-cards/card-1/README.md

@@ -0,0 +1,2 @@
+# Card 1
+This is an empty card.

test/media-cards/card-2/DCIM/100GOPRO/tmp.txt

@@ -0,0 +1 @@
+Hello

test/media-cards/card-2/README.md

@@ -0,0 +1 @@
+# Card 2

test/media-cards/card-3/DCIM/100GOPRO/tmp.txt

@@ -0,0 +1 @@
+Hello

test/media-cards/card-3/README.md

@@ -0,0 +1 @@
+# Card 3