Compare commits

..

No commits in common. "main" and "v1.0.0" have entirely different histories.
main ... v1.0.0

14 changed files with 152 additions and 444 deletions

View File

@ -22,15 +22,4 @@ This tool comes with a variety of options, which you can view by running `gopro-
- `--force` - Forcibly overwrite existing files.
- `--dryRun` - Perform a dry-run (and don't actually copy anything).
- `--bufferSize` - The size of the memory buffer for copying.
- `--clean` - Delete copied items from the GoPro's media card afterwards.
- `--help` - Shows help information.
### System Compatibility
Currently, this software is known to work with the following GoPro models:
- Hero 9
And the following operating systems:
- Linux/Ubuntu
- MacOS
If you've got the time and hardware, I'd greatly appreciate if you could test this software on your own device, and make a PR to add it to the list here, or an issue if something doesn't work.

View File

@ -4,6 +4,7 @@
],
"copyright": "Copyright © 2022, Andrew Lalis",
"dependencies": {
"dsh": "~>1.6.1",
"filesizes": "~>1.1.0",
"progress": "~>5.0.2"
},

View File

@ -1,6 +1,7 @@
{
"fileVersion": 1,
"versions": {
"dsh": "1.6.1",
"filesizes": "1.1.0",
"progress": "5.0.2"
}

BIN
icon.png

Binary file not shown.

Before

Width:  |  Height:  |  Size: 79 KiB

View File

@ -1,84 +1,160 @@
import utils;
import ingest;
import dsh;
import std.typecons;
import std.path;
import std.string;
import std.algorithm;
import std.getopt;
import filesizes;
import progress;
const DEFAULT_OUTPUT_DIR = "raw";
const DEFAULT_MEDIA_DIR = "/media";
const GOPRO_CONTENT_DIR = "DCIM/100GOPRO";
const DEFAULT_BUFFER_SIZE = 1024 * 1024;
int main(string[] args) {
printBanner();
CliResult result = parseArgs(args);
if (result.type == CliResultType.NO_CONTENT) {
writeln(
"+---------------------------------+\n" ~
"| |\n" ~
"| GoPro Ingester |\n" ~
"| v1.0.0 |\n" ~
"| by Andrew Lalis |\n" ~
"| |\n" ~
"+---------------------------------+\n"
);
string mediaSearchDir = DEFAULT_MEDIA_DIR;
string outputDir = buildPath(getcwd(), DEFAULT_OUTPUT_DIR);
size_t bufferSize = DEFAULT_BUFFER_SIZE;
bool force = false;
bool dryRun = false;
auto helpInfo = getopt(
args,
"mediaDir|i",
format!"The base directory from which to search for the GoPro media. Defaults to \"%s\"."(mediaSearchDir),
&mediaSearchDir,
"outputDir|o",
format!"The directory to copy data to. Defaults to \"%s\". Will create the directory if it doesn't exist yet."(outputDir),
&outputDir,
"force|f",
format!"Whether to forcibly overwrite existing files. Defaults to %s."(force),
&force,
"dryRun|d",
format!"Whether to perform a dry-run (don't actually copy anything). Defaults to %s."(dryRun),
&dryRun,
"bufferSize|b",
format!"The size of the buffer for copying files, in bytes. Defaults to %s."(formatFilesize(DEFAULT_BUFFER_SIZE)),
&bufferSize
);
if (helpInfo.helpWanted) {
defaultGetoptPrinter("Ingestion tool for importing data from GoPro media cards.", helpInfo.options);
return 0;
} else if (result.type == CliResultType.MISSING_MEDIA) {
}
auto nullableGoProDir = getGoProDir(mediaSearchDir);
if (nullableGoProDir.isNull) {
error("Couldn't find GoPro directory.");
return 1;
} else {
return copyFiles(result.config);
}
string goProDir = nullableGoProDir.get();
writefln!"Found GoPro media at %s."(goProDir);
return copyFiles(goProDir, outputDir, bufferSize, force, dryRun);
}
unittest {
// First some utilities to make the tests simpler.
import std.stdio;
import std.file;
import std.path;
import std.algorithm;
import std.string;
import utils;
struct DirView {
DirEntry[] entries;
}
DirView getDirView(string dir) {
DirView view;
foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow)) {
view.entries ~= entry;
}
view.entries.sort!((a, b) => a.name > b.name);
return view;
}
int callApp(string[] args ...) {
return main(["app"] ~ args);
}
string getBaseCardDir(string name) {
return buildPath("test", "media-cards", "card-" ~ name);
}
string getTestCardDir(string name) {
return buildPath("test", "media-cards", "card-test-" ~ name);
}
void assertCardsUnchanged(string[] cards ...) {
foreach (string card; cards) {
string baseDir = getBaseCardDir(card);
string testDir = getTestCardDir(card);
if (exists(testDir)) {
assert(getDirView(baseDir) == getDirView(testDir));
}
/**
* Tries to find a GoPro's media directory.
* Params:
* baseDir = The base directory to start the search from.
* Returns: A nullable string that, if present, refers to the GoPro's media
* directory.
*/
Nullable!string getGoProDir(string baseDir) {
if (!exists(baseDir) || !isDir(baseDir)) return Nullable!string.init;
foreach (dir; std.file.dirEntries(baseDir, SpanMode.breadth)) {
string mediaPath = buildPath(dir.name, GOPRO_CONTENT_DIR);
if (exists(mediaPath) && isDir(mediaPath)) {
return nullable(mediaPath);
}
}
void prepareCardTests(string[] cards ...) {
foreach (string card; cards) {
copyDir(getBaseCardDir(card), getTestCardDir(card));
}
}
void cleanupCardTests(string[] cards ...) {
foreach (string card; cards) {
rmdirRecurse(getTestCardDir(card));
}
}
string[] allCards = ["1", "2", "3"];
prepareCardTests(allCards);
writeln(isDir(getBaseCardDir("1")));
assert(callApp("-h") == 0);
assertCardsUnchanged(allCards);
assert(callApp("--help") == 0);
assertCardsUnchanged(allCards);
assert(callApp("-f", "-h") == 0);
assertCardsUnchanged(allCards);
// Ingesting from an empty card shouldn't have any effect.
assert(callApp("-i", getTestCardDir("1")) == 0);
assertCardsUnchanged("1");
return Nullable!string.init;
}
/**
* Determines if we should copy a given file from the media card to the local
* directory.
* Params:
* entry = The entry for the file at its source.
* targetFile = The place to copy the file to.
* force = Whether the force flag has been set.
* Returns: True if we should copy the file, or false otherwise.
*/
bool shouldCopyFile(DirEntry entry, string targetFile, bool force) {
return entry.isFile() &&
(entry.name.endsWith(".MP4") || entry.name.endsWith(".WAV")) &&
(!exists(targetFile) || getSize(targetFile) != entry.size || force);
}
/**
* Copies files from a source to a target directory.
* Params:
* sourceDir = The source directory.
* targetDir = The target directory.
* bufferSize = The buffer size to use when copying.
* force = Whether to overwrite existing files.
* dryRun = Whether to perform a dry-run.
* Returns: An exit code.
*/
int copyFiles(string sourceDir, string targetDir, size_t bufferSize, bool force, bool dryRun) {
if (!exists(targetDir) && !dryRun) mkdirRecurse(targetDir);
DirEntry[] filesToCopy;
ulong totalFileSize = 0;
foreach (DirEntry entry; dirEntries(sourceDir, SpanMode.shallow)) {
string targetFile = buildPath(targetDir, baseName(entry.name));
if (shouldCopyFile(entry, targetFile, force)) {
filesToCopy ~= entry;
totalFileSize += entry.size;
}
}
if (filesToCopy.length == 0) {
writeln("No new files to copy.");
return 0;
}
if (getAvailableDiskSpace(targetDir) < totalFileSize) {
writefln!"Not enough disk space to copy all files: %s available, %s needed."(
formatFilesize(getAvailableDiskSpace(targetDir)),
formatFilesize(totalFileSize)
);
return 1;
}
writefln!"Copying %d files (%s) to %s."(filesToCopy.length, formatFilesize(totalFileSize), targetDir);
if (dryRun) writeln("(Dry Run)");
Bar progressBar = new FillingSquaresBar();
progressBar.width = 80;
progressBar.max = totalFileSize;
progressBar.start();
ubyte[] buffer = new ubyte[bufferSize];
foreach (DirEntry entry; filesToCopy) {
string filename = baseName(entry.name);
string targetFile = buildPath(targetDir, filename);
string verb = exists(targetFile) ? "Overwriting" : "Copying";
progressBar.message = { return std.string.format!"%s %s (%s)"(verb, filename, formatFilesize(entry.size)); };
if (!dryRun) {
File inputFile = File(entry.name, "rb");
File outputFile = File(targetFile, "wb");
foreach (ubyte[] localBuffer; inputFile.byChunk(buffer)) {
outputFile.rawWrite(localBuffer);
progressBar.next(localBuffer.length);
}
} else {
progressBar.next(getSize(entry.name));
}
}
progressBar.finish();
return 0;
}

View File

@ -1,154 +0,0 @@
module ingest;
import std.file;
import std.stdio;
import std.path;
import std.algorithm;
import std.string;
import std.typecons;
import filesizes;
import progress;
import utils;
const DEFAULT_BUFFER_SIZE = 1024 * 1024;
private struct IngestData {
DirEntry[] filesToCopy;
ulong totalFileSize() {
return filesToCopy.map!(f => f.size).sum;
}
size_t fileCount() {
return filesToCopy.length;
}
}
public struct IngestConfig {
string inputDir;
string outputDir;
size_t bufferSize = DEFAULT_BUFFER_SIZE;
bool force = false;
bool dryRun = false;
bool clean = false;
}
/**
* Determines if we should copy a given file from the media card to the local
* directory.
* Params:
* entry = The entry for the file at its source.
* targetFile = The place to copy the file to.
* force = Whether the force flag has been set.
* Returns: True if we should copy the file, or false otherwise.
*/
private bool shouldCopyFile(DirEntry entry, string targetFile, bool force) {
return entry.isFile() &&
(endsWithAny(entry.name, ".MP4", ".mp4", ".WAV", ".wav")) &&
(!exists(targetFile) || getSize(targetFile) != entry.size || force);
}
/**
* Searches for relevant files to ingest.
* Params:
* config = The ingest config.
* Returns: Data about what to ingest.
*/
private IngestData discoverIngestData(IngestConfig config) {
IngestData data;
foreach (DirEntry entry; dirEntries(config.inputDir, SpanMode.shallow)) {
string targetFile = buildPath(config.outputDir, baseName(entry.name));
if (shouldCopyFile(entry, targetFile, config.force)) {
data.filesToCopy ~= entry;
}
}
return data;
}
/**
* Copies files from a source to a target directory.
* Params:
* config = The configuration for the ingest operation.
* Returns: An exit code.
*/
public int copyFiles(IngestConfig config) {
IngestData ingestData = discoverIngestData(config);
if (ingestData.fileCount == 0) {
writeln("No new files to copy.");
return 0;
}
if (getAvailableDiskSpace(config.outputDir) < ingestData.totalFileSize) {
writefln!"Not enough disk space to copy all files: %s available, %s needed."(
formatFilesize(getAvailableDiskSpace(config.outputDir)),
formatFilesize(ingestData.totalFileSize)
);
return 1;
}
writefln!"Copying %d files (%s) to %s."(ingestData.fileCount, formatFilesize(ingestData.totalFileSize), config.outputDir);
if (config.dryRun) writeln("(Dry Run)");
if (!exists(config.outputDir) && !config.dryRun) mkdirRecurse(config.outputDir);
ubyte[] buffer = new ubyte[config.bufferSize];
foreach (DirEntry entry; ingestData.filesToCopy) {
string filename = baseName(entry.name);
string targetFile = buildPath(config.outputDir, filename);
string verb = exists(targetFile) ? "Overwriting" : "Copying";
Bar progressBar = new FillingSquaresBar();
progressBar.width = 40;
progressBar.max = entry.size;
string message = format!"%s %s (%s)"(verb, filename, formatFilesize(entry.size))
.leftJustify(40, ' ');
progressBar.message = { return message; };
progressBar.start();
if (!config.dryRun) {
File inputFile = File(entry.name, "rb");
File outputFile = File(targetFile, "wb");
foreach (ubyte[] localBuffer; inputFile.byChunk(buffer)) {
outputFile.rawWrite(localBuffer);
progressBar.next(localBuffer.length);
}
} else {
progressBar.next(getSize(entry.name));
}
progressBar.finish();
}
if (config.clean && !config.dryRun) {
writeln("Cleaning GoPro media card.");
string[] filesToRemove;
foreach (string filename; dirEntries(config.inputDir, SpanMode.shallow)) {
filesToRemove ~= filename;
}
Bar progressBar = new FillingSquaresBar();
progressBar.max = filesToRemove.length;
progressBar.width = 80;
progressBar.start();
foreach (string filename; filesToRemove) {
std.file.remove(filename);
progressBar.next();
}
progressBar.finish();
string trashDir = buildNormalizedPath(config.inputDir, "..", "..", ".Trash-1000");
if (exists(trashDir) && isDir(trashDir)) {
writefln!"Removing \"%s\"."(trashDir);
rmdirRecurse(trashDir);
}
}
return 0;
}
unittest {
import testutils;
// Test a dry run.
prepareCardTests("1");
IngestConfig c1;
c1.dryRun = true;
c1.inputDir = getTestCardDir("1");
c1.outputDir = "test-out";
assert(copyFiles(c1) == 0);
assertCardsUnchanged("1");
cleanupCardTests("1");
}

View File

@ -1,51 +0,0 @@
module testutils;
import std.stdio;
import std.file;
import std.path;
import std.algorithm;
import std.string;
import utils;
struct DirView {
DirEntry[] entries;
}
DirView getDirView(string dir) {
DirView view;
foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow)) {
view.entries ~= entry;
}
view.entries.sort!((a, b) => a.name > b.name);
return view;
}
string getBaseCardDir(string name) {
return buildPath("test", "media-cards", "card-" ~ name);
}
string getTestCardDir(string name) {
return buildPath("test", "media-cards", "card-test-" ~ name);
}
void assertCardsUnchanged(string[] cards ...) {
foreach (string card; cards) {
string baseDir = getBaseCardDir(card);
string testDir = getTestCardDir(card);
if (exists(testDir)) {
assert(getDirView(baseDir) == getDirView(testDir));
}
}
}
void prepareCardTests(string[] cards ...) {
foreach (string card; cards) {
copyDir(getBaseCardDir(card), getTestCardDir(card));
}
}
void cleanupCardTests(string[] cards ...) {
foreach (string card; cards) {
rmdirRecurse(getTestCardDir(card));
}
}

View File

@ -1,147 +0,0 @@
module utils;
import std.typecons;
import ingest;
const DEFAULT_MEDIA_DIR = "/media";
const DEFAULT_OUTPUT_DIR = "raw";
public enum CliResultType {
OK,
NO_CONTENT,
MISSING_MEDIA
}
public struct CliResult {
CliResultType type;
IngestConfig config;
}
public CliResult parseArgs(string[] args) {
import std.path;
import std.file;
import std.getopt;
import std.string;
import std.stdio;
import filesizes;
CliResult result;
IngestConfig config;
config.outputDir = buildPath(getcwd(), DEFAULT_OUTPUT_DIR);
string mediaSearchDir = DEFAULT_MEDIA_DIR;
auto helpInfo = getopt(
args,
"mediaDir|i",
format!"The base directory from which to search for the GoPro media. Defaults to \"%s\"."(mediaSearchDir),
&mediaSearchDir,
"outputDir|o",
format!"The directory to copy data to. Defaults to \"%s\". Will create the directory if it doesn't exist yet."(config.outputDir),
&config.outputDir,
"force|f",
format!"Whether to forcibly overwrite existing files. Defaults to %s."(config.force),
&config.force,
"dryRun|d",
format!"Whether to perform a dry-run (don't actually copy anything). Defaults to %s."(config.dryRun),
&config.dryRun,
"bufferSize|b",
format!"The size of the buffer for copying files, in bytes. Defaults to %s."(formatFilesize(config.bufferSize)),
&config.bufferSize,
"clean|c",
format!"Whether to remove files from the GoPro media card after copying. Defaults to %s."(config.clean),
&config.clean
);
if (helpInfo.helpWanted) {
defaultGetoptPrinter("Ingestion tool for importing data from GoPro media cards.", helpInfo.options);
result.type = CliResultType.NO_CONTENT;
return result;
} else {
auto nullableGoProDir = getGoProDir(mediaSearchDir);
if (nullableGoProDir.isNull) {
writeln("Couldn't find GoPro directory.");
result.type = CliResultType.MISSING_MEDIA;
} else {
config.inputDir = nullableGoProDir.get();
writefln!"Found GoPro media at %s."(config.inputDir);
result.type = CliResultType.OK;
result.config = config;
}
}
return result;
}
unittest {
assert(parseArgs(["app", "-h"]).type == CliResultType.NO_CONTENT);
// TODO: Expand tests.
}
public void printBanner() {
import std.stdio : writeln;
writeln(
"+---------------------------------+\n" ~
"| |\n" ~
"| GoPro Ingester |\n" ~
"| v1.0.0 |\n" ~
"| by Andrew Lalis |\n" ~
"| |\n" ~
"+---------------------------------+\n"
);
}
public bool endsWithAny(string s, string[] suffixes ...) {
import std.algorithm : endsWith;
foreach (string suffix; suffixes) {
if (s.endsWith(suffix)) return true;
}
return false;
}
unittest {
assert(endsWithAny("abc", "a", "b", "c"));
assert(!endsWithAny("abc", "d"));
assert(!endsWithAny("abc", "A", "B", "C"));
}
/**
* Copies all files from the given source directory, to the given destination
* directory. Will create the destination directory if it doesn't exist yet.
* Overwrites any files that already exist in the destination directory.
* Params:
* sourceDir = The source directory to copy from.
* destDir = The destination directory to copy to.
*/
public void copyDir(string sourceDir, string destDir) {
import std.file;
if (!isDir(sourceDir)) return;
if (exists(destDir) && !isDir(destDir)) return;
if (!exists(destDir)) mkdirRecurse(destDir);
import std.path : buildPath, baseName;
foreach (DirEntry entry; dirEntries(sourceDir, SpanMode.shallow)) {
string destPath = buildPath(destDir, baseName(entry.name));
if (entry.isDir) {
copyDir(entry.name, destPath);
} else if (entry.isFile) {
copy(entry.name, destPath);
}
}
}
/**
* Tries to find a GoPro's media directory.
* Params:
* baseDir = The base directory to start the search from.
* Returns: A nullable string that, if present, refers to the GoPro's media
* directory.
*/
private Nullable!string getGoProDir(string baseDir) {
import std.file;
import std.path;
if (!exists(baseDir) || !isDir(baseDir)) return Nullable!string.init;
foreach (dir; std.file.dirEntries(baseDir, SpanMode.breadth)) {
// We know that a GoPro contains DCIM/100GOPRO in it.
string mediaPath = buildPath(dir.name, "DCIM", "100GOPRO");
if (exists(mediaPath) && isDir(mediaPath)) {
return nullable(mediaPath);
}
}
return Nullable!string.init;
}

View File

@ -1 +0,0 @@
Hello

View File

@ -1,2 +0,0 @@
# Card 1
This is an empty card.

View File

@ -1 +0,0 @@
Hello

View File

@ -1 +0,0 @@
# Card 2

View File

@ -1 +0,0 @@
Hello

View File

@ -1 +0,0 @@
# Card 3