From eebcf0c91a0b384b6b6e88fd95419691d20eb756 Mon Sep 17 00:00:00 2001 From: andrewlalis Date: Mon, 17 Jul 2023 15:10:58 -0400 Subject: [PATCH] Added scheduled tasks and version management. --- build-and-deploy.sh | 3 +- cleaner.d | 26 --------- dub.json | 4 +- dub.selections.json | 11 +++- source/app.d | 126 ++---------------------------------------- source/csgs/extract.d | 123 +++++++++++++++++++++++++++++++++++++++++ source/csgs/http.d | 41 ++++++++++++++ source/csgs/package.d | 1 + source/csgs/task.d | 110 ++++++++++++++++++++++++++++++++++++ 9 files changed, 294 insertions(+), 151 deletions(-) delete mode 100644 cleaner.d create mode 100644 source/csgs/extract.d create mode 100644 source/csgs/http.d create mode 100644 source/csgs/package.d create mode 100644 source/csgs/task.d diff --git a/build-and-deploy.sh b/build-and-deploy.sh index efa30b7..3eb8eed 100755 --- a/build-and-deploy.sh +++ b/build-and-deploy.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash dub clean -dub build --build=release +rm -f create-schematic-gen-site +dub build echo "Stopping CSGS service." ssh -f root@andrewlalis.com 'systemctl stop csgs' echo "Uploading new binary." diff --git a/cleaner.d b/cleaner.d deleted file mode 100644 index 1e2dc93..0000000 --- a/cleaner.d +++ /dev/null @@ -1,26 +0,0 @@ -/** - * This standalone module is responsible for cleaning up the list of stored - * extracts, so only the recent ones remain. This is meant to be linked as a - * cron scheduled program. - */ -module cleaner; - -import std.stdio; -import std.file; -import std.path; -import std.datetime; - -const EXTRACTS_DIR = "extracts"; - -int main() { - if (!exists(EXTRACTS_DIR)) return 0; - immutable SysTime now = Clock.currTime(); - foreach (DirEntry entry; dirEntries(EXTRACTS_DIR, SpanMode.shallow, false)) { - Duration age = now - entry.timeLastModified(); - if (age.total!"days" > 5) { - writefln!"Removing directory %s because it's too old."(entry.name); - rmdirRecurse(entry.name); - } - } - return 0; -} diff --git a/dub.json b/dub.json index 0a1812b..f8b4ca7 100644 --- a/dub.json +++ b/dub.json @@ -4,7 +4,9 @@ ], "copyright": "Copyright © 2023, Andrew Lalis", "dependencies": { - "handy-httpd": "~>7.6.3", + "handy-httpd": "~>7.6.4", + "requests": "~>2.1.1", + "scheduled": "~>1.2.0", "slf4d": "~>2.4.2" }, "description": "HTTP server for generating schematic materials lists.", diff --git a/dub.selections.json b/dub.selections.json index 9afe0a3..b3b5e0c 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -1,9 +1,16 @@ { "fileVersion": 1, "versions": { - "handy-httpd": "7.6.3", + "automem": "0.6.9", + "cachetools": "0.4.1", + "cronexp": "0.1.0-beta3", + "handy-httpd": "7.6.4", "httparsed": "1.2.1", + "requests": "2.1.1", + "scheduled": "1.1.0", "slf4d": "2.4.2", - "streams": "3.5.0" + "streams": "3.5.0", + "test_allocator": "0.3.4", + "unit-threaded": "0.10.8" } } diff --git a/source/app.d b/source/app.d index 0d324a1..09f77d6 100644 --- a/source/app.d +++ b/source/app.d @@ -1,130 +1,14 @@ -import handy_httpd; -import handy_httpd.handlers.path_delegating_handler; -import handy_httpd.handlers.file_resolving_handler; import slf4d; import slf4d.default_provider; -import std.path; -import std.file; -import std.json; -const EXTRACTS_DIR = "extracts"; -const EXTRACT_FILENAME = "__EXTRACT__.json"; -const EXTRACT_COMMAND = ["java", "-jar", "materials-extractor-v1.0.0.jar"]; +import csgs.http; +import csgs.task; void main() { auto provider = new shared DefaultProvider(true, Levels.INFO); configureLoggingProvider(provider); - info("Starting create-schematic-gen-site API."); - - ServerConfig config = ServerConfig.defaultValues(); - config.workerPoolSize = 3; - config.connectionQueueSize = 10; - config.port = 8100; - - PathDelegatingHandler handler = new PathDelegatingHandler(); - handler.addMapping("POST", "/extracts", &handleExtract); - handler.addMapping("GET", "/extracts/{extractId}", &getExtract); - - FileResolvingHandler fileHandler = new FileResolvingHandler("site", DirectoryResolutionStrategies.serveIndexFiles); - handler.addMapping("/**", fileHandler); - new HttpServer(handler, config).start(); -} - -void handleExtract(ref HttpRequestContext ctx) { - import std.json; - import std.uuid; - import std.process; - import std.stdio; - - immutable UUID extractId = randomUUID(); - MultipartFormData data = ctx.request.readBodyAsMultipartFormData(); - if (!validateExtractRequest(data, ctx.response)) return; - const extractDir = buildPath(EXTRACTS_DIR, extractId.toString()); - if (!exists(extractDir)) { - mkdirRecurse(extractDir); - } - string[] filenames; - uint[] counts; - foreach (MultipartElement element; data.elements) { - if (element.name == "schematics") { - const filePath = buildPath(extractDir, element.filename.get()); - std.file.write(filePath, element.content); - filenames ~= filePath; - } else if (element.name == "counts") { - import std.conv; - immutable uint count = element.content.to!uint; - counts ~= count; - } - } - infoF!"Running extract process on files: %s"(filenames); - auto extractionResult = execute(EXTRACT_COMMAND ~ filenames); - immutable int exitCode = extractionResult.status; - infoF!"Exit code: %d"(exitCode); - rmdirRecurse(extractDir); - if (exitCode != 0) { - ctx.response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR); - ctx.response.writeBodyString(extractionResult.output); - } else { - const extractJsonPath = extractDir ~ ".json"; - // It was successful, so add __COUNT__ to each object in the extract data. - JSONValue extractData = parseJSON(extractionResult.output); - for (uint i = 0; i < extractData.array.length; i++) { - extractData.array[i].object["__COUNT__"] = JSONValue(counts[i]); - } - std.file.write(extractJsonPath, extractData.toPrettyString()); - - JSONValue result = JSONValue.emptyObject; - result.object["extractId"] = JSONValue(extractId.toString()); - ctx.response.writeBodyString(result.toJSON(), "application/json"); - } -} - -private bool validateExtractRequest(ref MultipartFormData data, ref HttpResponse response) { - if (data.elements.length < 2) { - response.setStatus(HttpStatus.BAD_REQUEST); - response.writeBodyString("Requires at least 2 form data elements (schematic file and count)."); - return false; - } - uint nextElementIdx = 0; - while (nextElementIdx < data.elements.length) { - MultipartElement element = data.elements[nextElementIdx++]; - if (element.name == "schematics") { - if (element.filename.isNull || element.filename.get().length < 5 || element.content.length < 10) { - response.setStatus(HttpStatus.BAD_REQUEST); - response.writeBodyString("Invalid or missing schematic file."); - return false; - } - const string filename = element.filename.get(); - if (nextElementIdx == data.elements.length) { - response.setStatus(HttpStatus.BAD_REQUEST); - response.writeBodyString("Missing count element for schematic: " ~ filename); - return false; - } - MultipartElement countElement = data.elements[nextElementIdx++]; - import std.conv; - try { - immutable uint count = countElement.content.to!uint; - if (count < 1 || count > 1000) throw new Exception("out of range: count should be between 1 and 1000, inclusive."); - } catch (Exception e) { - response.setStatus(HttpStatus.BAD_REQUEST); - response.writeBodyString("Invalid count element: " ~ e.msg); - return false; - } - } else if (element.name == "processing-terminal") { - // TODO: Check processing-terminal format. - } else { - response.setStatus(HttpStatus.BAD_REQUEST); - response.writeBodyString("Unknown element: " ~ element.name); - return false; - } - } - - return true; -} - -void getExtract(ref HttpRequestContext ctx) { - string extractId = ctx.request.getPathParamAs!string("extractId"); - const extractFile = buildPath(EXTRACTS_DIR, extractId ~ ".json"); - fileResponse(ctx.response, extractFile, "application/json"); + info("Starting create-schematic-gen-site API."); + startScheduledTasks(); + startServer(); } diff --git a/source/csgs/extract.d b/source/csgs/extract.d new file mode 100644 index 0000000..641addd --- /dev/null +++ b/source/csgs/extract.d @@ -0,0 +1,123 @@ +module csgs.extract; + +import handy_httpd; +import slf4d; + +import std.typecons; +import std.file; +import std.path; + +const EXTRACTS_DIR = "extracts"; +const EXTRACT_FILENAME = "__EXTRACT__.json"; +const EXTRACT_COMMAND = ["java", "-jar", "materials-extractor-v1.0.0.jar"]; + +bool validateExtractRequest(ref MultipartFormData data, ref HttpResponse response) { + if (data.elements.length < 2) { + response.setStatus(HttpStatus.BAD_REQUEST); + response.writeBodyString("Requires at least 2 form data elements (schematic file and count)."); + return false; + } + uint nextElementIdx = 0; + while (nextElementIdx < data.elements.length) { + MultipartElement element = data.elements[nextElementIdx++]; + if (element.name == "schematics") { + if (element.filename.isNull || element.filename.get().length < 5 || element.content.length < 10) { + response.setStatus(HttpStatus.BAD_REQUEST); + response.writeBodyString("Invalid or missing schematic file."); + return false; + } + const string filename = element.filename.get(); + if (nextElementIdx == data.elements.length) { + response.setStatus(HttpStatus.BAD_REQUEST); + response.writeBodyString("Missing count element for schematic: " ~ filename); + return false; + } + MultipartElement countElement = data.elements[nextElementIdx++]; + import std.conv; + try { + immutable uint count = countElement.content.to!uint; + if (count < 1 || count > 1000) throw new Exception("out of range: count should be between 1 and 1000, inclusive."); + } catch (Exception e) { + response.setStatus(HttpStatus.BAD_REQUEST); + response.writeBodyString("Invalid count element: " ~ e.msg); + return false; + } + } else if (element.name == "processing-terminal") { + // TODO: Check processing-terminal format. + } else { + response.setStatus(HttpStatus.BAD_REQUEST); + response.writeBodyString("Unknown element: " ~ element.name); + return false; + } + } + + return true; +} + +string doExtract(ref MultipartFormData data) { + import std.json; + import std.uuid; + import std.process; + + immutable UUID extractId = randomUUID(); + immutable string extractIdStr = extractId.toString(); + immutable string extractDir = buildPath(EXTRACTS_DIR, extractIdStr); + if (!exists(extractDir)) { + mkdirRecurse(extractDir); + } + string[] filenames; + uint[] counts; + foreach (MultipartElement element; data.elements) { + if (element.name == "schematics") { + const filePath = buildPath(extractDir, element.filename.get()); + std.file.write(filePath, element.content); + filenames ~= filePath; + } else if (element.name == "counts") { + import std.conv; + immutable uint count = element.content.to!uint; + counts ~= count; + } + } + + infoF!"Running extract process on files: %s"(filenames); + Nullable!string extractorPath = getExtractorProgramPath(); + if (extractorPath.isNull) { + throw new HttpStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not find extractor program."); + } + auto extractionResult = execute(["java", "-jar", extractorPath.get()] ~ filenames); + immutable int exitCode = extractionResult.status; + infoF!"Exit code: %d"(exitCode); + rmdirRecurse(extractDir); + if (exitCode != 0) { + throw new HttpStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Extraction program failed."); + } else { + immutable string extractJsonPath = extractDir ~ ".json"; + // It was successful, so add __COUNT__ to each object in the extract data. + JSONValue extractData = parseJSON(extractionResult.output); + for (uint i = 0; i < extractData.array.length; i++) { + extractData.array[i].object["__COUNT__"] = JSONValue(counts[i]); + } + std.file.write(extractJsonPath, extractData.toPrettyString()); + return extractIdStr; + } +} + +Nullable!string getExtractorProgramPath() { + import std.string; + foreach (DirEntry entry; dirEntries(getcwd(), SpanMode.shallow, false)) { + if (entry.isFile() && endsWith(entry.name, ".jar")) { + return nullable(entry.name); + } + } + return Nullable!string.init; +} + +Nullable!string getCurrentExtractorVersion() { + import std.regex; + Nullable!string extractorPath = getExtractorProgramPath(); + if (extractorPath.isNull) return Nullable!string.init; + auto r = regex(`v\d+\.\d+\.\d+`); + auto c = matchFirst(extractorPath.get(), r); + if (c.empty) return Nullable!string.init; + return nullable(c.front()); +} diff --git a/source/csgs/http.d b/source/csgs/http.d new file mode 100644 index 0000000..dff3585 --- /dev/null +++ b/source/csgs/http.d @@ -0,0 +1,41 @@ +module csgs.http; + +import handy_httpd; +import handy_httpd.handlers.path_delegating_handler; +import handy_httpd.handlers.file_resolving_handler; +import slf4d; +import std.path; + +import csgs.extract; + +void startServer() { + ServerConfig config = ServerConfig.defaultValues(); + config.workerPoolSize = 3; + config.connectionQueueSize = 10; + config.port = 8100; + + PathDelegatingHandler handler = new PathDelegatingHandler(); + handler.addMapping("POST", "/extracts", &handleExtract); + handler.addMapping("GET", "/extracts/{extractId}", &getExtract); + + FileResolvingHandler fileHandler = new FileResolvingHandler("site", DirectoryResolutionStrategies.serveIndexFiles); + handler.addMapping("/**", fileHandler); + new HttpServer(handler, config).start(); +} + +private void handleExtract(ref HttpRequestContext ctx) { + import std.json; + + MultipartFormData data = ctx.request.readBodyAsMultipartFormData(); + if (!validateExtractRequest(data, ctx.response)) return; + string extractId = doExtract(data); + JSONValue result = JSONValue.emptyObject; + result.object["extractId"] = JSONValue(extractId); + ctx.response.writeBodyString(result.toJSON(), "application/json"); +} + +private void getExtract(ref HttpRequestContext ctx) { + string extractId = ctx.request.getPathParamAs!string("extractId"); + const extractFile = buildPath(EXTRACTS_DIR, extractId ~ ".json"); + fileResponse(ctx.response, extractFile, "application/json"); +} \ No newline at end of file diff --git a/source/csgs/package.d b/source/csgs/package.d new file mode 100644 index 0000000..ed42f38 --- /dev/null +++ b/source/csgs/package.d @@ -0,0 +1 @@ +module csgs; diff --git a/source/csgs/task.d b/source/csgs/task.d new file mode 100644 index 0000000..06902c1 --- /dev/null +++ b/source/csgs/task.d @@ -0,0 +1,110 @@ +module csgs.task; + +import slf4d; +import scheduled; + +import std.datetime; +import std.typecons; +import std.json; +import std.string; +import std.file; +import std.stdio; + +import csgs.extract; + +void startScheduledTasks() { + JobScheduler scheduler = new ThreadedJobScheduler(); + Job cleanJob = new FunctionJob(&cleanOldExtracts); + scheduler.addJob(cleanJob, new FixedIntervalSchedule(days(1))); + Job updateCheckJob = new FunctionJob(&checkForExtractorUpdate); + scheduler.addJob(updateCheckJob, new FixedIntervalSchedule(hours(1))); + scheduler.start(); +} + +private void cleanOldExtracts() { + import std.file; + immutable MAX_AGE = days(5); + info("Cleaning old extracts."); + if (!exists(EXTRACTS_DIR)) return; + immutable SysTime now = Clock.currTime(); + foreach (DirEntry entry; dirEntries(EXTRACTS_DIR, SpanMode.shallow, false)) { + if (entry.isDir) { + infoF!"Removing directory %s."(entry.name); + rmdirRecurse(entry.name); + } else if (entry.isFile) { + immutable Duration age = now - entry.timeLastModified(); + if (age > MAX_AGE) { + infoF!"Removing extract %s because it's too old."(entry.name); + std.file.remove(entry.name); + } + } + } +} + +private void checkForExtractorUpdate() { + import requests; + import std.json; + import std.stdio; + import std.string; + import std.file; + info("Checking for MaterialsExtractor program updates."); + + // First request the latest release from GitHub API: + Request req = Request(); + req.addHeaders(["Accept": "application/vnd.github+json"]); + Response resp = req.get("https://api.github.com/repos/andrewlalis/MaterialsExtractor/releases/latest"); + if (resp.code() != 200) { + warnF!"Failed to get the latest MaterialsExtractor release. Status code %d."(resp.code); + return; + } + JSONValue responseJson = parseJSON(resp.responseBody.toString()); + + // Get the release's version, compare it with our current version. + immutable string releaseVersion = responseJson.object["tag_name"].str; + infoF!"Found release version %s"(releaseVersion); + Nullable!string currentVersion = getCurrentExtractorVersion(); + if (!currentVersion.isNull && currentVersion.get() >= releaseVersion) { + infoF!"Skipping this version since it's not newer than the current %s."(currentVersion.get); + return; + } + + // Find the release's asset, and download it. + Nullable!JSONValue nullableAsset = findJarAsset(responseJson); + if (nullableAsset.isNull) { + warn("Latest release has no asset. Ignoring."); + return; + } + JSONValue asset = nullableAsset.get(); + immutable string filename = asset.object["name"].str; + immutable string downloadUrl = asset.object["browser_download_url"].str; + infoF!"Found asset: %s. Downloading from %s."(filename, downloadUrl); + if (exists(filename)) { + std.file.remove(filename); + } + + Request downloadRequest = Request(); + downloadRequest.useStreaming = true; + Response downloadResponse = downloadRequest.get(downloadUrl); + File f = File(filename, "wb"); + auto stream = downloadResponse.receiveAsRange(); + while (!stream.empty) { + f.rawWrite(stream.front); + stream.popFront(); + } + f.close(); + infoF!"Downloaded file to %s"(f.name); +} + +private Nullable!JSONValue findJarAsset(ref JSONValue response) { + foreach (JSONValue asset; response.object["assets"].array) { + immutable string name = asset.object["name"].str; + if ( + startsWith(name, "materials-extractor") && + endsWith(name, ".jar") && + asset.object["content_type"].str == "application/java-archive" + ) { + return nullable(asset); + } + } + return Nullable!JSONValue.init; +}