diff --git a/deploy.sh b/deploy.sh index 19f7706..5b173f4 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash +# A personal script I use to deploy a sitestat instance to andrewlalis.com echo "Building sitestat" dub clean dub build --build=release --compiler=/opt/ldc2/ldc2-1.33.0-linux-x86_64/bin/ldc2 +echo "Deploying to andrewlalis.com" ssh -f root@andrewlalis.com 'systemctl stop sitestat.service' scp sitestat root@andrewlalis.com:/opt/sitestat/ ssh -f root@andrewlalis.com 'systemctl start sitestat.service' +echo "Done!" diff --git a/dub.json b/dub.json index 3d595cd..ca9a11a 100644 --- a/dub.json +++ b/dub.json @@ -5,7 +5,7 @@ "copyright": "Copyright © 2023, Andrew Lalis", "dependencies": { "d-properties": "~>1.0.5", - "handy-httpd": "~>7.11.1", + "handy-httpd": "~>7.13.0", "d2sqlite3": "~>1.0.0" }, "description": "A simple webserver and script for tracking basic, non-intrusive site statistics.", diff --git a/dub.selections.json b/dub.selections.json index c3df505..178cc08 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -3,8 +3,9 @@ "versions": { "d-properties": "1.0.5", "d2sqlite3": "1.0.0", - "handy-httpd": "7.11.1", + "handy-httpd": "7.13.0", "httparsed": "1.2.1", + "path-matcher": "1.1.3", "slf4d": "2.4.3", "streams": "3.5.0" } diff --git a/source/app.d b/source/app.d index 32d4385..b8454a6 100644 --- a/source/app.d +++ b/source/app.d @@ -1,7 +1,7 @@ import server : startServer; -import report : makeReport; +import report.gen : makeReport; -void main(string[] args) { +int main(string[] args) { import slf4d; import slf4d.default_provider; auto provider = new shared DefaultProvider(false, Levels.INFO); @@ -11,7 +11,12 @@ void main(string[] args) { if (args.length <= 1) { startServer(); + return 0; } else if (args[1] == "report") { - makeReport(args[2..$]); + return makeReport(args[2..$]); + } else { + import std.stdio; + writeln("Invalid command. Expected no-args to start server, or \"report\" for report generation."); + return 1; } } diff --git a/source/report.d b/source/report.d deleted file mode 100644 index d2a9482..0000000 --- a/source/report.d +++ /dev/null @@ -1,99 +0,0 @@ -module report; - -import utils; -import std.stdio; -import std.datetime; -import std.algorithm; -import std.array; -import d2sqlite3; -import data; -import std.json; - -void makeReport(string[] sites) { - string[] sitesToReport = sites.length == 0 ? listAllOrigins() : sites; - if (sitesToReport.length == 0) { - writeln("No sites to report."); - return; - } - writeln("Creating report for " ~ createSiteNamesSummary(sitesToReport) ~ "."); - foreach (site; sitesToReport) { - writeln("Site: " ~ site); - SiteReport report = generateReport(site, Clock.currTime(UTC()) - hours(48), Clock.currTime(UTC())); - writeln(report.toJson().toPrettyString()); - } -} - -string createSiteNamesSummary(string[] sites) { - if (sites.length == 0) return "all sites"; - if (sites.length == 1) return "site " ~ sites[0]; - import std.array; - auto app = appender!string(); - app ~= "sites "; - foreach (i, site; sites) { - app ~= site; - if (i + 2 == sites.length) { - app ~= ", and "; - } else if (i + 1 < sites.length) { - app ~= ", "; - } - } - return app[]; -} - -struct SiteReport { - string siteName; - SysTime periodStart; - SysTime periodEnd; - - ulong totalSessions; - double meanSessionDurationSeconds; - double meanEventsPerSession; - - JSONValue toJson() const { - JSONValue obj = JSONValue(string[string].init); - obj.object["siteName"] = siteName; - obj.object["periodStart"] = periodStart.toISOExtString(); - obj.object["periodEnd"] = periodEnd.toISOExtString(); - obj.object["totalSessions"] = totalSessions; - obj.object["meanSessionDurationSeconds"] = meanSessionDurationSeconds; - obj.object["meanEventsPerSession"] = meanEventsPerSession; - return obj; - } -} - -SiteReport generateReport(string site, SysTime periodStart, SysTime periodEnd) { - SiteReport report; - report.siteName = site; - report.periodStart = periodStart; - report.periodEnd = periodEnd; - immutable string TS_START = formatSqliteTimestamp(periodStart); - immutable string TS_END = formatSqliteTimestamp(periodEnd); - writefln!"TS_START = %s, TS_END = %s"(TS_START, TS_END); - Database db = getOrCreateDatabase(site); - - report.totalSessions = db.execute(q"SQL - SELECT COUNT(id) - FROM session - WHERE start_timestamp >= ? AND end_timestamp <= ? -SQL", - TS_START, TS_END - ).oneValue!ulong(); - - report.meanEventsPerSession = db.execute(q"SQL - SELECT AVG(event_count) - FROM session - WHERE start_timestamp >= ? AND end_timestamp <= ? -SQL", - TS_START, TS_END - ).oneValue!double(); - - report.meanSessionDurationSeconds = db.execute(q"SQL - SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60) - FROM session - WHERE start_timestamp >= ? AND end_timestamp <= ? -SQL", - TS_START, TS_END - ).oneValue!double(); - - return report; -} diff --git a/source/report/gen.d b/source/report/gen.d new file mode 100644 index 0000000..304c262 --- /dev/null +++ b/source/report/gen.d @@ -0,0 +1,211 @@ +module report.gen; + +import utils; +import std.stdio; +import std.datetime; +import std.algorithm; +import std.array; +import std.file; +import d2sqlite3; +import data; +import std.json; + +import report.output; + +int makeReport(string[] args) { + if (args.length == 1 && (args[0] == "-h" || args[0] == "--help")) { + printHelp(); + return 0; + } + string[] sitesToReport = parseSites(args); + if (sitesToReport.length == 0) { + writeln("No sites to report."); + return 1; + } + ReportPeriod period = parseReportPeriod(args); + ReportOutputGenerator outputGen = parseReportFormat(args); + SiteReport[] reports; + foreach (site; sitesToReport) { + reports ~= generateReport(site, period); + } + outputGen.generate(reports); + return 0; +} + +void printHelp() { + writeln(q"TXT +sitestat report help information: + +The "report" command is used to generate aggregate reports for one or more +sites tracked by sitestat, over a given time period. The following arguments +are accepted: + +-s | --site + Specify the name of a site to include in the report. If no sites are + specified, then all tracked sites will be included in reports. + +--start + Set the starting timestamp of the reporting period, using an ISO-8601 + formatted date and time. For example, 2023-11-05T12:00:00. Timestamps are + always interpretted as UTC timezone. Defaults to 1970-01-01. + +--end + Set the end timestamp of the reporting period, using an ISO-8601 formatted + date and time. Timestamps are always interpretted as UTC timezone. Defaults + to the current time. + +-f | --format + Specify the desired output format for the report(s) generated by this + command. The following formats are available: text, json, csv. + Defaults to the "text" format if none is specified. +TXT"); +} + +string[] parseSites(string[] args) { + string[] sites; + for (size_t i = 0; i < args.length; i++) { + if (i + 1 < args.length) { + if (args[i] == "-s" || args[i] == "--site") { + string site = args[i + 1]; + if (!std.file.exists(dbPath(site))) { + throw new Exception("site " ~ site ~ " doesn't exist."); + } + sites ~= site; + } + } + } + if (sites.length == 0) return listAllOrigins(); + return sites; +} + +ReportPeriod parseReportPeriod(string[] args) { + SysTime start = SysTime(Date(1970, 1, 1), UTC()); + SysTime end = Clock.currTime(UTC()); + + /// Helper function to parse a timestamp from a string. + SysTime tryParseTime(string s) { + try { + return SysTime.fromISOExtString(s, UTC()); + } catch (DateTimeException e) { + throw new Exception( + "Failed to parse timestamp. Expected an ISO-8601 formatted datetime string: YYYY-MM-DDTHH:MM:SS", + e + ); + } + } + + bool startParsed = false; + bool endParsed = false; + for (size_t i = 0; i < args.length; i++) { + if (i + 1 < args.length) { + if (args[i] == "--start" && !startParsed) { + start = tryParseTime(args[i + 1]); + startParsed = true; + } else if (args[i] == "--end" && !endParsed) { + end = tryParseTime(args[i + 1]); + endParsed = true; + } + } + } + + return ReportPeriod(start, end); +} + +string createSiteNamesSummary(string[] sites) { + if (sites.length == 0) return "all sites"; + if (sites.length == 1) return "site " ~ sites[0]; + import std.array; + auto app = appender!string(); + app ~= "sites "; + foreach (i, site; sites) { + app ~= site; + if (i + 2 == sites.length) { + app ~= ", and "; + } else if (i + 1 < sites.length) { + app ~= ", "; + } + } + return app[]; +} + +ReportOutputGenerator parseReportFormat(string[] args) { + for (size_t i = 0; i < args.length; i++) { + if (i + 1 < args.length) { + if (args[i] == "-f" || args[i] == "--format") { + import std.uni : toLower; + import std.string : strip; + string formatStr = toLower(strip(args[i + 1])); + if (formatStr == "text") return new ReportTextOutputGenerator(); + if (formatStr == "json") return new ReportJsonOutputGenerator(); + if (formatStr == "csv") return new ReportCsvOutputGenerator(); + throw new Exception("Invalid report output format: " ~ args[i + 1]); + } + } + } + return new ReportTextOutputGenerator(); +} + +immutable struct ReportPeriod { + SysTime start; + SysTime end; +} + +struct SiteReport { + string siteName; + ReportPeriod period; + + ulong totalSessions; + double meanSessionDurationSeconds; + double meanEventsPerSession; + ulong[string] userAgents; + + this(string siteName, ReportPeriod period) { + this.siteName = siteName; + this.period = period; + } +} + +SiteReport generateReport(string site, ReportPeriod period) { + SiteReport report = SiteReport(site, period); + immutable string TS_START = formatSqliteTimestamp(period.start); + immutable string TS_END = formatSqliteTimestamp(period.end); + Database db = getOrCreateDatabase(site); + + report.totalSessions = db.execute(q"SQL + SELECT COUNT(id) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!ulong(); + + report.meanEventsPerSession = db.execute(q"SQL + SELECT AVG(event_count) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!double(); + + report.meanSessionDurationSeconds = db.execute(q"SQL + SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!double(); + + ResultRange userAgentsResult = db.execute(q"SQL + SELECT user_agent, COUNT(user_agent) AS c + FROM session + GROUP BY user_agent + ORDER BY c DESC +SQL"); + foreach (Row row; userAgentsResult) { + string userAgent = row["user_agent"].as!string; + ulong count = row["c"].as!ulong; + report.userAgents[userAgent] = count; + } + + return report; +} diff --git a/source/report/output.d b/source/report/output.d new file mode 100644 index 0000000..4d2326a --- /dev/null +++ b/source/report/output.d @@ -0,0 +1,74 @@ +module report.output; + +import std.stdio; +import report.gen : SiteReport; + +interface ReportOutputGenerator { + void generate(const(SiteReport[]) reports); +} + +/** + * Report output generator that outputs reports as human-readable text, without + * any particular format. + */ +class ReportTextOutputGenerator : ReportOutputGenerator { + void generate(const(SiteReport[]) reports) { + foreach (report; reports) { + writefln!"Report for site %s from %s to %s:"( + report.siteName, + report.period.start.toISOExtString(), + report.period.end.toISOExtString() + ); + writefln!" Total sessions: %d"(report.totalSessions); + writefln!" Mean session duration (seconds): %.3f"(report.meanSessionDurationSeconds); + writefln!" Mean events per sesson: %.3f"(report.meanEventsPerSession); + foreach (string userAgent, ulong count; report.userAgents) { + writefln!" User agent: %s"(userAgent); + writefln!" Count: %d"(count); + } + } + } +} + +/** + * Report output generator that outputs reports as a JSON array of objects. + */ +class ReportJsonOutputGenerator : ReportOutputGenerator { + import std.json; + + void generate(const(SiteReport[]) reports) { + JSONValue jsonArray = JSONValue(string[].init); + foreach (report; reports) { + JSONValue obj = JSONValue(string[string].init); + obj.object["siteName"] = report.siteName; + obj.object["periodStart"] = report.period.start.toISOExtString(); + obj.object["periodEnd"] = report.period.end.toISOExtString(); + obj.object["totalSessions"] = report.totalSessions; + obj.object["meanSessionDurationSeconds"] = report.meanSessionDurationSeconds; + obj.object["meanEventsPerSession"] = report.meanEventsPerSession; + obj.object["userAgents"] = JSONValue(string[string].init); + foreach (string userAgent, ulong count; report.userAgents) { + obj.object["userAgents"].object[userAgent] = count; + } + jsonArray.array ~= obj; + } + writeln(jsonArray.toPrettyString()); + } +} + +/** + * Report output generator that generates output as CSV text, grouped by each + * site's name. + */ +class ReportCsvOutputGenerator : ReportOutputGenerator { + void generate(const(SiteReport[]) reports) { + writeln("site, statistic, value"); // Headers. + foreach (report; reports) { + writefln!"%s, periodStart, %s"(report.siteName, report.period.start.toISOExtString()); + writefln!"%s, periodEnd, %s"(report.siteName, report.period.end.toISOExtString()); + writefln!"%s, totalSessions, %d"(report.siteName, report.totalSessions); + writefln!"%s, meanSessionDurationSeconds, %.3f"(report.siteName, report.meanSessionDurationSeconds); + writefln!"%s, meanEventsPerSession, %.3f"(report.siteName, report.meanEventsPerSession); + } + } +} diff --git a/source/report/package.d b/source/report/package.d new file mode 100644 index 0000000..1407fa6 --- /dev/null +++ b/source/report/package.d @@ -0,0 +1,3 @@ +module report; + +public import report.gen; diff --git a/source/server.d b/source/server.d index 3ccd426..2e9ce65 100644 --- a/source/server.d +++ b/source/server.d @@ -2,7 +2,7 @@ module server; import std.file; import handy_httpd; -import handy_httpd.handlers.path_delegating_handler; +import handy_httpd.handlers.path_handler; import d_properties; import slf4d; @@ -22,7 +22,7 @@ void startServer() { */ private HttpRequestHandler prepareHandler(Properties props) { import live_tracker; - PathDelegatingHandler pathHandler = new PathDelegatingHandler(); + PathHandler pathHandler = new PathHandler(); pathHandler.addMapping( Method.GET, "/ws",