andrew
/
sitestat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored the report generation feature quite a bit.

This commit is contained in:
Andrew Lalis 2023-11-05 10:18:12 -05:00
parent 391fc48829
commit 5e0b60e9fd
9 changed files with 304 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deploy.sh
View File

@ -1,9 +1,12 @@
#!/usr/bin/env bash
# A personal script I use to deploy a sitestat instance to andrewlalis.com
echo "Building sitestat"
dub clean
dub build --build=release --compiler=/opt/ldc2/ldc2-1.33.0-linux-x86_64/bin/ldc2
echo "Deploying to andrewlalis.com"
ssh -f root@andrewlalis.com 'systemctl stop sitestat.service'
scp sitestat root@andrewlalis.com:/opt/sitestat/
ssh -f root@andrewlalis.com 'systemctl start sitestat.service'
echo "Done!"

2
dub.json
View File

@ -5,7 +5,7 @@
"copyright": "Copyright © 2023, Andrew Lalis",
"dependencies": {
"d-properties": "~>1.0.5",
"handy-httpd": "~>7.11.1",
"handy-httpd": "~>7.13.0",
"d2sqlite3": "~>1.0.0"
},
"description": "A simple webserver and script for tracking basic, non-intrusive site statistics.",

3
dub.selections.json
View File

@ -3,8 +3,9 @@
"versions": {
"d-properties": "1.0.5",
"d2sqlite3": "1.0.0",
"handy-httpd": "7.11.1",
"handy-httpd": "7.13.0",
"httparsed": "1.2.1",
"path-matcher": "1.1.3",
"slf4d": "2.4.3",
"streams": "3.5.0"
}

11
source/app.d
View File

@ -1,7 +1,7 @@
import server : startServer;
import report : makeReport;
import report.gen : makeReport;
void main(string[] args) {
int main(string[] args) {
import slf4d;
import slf4d.default_provider;
auto provider = new shared DefaultProvider(false, Levels.INFO);
@ -11,7 +11,12 @@ void main(string[] args) {
if (args.length <= 1) {
startServer();
return 0;
} else if (args[1] == "report") {
makeReport(args[2..$]);
return makeReport(args[2..$]);
} else {
import std.stdio;
writeln("Invalid command. Expected no-args to start server, or \"report\" for report generation.");
return 1;
}
}

99
source/report.d
View File

@ -1,99 +0,0 @@
module report;
import utils;
import std.stdio;
import std.datetime;
import std.algorithm;
import std.array;
import d2sqlite3;
import data;
import std.json;
void makeReport(string[] sites) {
string[] sitesToReport = sites.length == 0 ? listAllOrigins() : sites;
if (sitesToReport.length == 0) {
writeln("No sites to report.");
return;
}
writeln("Creating report for " ~ createSiteNamesSummary(sitesToReport) ~ ".");
foreach (site; sitesToReport) {
writeln("Site: " ~ site);
SiteReport report = generateReport(site, Clock.currTime(UTC()) - hours(48), Clock.currTime(UTC()));
writeln(report.toJson().toPrettyString());
}
}
string createSiteNamesSummary(string[] sites) {
if (sites.length == 0) return "all sites";
if (sites.length == 1) return "site " ~ sites[0];
import std.array;
auto app = appender!string();
app ~= "sites ";
foreach (i, site; sites) {
app ~= site;
if (i + 2 == sites.length) {
app ~= ", and ";
} else if (i + 1 < sites.length) {
app ~= ", ";
}
}
return app[];
}
struct SiteReport {
string siteName;
SysTime periodStart;
SysTime periodEnd;
ulong totalSessions;
double meanSessionDurationSeconds;
double meanEventsPerSession;
JSONValue toJson() const {
JSONValue obj = JSONValue(string[string].init);
obj.object["siteName"] = siteName;
obj.object["periodStart"] = periodStart.toISOExtString();
obj.object["periodEnd"] = periodEnd.toISOExtString();
obj.object["totalSessions"] = totalSessions;
obj.object["meanSessionDurationSeconds"] = meanSessionDurationSeconds;
obj.object["meanEventsPerSession"] = meanEventsPerSession;
return obj;
}
}
SiteReport generateReport(string site, SysTime periodStart, SysTime periodEnd) {
SiteReport report;
report.siteName = site;
report.periodStart = periodStart;
report.periodEnd = periodEnd;
immutable string TS_START = formatSqliteTimestamp(periodStart);
immutable string TS_END = formatSqliteTimestamp(periodEnd);
writefln!"TS_START = %s, TS_END = %s"(TS_START, TS_END);
Database db = getOrCreateDatabase(site);
report.totalSessions = db.execute(q"SQL
SELECT COUNT(id)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!ulong();
report.meanEventsPerSession = db.execute(q"SQL
SELECT AVG(event_count)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
report.meanSessionDurationSeconds = db.execute(q"SQL
SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
return report;
}

211
source/report/gen.d Normal file
View File

@ -0,0 +1,211 @@
module report.gen;
import utils;
import std.stdio;
import std.datetime;
import std.algorithm;
import std.array;
import std.file;
import d2sqlite3;
import data;
import std.json;
import report.output;
int makeReport(string[] args) {
if (args.length == 1 && (args[0] == "-h" || args[0] == "--help")) {
printHelp();
return 0;
}
string[] sitesToReport = parseSites(args);
if (sitesToReport.length == 0) {
writeln("No sites to report.");
return 1;
}
ReportPeriod period = parseReportPeriod(args);
ReportOutputGenerator outputGen = parseReportFormat(args);
SiteReport[] reports;
foreach (site; sitesToReport) {
reports ~= generateReport(site, period);
}
outputGen.generate(reports);
return 0;
}
void printHelp() {
writeln(q"TXT
sitestat report help information:
The "report" command is used to generate aggregate reports for one or more
sites tracked by sitestat, over a given time period. The following arguments
are accepted:
-s <sitename> | --site <sitename>
Specify the name of a site to include in the report. If no sites are
specified, then all tracked sites will be included in reports.
--start <timestamp>
Set the starting timestamp of the reporting period, using an ISO-8601
formatted date and time. For example, 2023-11-05T12:00:00. Timestamps are
always interpretted as UTC timezone. Defaults to 1970-01-01.
--end <timestamp>
Set the end timestamp of the reporting period, using an ISO-8601 formatted
date and time. Timestamps are always interpretted as UTC timezone. Defaults
to the current time.
-f <format> | --format <format>
Specify the desired output format for the report(s) generated by this
command. The following formats are available: text, json, csv.
Defaults to the "text" format if none is specified.
TXT");
}
string[] parseSites(string[] args) {
string[] sites;
for (size_t i = 0; i < args.length; i++) {
if (i + 1 < args.length) {
if (args[i] == "-s" || args[i] == "--site") {
string site = args[i + 1];
if (!std.file.exists(dbPath(site))) {
throw new Exception("site " ~ site ~ " doesn't exist.");
}
sites ~= site;
}
}
}
if (sites.length == 0) return listAllOrigins();
return sites;
}
ReportPeriod parseReportPeriod(string[] args) {
SysTime start = SysTime(Date(1970, 1, 1), UTC());
SysTime end = Clock.currTime(UTC());
/// Helper function to parse a timestamp from a string.
SysTime tryParseTime(string s) {
try {
return SysTime.fromISOExtString(s, UTC());
} catch (DateTimeException e) {
throw new Exception(
"Failed to parse timestamp. Expected an ISO-8601 formatted datetime string: YYYY-MM-DDTHH:MM:SS",
e
);
}
}
bool startParsed = false;
bool endParsed = false;
for (size_t i = 0; i < args.length; i++) {
if (i + 1 < args.length) {
if (args[i] == "--start" && !startParsed) {
start = tryParseTime(args[i + 1]);
startParsed = true;
} else if (args[i] == "--end" && !endParsed) {
end = tryParseTime(args[i + 1]);
endParsed = true;
}
}
}
return ReportPeriod(start, end);
}
string createSiteNamesSummary(string[] sites) {
if (sites.length == 0) return "all sites";
if (sites.length == 1) return "site " ~ sites[0];
import std.array;
auto app = appender!string();
app ~= "sites ";
foreach (i, site; sites) {
app ~= site;
if (i + 2 == sites.length) {
app ~= ", and ";
} else if (i + 1 < sites.length) {
app ~= ", ";
}
}
return app[];
}
ReportOutputGenerator parseReportFormat(string[] args) {
for (size_t i = 0; i < args.length; i++) {
if (i + 1 < args.length) {
if (args[i] == "-f" || args[i] == "--format") {
import std.uni : toLower;
import std.string : strip;
string formatStr = toLower(strip(args[i + 1]));
if (formatStr == "text") return new ReportTextOutputGenerator();
if (formatStr == "json") return new ReportJsonOutputGenerator();
if (formatStr == "csv") return new ReportCsvOutputGenerator();
throw new Exception("Invalid report output format: " ~ args[i + 1]);
}
}
}
return new ReportTextOutputGenerator();
}
immutable struct ReportPeriod {
SysTime start;
SysTime end;
}
struct SiteReport {
string siteName;
ReportPeriod period;
ulong totalSessions;
double meanSessionDurationSeconds;
double meanEventsPerSession;
ulong[string] userAgents;
this(string siteName, ReportPeriod period) {
this.siteName = siteName;
this.period = period;
}
}
SiteReport generateReport(string site, ReportPeriod period) {
SiteReport report = SiteReport(site, period);
immutable string TS_START = formatSqliteTimestamp(period.start);
immutable string TS_END = formatSqliteTimestamp(period.end);
Database db = getOrCreateDatabase(site);
report.totalSessions = db.execute(q"SQL
SELECT COUNT(id)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!ulong();
report.meanEventsPerSession = db.execute(q"SQL
SELECT AVG(event_count)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
report.meanSessionDurationSeconds = db.execute(q"SQL
SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
ResultRange userAgentsResult = db.execute(q"SQL
SELECT user_agent, COUNT(user_agent) AS c
FROM session
GROUP BY user_agent
ORDER BY c DESC
SQL");
foreach (Row row; userAgentsResult) {
string userAgent = row["user_agent"].as!string;
ulong count = row["c"].as!ulong;
report.userAgents[userAgent] = count;
}
return report;
}

74
source/report/output.d Normal file
View File

@ -0,0 +1,74 @@
module report.output;
import std.stdio;
import report.gen : SiteReport;
interface ReportOutputGenerator {
void generate(const(SiteReport[]) reports);
}
/**
* Report output generator that outputs reports as human-readable text, without
* any particular format.
*/
class ReportTextOutputGenerator : ReportOutputGenerator {
void generate(const(SiteReport[]) reports) {
foreach (report; reports) {
writefln!"Report for site %s from %s to %s:"(
report.siteName,
report.period.start.toISOExtString(),
report.period.end.toISOExtString()
);
writefln!" Total sessions: %d"(report.totalSessions);
writefln!" Mean session duration (seconds): %.3f"(report.meanSessionDurationSeconds);
writefln!" Mean events per sesson: %.3f"(report.meanEventsPerSession);
foreach (string userAgent, ulong count; report.userAgents) {
writefln!" User agent: %s"(userAgent);
writefln!" Count: %d"(count);
}
}
}
}
/**
* Report output generator that outputs reports as a JSON array of objects.
*/
class ReportJsonOutputGenerator : ReportOutputGenerator {
import std.json;
void generate(const(SiteReport[]) reports) {
JSONValue jsonArray = JSONValue(string[].init);
foreach (report; reports) {
JSONValue obj = JSONValue(string[string].init);
obj.object["siteName"] = report.siteName;
obj.object["periodStart"] = report.period.start.toISOExtString();
obj.object["periodEnd"] = report.period.end.toISOExtString();
obj.object["totalSessions"] = report.totalSessions;
obj.object["meanSessionDurationSeconds"] = report.meanSessionDurationSeconds;
obj.object["meanEventsPerSession"] = report.meanEventsPerSession;
obj.object["userAgents"] = JSONValue(string[string].init);
foreach (string userAgent, ulong count; report.userAgents) {
obj.object["userAgents"].object[userAgent] = count;
}
jsonArray.array ~= obj;
}
writeln(jsonArray.toPrettyString());
}
}
/**
* Report output generator that generates output as CSV text, grouped by each
* site's name.
*/
class ReportCsvOutputGenerator : ReportOutputGenerator {
void generate(const(SiteReport[]) reports) {
writeln("site, statistic, value"); // Headers.
foreach (report; reports) {
writefln!"%s, periodStart, %s"(report.siteName, report.period.start.toISOExtString());
writefln!"%s, periodEnd, %s"(report.siteName, report.period.end.toISOExtString());
writefln!"%s, totalSessions, %d"(report.siteName, report.totalSessions);
writefln!"%s, meanSessionDurationSeconds, %.3f"(report.siteName, report.meanSessionDurationSeconds);
writefln!"%s, meanEventsPerSession, %.3f"(report.siteName, report.meanEventsPerSession);
}
}
}

3
source/report/package.d Normal file
View File

@ -0,0 +1,3 @@
module report;
public import report.gen;

4
source/server.d
View File

@ -2,7 +2,7 @@ module server;
import std.file;
import handy_httpd;
import handy_httpd.handlers.path_delegating_handler;
import handy_httpd.handlers.path_handler;
import d_properties;
import slf4d;
@ -22,7 +22,7 @@ void startServer() {
*/
private HttpRequestHandler prepareHandler(Properties props) {
import live_tracker;
PathDelegatingHandler pathHandler = new PathDelegatingHandler();
PathHandler pathHandler = new PathHandler();
pathHandler.addMapping(
Method.GET,
"/ws",