Added improved reporting and utils

deploy.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "Building sitestat"
+dub clean
+dub build --build=release --compiler=/opt/ldc2/ldc2-1.33.0-linux-x86_64/bin/ldc2
+
+ssh -f root@andrewlalis.com 'systemctl stop sitestat.service'
+scp sitestat root@andrewlalis.com:/opt/sitestat/
+ssh -f root@andrewlalis.com 'systemctl start sitestat.service'

dub.json

@@ -5,7 +5,7 @@
 	"copyright": "Copyright © 2023, Andrew Lalis",
 	"dependencies": {
 		"d-properties": "~>1.0.5",
-		"handy-httpd": "~>7.11.0",
+		"handy-httpd": "~>7.11.1",
 		"d2sqlite3": "~>1.0.0"
 	},
 	"description": "A simple webserver and script for tracking basic, non-intrusive site statistics.",

dub.selections.json

@@ -3,7 +3,7 @@
 	"versions": {
 		"d-properties": "1.0.5",
 		"d2sqlite3": "1.0.0",
-		"handy-httpd": "7.11.0",
+		"handy-httpd": "7.11.1",
 		"httparsed": "1.2.1",
 		"slf4d": "2.4.3",
 		"streams": "3.5.0"

sitestat.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=sitestat
+After=network.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/opt/sitestat
+ExecStart=/opt/sitestat/sitestat
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

source/app.d

@@ -2,6 +2,13 @@ import server : startServer;
 import report : makeReport;
 
 void main(string[] args) {
+	import slf4d;
+	import slf4d.default_provider;
+	auto provider = new shared DefaultProvider(false, Levels.INFO);
+	provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN);
+	// provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG);
+	configureLoggingProvider(provider);
+
 	if (args.length <= 1) {
 		startServer();
 	} else if (args[1] == "report") {

source/data.d

@@ -1,8 +1,8 @@
 module data;
 
+import utils;
 import std.file;
 import std.datetime;
-import std.format;
 import d2sqlite3;
 
 static immutable FS_ORIGIN = "_LOCAL_FILESYSTEM_";
@@ -30,8 +30,8 @@ void storeSession(StoredSession s) {
         "(start_timestamp, end_timestamp, url, full_url, user_agent, event_count) " ~
         "VALUES (?, ?, ?, ?, ?, ?)"
     );
-    stmt.bind(1, formatTimestamp(s.startTimestamp));
+    stmt.bind(1, formatSqliteTimestamp(s.startTimestamp));
-    stmt.bind(2, formatTimestamp(s.endTimestamp));
+    stmt.bind(2, formatSqliteTimestamp(s.endTimestamp));
     stmt.bind(3, s.url);
     stmt.bind(4, s.fullUrl);
     stmt.bind(5, s.userAgent);
@@ -63,62 +63,3 @@ private void initDb(string path) {
 SQL"
     );
 }
-
-ulong countSessions(string origin) {
-    Database db = getOrCreateDatabase(origin);
-    return db.execute("SELECT COUNT(id) FROM session;").oneValue!ulong;
-}
-
-private string formatTimestamp(SysTime t) {
-    return format!"%04d-%02d-%02d %02d:%02d:%02d"(
-        t.year, t.month, t.day,
-        t.hour, t.minute, t.second
-    );
-}
-
-string extractOrigin(string url) {
-    import std.algorithm : countUntil, startsWith;
-    ptrdiff_t idx = countUntil(url, "://");
-    if (idx == -1) return null;
-    string origin = url[idx + 3 .. $];
-    ptrdiff_t trailingSlashIdx = countUntil(origin, "/");
-    if (trailingSlashIdx != -1) {
-        origin = origin[0 .. trailingSlashIdx];
-    }
-    if (startsWith(origin, "www.")) {
-        origin = origin[4 .. $];
-    }
-    return origin;
-}
-
-unittest {
-    assert(extractOrigin("https://www.google.com/search") == "google.com");
-    assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com");
-}
-
-string dbPath(string origin) {
-    return "sitestat-db_" ~ origin ~ ".sqlite";
-}
-
-string originFromDbPath(string path) {
-    import std.algorithm : countUntil;
-    ptrdiff_t idx = countUntil(path, "sitestat-db_");
-    if (idx == -1) return null;
-    return path[(idx + 12)..$-7];
-}
-
-unittest {
-    assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_");
-}
-
-string[] listAllOrigins(string dir = ".") {
-    import std.array;
-    auto app = appender!(string[]);
-    foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) {
-        string origin = originFromDbPath(entry.name);
-        if (origin !is null) {
-            app ~= origin;
-        }
-    }
-    return app[];
-}

source/live_tracker.d

@@ -16,7 +16,7 @@ import core.sync.rwmutex;
  */
 class LiveTracker : WebSocketMessageHandler {
     private StatSession[UUID] sessions;
-    static immutable uint DEFAULT_MIN_SESSION_DURATION = 1000;
+    static immutable uint DEFAULT_MIN_SESSION_DURATION = 2000;
     private immutable uint minSessionDurationMillis;
     private TaskPool sessionPersistencePool;
     private ReadWriteMutex sessionsMutex;
@@ -35,7 +35,7 @@ class LiveTracker : WebSocketMessageHandler {
                 Clock.currTime(UTC())
             );
         }
-        infoF!"Started tracking session %s"(conn.id);
+        infoF!"Started tracking session %s. Tracking %d sessions right now."(conn.id, sessions.length);
     }
 
     override void onTextMessage(WebSocketTextMessage msg) {
@@ -70,17 +70,17 @@ class LiveTracker : WebSocketMessageHandler {
             Duration dur = endTimestamp - session.connectedAt;
             if (dur.total!"msecs" >= minSessionDurationMillis) {
                 infoF!"Session lasted %d seconds, %d events."(dur.total!"seconds", session.eventCount);
+                immutable storedSession = StoredSession(
+                    -1,
+                    session.connectedAt,
+                    endTimestamp,
+                    session.url,
+                    session.href,
+                    session.userAgent,
+                    session.eventCount
+                );
+                this.sessionPersistencePool.put(task!storeSession(storedSession));
             }
-            immutable storedSession = StoredSession(
-                -1,
-                session.connectedAt,
-                endTimestamp,
-                session.url,
-                session.href,
-                session.userAgent,
-                session.eventCount
-            );
-            this.sessionPersistencePool.put(task!storeSession(storedSession));
         }
         synchronized(sessionsMutex.writer) {
             sessions.remove(conn.id);

source/report.d

@@ -1,17 +1,25 @@
 module report;
 
+import utils;
 import std.stdio;
 import std.datetime;
 import std.algorithm;
 import std.array;
+import d2sqlite3;
 import data;
+import std.json;
 
 void makeReport(string[] sites) {
     string[] sitesToReport = sites.length == 0 ? listAllOrigins() : sites;
+    if (sitesToReport.length == 0) {
+        writeln("No sites to report.");
+        return;
+    }
     writeln("Creating report for " ~ createSiteNamesSummary(sitesToReport) ~ ".");
     foreach (site; sitesToReport) {
         writeln("Site: " ~ site);
-        writefln!"\nTotal sessions recorded: %d"(countSessions(site));
+        SiteReport report = generateReport(site, Clock.currTime(UTC()) - hours(48), Clock.currTime(UTC()));
+        writeln(report.toJson().toPrettyString());
     }
 }
 
@@ -37,5 +45,55 @@ struct SiteReport {
     SysTime periodStart;
     SysTime periodEnd;
 
-    ulong totalVisitors;
+    ulong totalSessions;
+    double meanSessionDurationSeconds;
+    double meanEventsPerSession;
+
+    JSONValue toJson() const {
+        JSONValue obj = JSONValue(string[string].init);
+        obj.object["siteName"] = siteName;
+        obj.object["periodStart"] = periodStart.toISOExtString();
+        obj.object["periodEnd"] = periodEnd.toISOExtString();
+        obj.object["totalSessions"] = totalSessions;
+        obj.object["meanSessionDurationSeconds"] = meanSessionDurationSeconds;
+        obj.object["meanEventsPerSession"] = meanEventsPerSession;
+        return obj;
+    }
+}
+
+SiteReport generateReport(string site, SysTime periodStart, SysTime periodEnd) {
+    SiteReport report;
+    report.siteName = site;
+    report.periodStart = periodStart;
+    report.periodEnd = periodEnd;
+    immutable string TS_START = formatSqliteTimestamp(periodStart);
+    immutable string TS_END = formatSqliteTimestamp(periodEnd);
+    writefln!"TS_START = %s, TS_END = %s"(TS_START, TS_END);
+    Database db = getOrCreateDatabase(site);
+
+    report.totalSessions = db.execute(q"SQL
+        SELECT COUNT(id)
+        FROM session
+        WHERE start_timestamp >= ? AND end_timestamp <= ?
+SQL",
+        TS_START, TS_END
+    ).oneValue!ulong();
+
+    report.meanEventsPerSession = db.execute(q"SQL
+        SELECT AVG(event_count)
+        FROM session
+        WHERE start_timestamp >= ? AND end_timestamp <= ?
+SQL",
+        TS_START, TS_END
+    ).oneValue!double();
+
+    report.meanSessionDurationSeconds = db.execute(q"SQL
+        SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60)
+        FROM session
+        WHERE start_timestamp >= ? AND end_timestamp <= ?
+SQL",
+        TS_START, TS_END
+    ).oneValue!double();
+
+    return report;
 }

source/server.d

@@ -4,20 +4,15 @@ import std.file;
 import handy_httpd;
 import handy_httpd.handlers.path_delegating_handler;
 import d_properties;
+import slf4d;
 
 void startServer() {
 	Properties props;
 	if (exists("sitestat.properties")) {
 		props = Properties("sitestat.properties");
+	} else {
+		warn("No sitestat.properties file was found. Using defaults!");
 	}
-
-	import slf4d;
-	import slf4d.default_provider;
-	auto provider = new shared DefaultProvider(true, Levels.INFO);
-	provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN);
-	// provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG);
-	configureLoggingProvider(provider);
-	
 	new HttpServer(prepareHandler(props), prepareConfig(props)).start();
 }
 
@@ -58,5 +53,8 @@ private ServerConfig prepareConfig(Properties props) {
 	if (props.has("server.workers")) {
 		config.workerPoolSize = props.get!size_t("server.workers");
 	}
+	config.defaultHeaders["Access-Control-Allow-Origin"] = "*";
+	config.defaultHeaders["Access-Control-Allow-Methods"] = "*";
+	config.defaultHeaders["Access-Control-Allow-Headers"] = "*";
 	return config;
 }

source/utils.d

@@ -0,0 +1,59 @@
+module utils;
+
+import std.datetime : SysTime;
+
+string formatSqliteTimestamp(SysTime t) {
+    import std.format : format;
+    return format!"%04d-%02d-%02d %02d:%02d:%02d.%03d"(
+        t.year, t.month, t.day,
+        t.hour, t.minute, t.second, t.fracSecs.total!"msecs"
+    );
+}
+
+string extractOrigin(string url) {
+    import std.algorithm : countUntil, startsWith;
+    ptrdiff_t idx = countUntil(url, "://");
+    if (idx == -1) return null;
+    string origin = url[idx + 3 .. $];
+    ptrdiff_t trailingSlashIdx = countUntil(origin, "/");
+    if (trailingSlashIdx != -1) {
+        origin = origin[0 .. trailingSlashIdx];
+    }
+    if (startsWith(origin, "www.")) {
+        origin = origin[4 .. $];
+    }
+    return origin;
+}
+
+unittest {
+    assert(extractOrigin("https://www.google.com/search") == "google.com");
+    assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com");
+}
+
+string dbPath(string origin) {
+    return "sitestat-db_" ~ origin ~ ".sqlite";
+}
+
+string originFromDbPath(string path) {
+    import std.algorithm : countUntil;
+    ptrdiff_t idx = countUntil(path, "sitestat-db_");
+    if (idx == -1) return null;
+    return path[(idx + 12)..$-7];
+}
+
+unittest {
+    assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_");
+}
+
+string[] listAllOrigins(string dir = ".") {
+    import std.file;
+    import std.array;
+    auto app = appender!(string[]);
+    foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) {
+        string origin = originFromDbPath(entry.name);
+        if (origin !is null) {
+            app ~= origin;
+        }
+    }
+    return app[];
+}