diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..19f7706 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +echo "Building sitestat" +dub clean +dub build --build=release --compiler=/opt/ldc2/ldc2-1.33.0-linux-x86_64/bin/ldc2 + +ssh -f root@andrewlalis.com 'systemctl stop sitestat.service' +scp sitestat root@andrewlalis.com:/opt/sitestat/ +ssh -f root@andrewlalis.com 'systemctl start sitestat.service' diff --git a/dub.json b/dub.json index dca8c48..3d595cd 100644 --- a/dub.json +++ b/dub.json @@ -5,7 +5,7 @@ "copyright": "Copyright © 2023, Andrew Lalis", "dependencies": { "d-properties": "~>1.0.5", - "handy-httpd": "~>7.11.0", + "handy-httpd": "~>7.11.1", "d2sqlite3": "~>1.0.0" }, "description": "A simple webserver and script for tracking basic, non-intrusive site statistics.", diff --git a/dub.selections.json b/dub.selections.json index 81d34ff..c3df505 100644 --- a/dub.selections.json +++ b/dub.selections.json @@ -3,7 +3,7 @@ "versions": { "d-properties": "1.0.5", "d2sqlite3": "1.0.0", - "handy-httpd": "7.11.0", + "handy-httpd": "7.11.1", "httparsed": "1.2.1", "slf4d": "2.4.3", "streams": "3.5.0" diff --git a/sitestat.service b/sitestat.service new file mode 100644 index 0000000..1ea2606 --- /dev/null +++ b/sitestat.service @@ -0,0 +1,13 @@ +[Unit] +Description=sitestat +After=network.target + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/sitestat +ExecStart=/opt/sitestat/sitestat +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/source/app.d b/source/app.d index ebea65c..32d4385 100644 --- a/source/app.d +++ b/source/app.d @@ -2,6 +2,13 @@ import server : startServer; import report : makeReport; void main(string[] args) { + import slf4d; + import slf4d.default_provider; + auto provider = new shared DefaultProvider(false, Levels.INFO); + provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN); + // provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG); + configureLoggingProvider(provider); + if (args.length <= 1) { startServer(); } else if (args[1] == "report") { diff --git a/source/data.d b/source/data.d index 75831f1..faeb924 100644 --- a/source/data.d +++ b/source/data.d @@ -1,8 +1,8 @@ module data; +import utils; import std.file; import std.datetime; -import std.format; import d2sqlite3; static immutable FS_ORIGIN = "_LOCAL_FILESYSTEM_"; @@ -30,8 +30,8 @@ void storeSession(StoredSession s) { "(start_timestamp, end_timestamp, url, full_url, user_agent, event_count) " ~ "VALUES (?, ?, ?, ?, ?, ?)" ); - stmt.bind(1, formatTimestamp(s.startTimestamp)); - stmt.bind(2, formatTimestamp(s.endTimestamp)); + stmt.bind(1, formatSqliteTimestamp(s.startTimestamp)); + stmt.bind(2, formatSqliteTimestamp(s.endTimestamp)); stmt.bind(3, s.url); stmt.bind(4, s.fullUrl); stmt.bind(5, s.userAgent); @@ -63,62 +63,3 @@ private void initDb(string path) { SQL" ); } - -ulong countSessions(string origin) { - Database db = getOrCreateDatabase(origin); - return db.execute("SELECT COUNT(id) FROM session;").oneValue!ulong; -} - -private string formatTimestamp(SysTime t) { - return format!"%04d-%02d-%02d %02d:%02d:%02d"( - t.year, t.month, t.day, - t.hour, t.minute, t.second - ); -} - -string extractOrigin(string url) { - import std.algorithm : countUntil, startsWith; - ptrdiff_t idx = countUntil(url, "://"); - if (idx == -1) return null; - string origin = url[idx + 3 .. $]; - ptrdiff_t trailingSlashIdx = countUntil(origin, "/"); - if (trailingSlashIdx != -1) { - origin = origin[0 .. trailingSlashIdx]; - } - if (startsWith(origin, "www.")) { - origin = origin[4 .. $]; - } - return origin; -} - -unittest { - assert(extractOrigin("https://www.google.com/search") == "google.com"); - assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com"); -} - -string dbPath(string origin) { - return "sitestat-db_" ~ origin ~ ".sqlite"; -} - -string originFromDbPath(string path) { - import std.algorithm : countUntil; - ptrdiff_t idx = countUntil(path, "sitestat-db_"); - if (idx == -1) return null; - return path[(idx + 12)..$-7]; -} - -unittest { - assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_"); -} - -string[] listAllOrigins(string dir = ".") { - import std.array; - auto app = appender!(string[]); - foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) { - string origin = originFromDbPath(entry.name); - if (origin !is null) { - app ~= origin; - } - } - return app[]; -} diff --git a/source/live_tracker.d b/source/live_tracker.d index 73fc295..18ad6a7 100644 --- a/source/live_tracker.d +++ b/source/live_tracker.d @@ -16,7 +16,7 @@ import core.sync.rwmutex; */ class LiveTracker : WebSocketMessageHandler { private StatSession[UUID] sessions; - static immutable uint DEFAULT_MIN_SESSION_DURATION = 1000; + static immutable uint DEFAULT_MIN_SESSION_DURATION = 2000; private immutable uint minSessionDurationMillis; private TaskPool sessionPersistencePool; private ReadWriteMutex sessionsMutex; @@ -35,7 +35,7 @@ class LiveTracker : WebSocketMessageHandler { Clock.currTime(UTC()) ); } - infoF!"Started tracking session %s"(conn.id); + infoF!"Started tracking session %s. Tracking %d sessions right now."(conn.id, sessions.length); } override void onTextMessage(WebSocketTextMessage msg) { @@ -70,17 +70,17 @@ class LiveTracker : WebSocketMessageHandler { Duration dur = endTimestamp - session.connectedAt; if (dur.total!"msecs" >= minSessionDurationMillis) { infoF!"Session lasted %d seconds, %d events."(dur.total!"seconds", session.eventCount); + immutable storedSession = StoredSession( + -1, + session.connectedAt, + endTimestamp, + session.url, + session.href, + session.userAgent, + session.eventCount + ); + this.sessionPersistencePool.put(task!storeSession(storedSession)); } - immutable storedSession = StoredSession( - -1, - session.connectedAt, - endTimestamp, - session.url, - session.href, - session.userAgent, - session.eventCount - ); - this.sessionPersistencePool.put(task!storeSession(storedSession)); } synchronized(sessionsMutex.writer) { sessions.remove(conn.id); diff --git a/source/report.d b/source/report.d index bf426ba..d2a9482 100644 --- a/source/report.d +++ b/source/report.d @@ -1,17 +1,25 @@ module report; +import utils; import std.stdio; import std.datetime; import std.algorithm; import std.array; +import d2sqlite3; import data; +import std.json; void makeReport(string[] sites) { string[] sitesToReport = sites.length == 0 ? listAllOrigins() : sites; + if (sitesToReport.length == 0) { + writeln("No sites to report."); + return; + } writeln("Creating report for " ~ createSiteNamesSummary(sitesToReport) ~ "."); foreach (site; sitesToReport) { writeln("Site: " ~ site); - writefln!"\nTotal sessions recorded: %d"(countSessions(site)); + SiteReport report = generateReport(site, Clock.currTime(UTC()) - hours(48), Clock.currTime(UTC())); + writeln(report.toJson().toPrettyString()); } } @@ -37,5 +45,55 @@ struct SiteReport { SysTime periodStart; SysTime periodEnd; - ulong totalVisitors; + ulong totalSessions; + double meanSessionDurationSeconds; + double meanEventsPerSession; + + JSONValue toJson() const { + JSONValue obj = JSONValue(string[string].init); + obj.object["siteName"] = siteName; + obj.object["periodStart"] = periodStart.toISOExtString(); + obj.object["periodEnd"] = periodEnd.toISOExtString(); + obj.object["totalSessions"] = totalSessions; + obj.object["meanSessionDurationSeconds"] = meanSessionDurationSeconds; + obj.object["meanEventsPerSession"] = meanEventsPerSession; + return obj; + } +} + +SiteReport generateReport(string site, SysTime periodStart, SysTime periodEnd) { + SiteReport report; + report.siteName = site; + report.periodStart = periodStart; + report.periodEnd = periodEnd; + immutable string TS_START = formatSqliteTimestamp(periodStart); + immutable string TS_END = formatSqliteTimestamp(periodEnd); + writefln!"TS_START = %s, TS_END = %s"(TS_START, TS_END); + Database db = getOrCreateDatabase(site); + + report.totalSessions = db.execute(q"SQL + SELECT COUNT(id) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!ulong(); + + report.meanEventsPerSession = db.execute(q"SQL + SELECT AVG(event_count) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!double(); + + report.meanSessionDurationSeconds = db.execute(q"SQL + SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60) + FROM session + WHERE start_timestamp >= ? AND end_timestamp <= ? +SQL", + TS_START, TS_END + ).oneValue!double(); + + return report; } diff --git a/source/server.d b/source/server.d index 785960f..3ccd426 100644 --- a/source/server.d +++ b/source/server.d @@ -4,20 +4,15 @@ import std.file; import handy_httpd; import handy_httpd.handlers.path_delegating_handler; import d_properties; +import slf4d; void startServer() { Properties props; if (exists("sitestat.properties")) { props = Properties("sitestat.properties"); + } else { + warn("No sitestat.properties file was found. Using defaults!"); } - - import slf4d; - import slf4d.default_provider; - auto provider = new shared DefaultProvider(true, Levels.INFO); - provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN); - // provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG); - configureLoggingProvider(provider); - new HttpServer(prepareHandler(props), prepareConfig(props)).start(); } @@ -58,5 +53,8 @@ private ServerConfig prepareConfig(Properties props) { if (props.has("server.workers")) { config.workerPoolSize = props.get!size_t("server.workers"); } + config.defaultHeaders["Access-Control-Allow-Origin"] = "*"; + config.defaultHeaders["Access-Control-Allow-Methods"] = "*"; + config.defaultHeaders["Access-Control-Allow-Headers"] = "*"; return config; } \ No newline at end of file diff --git a/source/utils.d b/source/utils.d new file mode 100644 index 0000000..1bfbe80 --- /dev/null +++ b/source/utils.d @@ -0,0 +1,59 @@ +module utils; + +import std.datetime : SysTime; + +string formatSqliteTimestamp(SysTime t) { + import std.format : format; + return format!"%04d-%02d-%02d %02d:%02d:%02d.%03d"( + t.year, t.month, t.day, + t.hour, t.minute, t.second, t.fracSecs.total!"msecs" + ); +} + +string extractOrigin(string url) { + import std.algorithm : countUntil, startsWith; + ptrdiff_t idx = countUntil(url, "://"); + if (idx == -1) return null; + string origin = url[idx + 3 .. $]; + ptrdiff_t trailingSlashIdx = countUntil(origin, "/"); + if (trailingSlashIdx != -1) { + origin = origin[0 .. trailingSlashIdx]; + } + if (startsWith(origin, "www.")) { + origin = origin[4 .. $]; + } + return origin; +} + +unittest { + assert(extractOrigin("https://www.google.com/search") == "google.com"); + assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com"); +} + +string dbPath(string origin) { + return "sitestat-db_" ~ origin ~ ".sqlite"; +} + +string originFromDbPath(string path) { + import std.algorithm : countUntil; + ptrdiff_t idx = countUntil(path, "sitestat-db_"); + if (idx == -1) return null; + return path[(idx + 12)..$-7]; +} + +unittest { + assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_"); +} + +string[] listAllOrigins(string dir = ".") { + import std.file; + import std.array; + auto app = appender!(string[]); + foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) { + string origin = originFromDbPath(entry.name); + if (origin !is null) { + app ~= origin; + } + } + return app[]; +} \ No newline at end of file