andrew
/
sitestat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added improved reporting and utils

This commit is contained in:
Andrew Lalis 2023-10-05 12:35:52 -04:00
parent aabec581b2
commit 391fc48829
10 changed files with 171 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
deploy.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
echo "Building sitestat"
dub clean
dub build --build=release --compiler=/opt/ldc2/ldc2-1.33.0-linux-x86_64/bin/ldc2
ssh -f root@andrewlalis.com 'systemctl stop sitestat.service'
scp sitestat root@andrewlalis.com:/opt/sitestat/
ssh -f root@andrewlalis.com 'systemctl start sitestat.service'

2
dub.json
View File

@ -5,7 +5,7 @@
"copyright": "Copyright © 2023, Andrew Lalis",
"dependencies": {
"d-properties": "~>1.0.5",
"handy-httpd": "~>7.11.0",
"handy-httpd": "~>7.11.1",
"d2sqlite3": "~>1.0.0"
},
"description": "A simple webserver and script for tracking basic, non-intrusive site statistics.",

2
dub.selections.json
View File

@ -3,7 +3,7 @@
"versions": {
"d-properties": "1.0.5",
"d2sqlite3": "1.0.0",
"handy-httpd": "7.11.0",
"handy-httpd": "7.11.1",
"httparsed": "1.2.1",
"slf4d": "2.4.3",
"streams": "3.5.0"

13
sitestat.service Normal file
View File

@ -0,0 +1,13 @@
[Unit]
Description=sitestat
After=network.target
[Service]
Type=simple
User=root
WorkingDirectory=/opt/sitestat
ExecStart=/opt/sitestat/sitestat
Restart=always
[Install]
WantedBy=multi-user.target

7
source/app.d
View File

@ -2,6 +2,13 @@ import server : startServer;
import report : makeReport;
void main(string[] args) {
import slf4d;
import slf4d.default_provider;
auto provider = new shared DefaultProvider(false, Levels.INFO);
provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN);
// provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG);
configureLoggingProvider(provider);
if (args.length <= 1) {
startServer();
} else if (args[1] == "report") {

65
source/data.d
View File

@ -1,8 +1,8 @@
module data;
import utils;
import std.file;
import std.datetime;
import std.format;
import d2sqlite3;
static immutable FS_ORIGIN = "_LOCAL_FILESYSTEM_";
@ -30,8 +30,8 @@ void storeSession(StoredSession s) {
"(start_timestamp, end_timestamp, url, full_url, user_agent, event_count) " ~
"VALUES (?, ?, ?, ?, ?, ?)"
);
stmt.bind(1, formatTimestamp(s.startTimestamp));
stmt.bind(2, formatTimestamp(s.endTimestamp));
stmt.bind(1, formatSqliteTimestamp(s.startTimestamp));
stmt.bind(2, formatSqliteTimestamp(s.endTimestamp));
stmt.bind(3, s.url);
stmt.bind(4, s.fullUrl);
stmt.bind(5, s.userAgent);
@ -63,62 +63,3 @@ private void initDb(string path) {
SQL"
);
}
ulong countSessions(string origin) {
Database db = getOrCreateDatabase(origin);
return db.execute("SELECT COUNT(id) FROM session;").oneValue!ulong;
}
private string formatTimestamp(SysTime t) {
return format!"%04d-%02d-%02d %02d:%02d:%02d"(
t.year, t.month, t.day,
t.hour, t.minute, t.second
);
}
string extractOrigin(string url) {
import std.algorithm : countUntil, startsWith;
ptrdiff_t idx = countUntil(url, "://");
if (idx == -1) return null;
string origin = url[idx + 3 .. $];
ptrdiff_t trailingSlashIdx = countUntil(origin, "/");
if (trailingSlashIdx != -1) {
origin = origin[0 .. trailingSlashIdx];
}
if (startsWith(origin, "www.")) {
origin = origin[4 .. $];
}
return origin;
}
unittest {
assert(extractOrigin("https://www.google.com/search") == "google.com");
assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com");
}
string dbPath(string origin) {
return "sitestat-db_" ~ origin ~ ".sqlite";
}
string originFromDbPath(string path) {
import std.algorithm : countUntil;
ptrdiff_t idx = countUntil(path, "sitestat-db_");
if (idx == -1) return null;
return path[(idx + 12)..$-7];
}
unittest {
assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_");
}
string[] listAllOrigins(string dir = ".") {
import std.array;
auto app = appender!(string[]);
foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) {
string origin = originFromDbPath(entry.name);
if (origin !is null) {
app ~= origin;
}
}
return app[];
}

6
source/live_tracker.d
View File

@ -16,7 +16,7 @@ import core.sync.rwmutex;
*/
class LiveTracker : WebSocketMessageHandler {
private StatSession[UUID] sessions;
static immutable uint DEFAULT_MIN_SESSION_DURATION = 1000;
static immutable uint DEFAULT_MIN_SESSION_DURATION = 2000;
private immutable uint minSessionDurationMillis;
private TaskPool sessionPersistencePool;
private ReadWriteMutex sessionsMutex;
@ -35,7 +35,7 @@ class LiveTracker : WebSocketMessageHandler {
Clock.currTime(UTC())
);
}
infoF!"Started tracking session %s"(conn.id);
infoF!"Started tracking session %s. Tracking %d sessions right now."(conn.id, sessions.length);
}
override void onTextMessage(WebSocketTextMessage msg) {
@ -70,7 +70,6 @@ class LiveTracker : WebSocketMessageHandler {
Duration dur = endTimestamp - session.connectedAt;
if (dur.total!"msecs" >= minSessionDurationMillis) {
infoF!"Session lasted %d seconds, %d events."(dur.total!"seconds", session.eventCount);
}
immutable storedSession = StoredSession(
-1,
session.connectedAt,
@ -82,6 +81,7 @@ class LiveTracker : WebSocketMessageHandler {
);
this.sessionPersistencePool.put(task!storeSession(storedSession));
}
}
synchronized(sessionsMutex.writer) {
sessions.remove(conn.id);
}

62
source/report.d
View File

@ -1,17 +1,25 @@
module report;
import utils;
import std.stdio;
import std.datetime;
import std.algorithm;
import std.array;
import d2sqlite3;
import data;
import std.json;
void makeReport(string[] sites) {
string[] sitesToReport = sites.length == 0 ? listAllOrigins() : sites;
if (sitesToReport.length == 0) {
writeln("No sites to report.");
return;
}
writeln("Creating report for " ~ createSiteNamesSummary(sitesToReport) ~ ".");
foreach (site; sitesToReport) {
writeln("Site: " ~ site);
writefln!"\nTotal sessions recorded: %d"(countSessions(site));
SiteReport report = generateReport(site, Clock.currTime(UTC()) - hours(48), Clock.currTime(UTC()));
writeln(report.toJson().toPrettyString());
}
}
@ -37,5 +45,55 @@ struct SiteReport {
SysTime periodStart;
SysTime periodEnd;
ulong totalVisitors;
ulong totalSessions;
double meanSessionDurationSeconds;
double meanEventsPerSession;
JSONValue toJson() const {
JSONValue obj = JSONValue(string[string].init);
obj.object["siteName"] = siteName;
obj.object["periodStart"] = periodStart.toISOExtString();
obj.object["periodEnd"] = periodEnd.toISOExtString();
obj.object["totalSessions"] = totalSessions;
obj.object["meanSessionDurationSeconds"] = meanSessionDurationSeconds;
obj.object["meanEventsPerSession"] = meanEventsPerSession;
return obj;
}
}
SiteReport generateReport(string site, SysTime periodStart, SysTime periodEnd) {
SiteReport report;
report.siteName = site;
report.periodStart = periodStart;
report.periodEnd = periodEnd;
immutable string TS_START = formatSqliteTimestamp(periodStart);
immutable string TS_END = formatSqliteTimestamp(periodEnd);
writefln!"TS_START = %s, TS_END = %s"(TS_START, TS_END);
Database db = getOrCreateDatabase(site);
report.totalSessions = db.execute(q"SQL
SELECT COUNT(id)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!ulong();
report.meanEventsPerSession = db.execute(q"SQL
SELECT AVG(event_count)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
report.meanSessionDurationSeconds = db.execute(q"SQL
SELECT AVG((julianday(end_timestamp) - julianday(start_timestamp)) * 24 * 60 * 60)
FROM session
WHERE start_timestamp >= ? AND end_timestamp <= ?
SQL",
TS_START, TS_END
).oneValue!double();
return report;
}

14
source/server.d
View File

@ -4,20 +4,15 @@ import std.file;
import handy_httpd;
import handy_httpd.handlers.path_delegating_handler;
import d_properties;
import slf4d;
void startServer() {
Properties props;
if (exists("sitestat.properties")) {
props = Properties("sitestat.properties");
} else {
warn("No sitestat.properties file was found. Using defaults!");
}
import slf4d;
import slf4d.default_provider;
auto provider = new shared DefaultProvider(true, Levels.INFO);
provider.getLoggerFactory().setModuleLevelPrefix("handy_httpd", Levels.WARN);
// provider.getLoggerFactory().setModuleLevel("live_tracker", Levels.DEBUG);
configureLoggingProvider(provider);
new HttpServer(prepareHandler(props), prepareConfig(props)).start();
}
@ -58,5 +53,8 @@ private ServerConfig prepareConfig(Properties props) {
if (props.has("server.workers")) {
config.workerPoolSize = props.get!size_t("server.workers");
}
config.defaultHeaders["Access-Control-Allow-Origin"] = "*";
config.defaultHeaders["Access-Control-Allow-Methods"] = "*";
config.defaultHeaders["Access-Control-Allow-Headers"] = "*";
return config;
}

59
source/utils.d Normal file
View File

@ -0,0 +1,59 @@
module utils;
import std.datetime : SysTime;
string formatSqliteTimestamp(SysTime t) {
import std.format : format;
return format!"%04d-%02d-%02d %02d:%02d:%02d.%03d"(
t.year, t.month, t.day,
t.hour, t.minute, t.second, t.fracSecs.total!"msecs"
);
}
string extractOrigin(string url) {
import std.algorithm : countUntil, startsWith;
ptrdiff_t idx = countUntil(url, "://");
if (idx == -1) return null;
string origin = url[idx + 3 .. $];
ptrdiff_t trailingSlashIdx = countUntil(origin, "/");
if (trailingSlashIdx != -1) {
origin = origin[0 .. trailingSlashIdx];
}
if (startsWith(origin, "www.")) {
origin = origin[4 .. $];
}
return origin;
}
unittest {
assert(extractOrigin("https://www.google.com/search") == "google.com");
assert(extractOrigin("https://litelist.andrewlalis.com") == "litelist.andrewlalis.com");
}
string dbPath(string origin) {
return "sitestat-db_" ~ origin ~ ".sqlite";
}
string originFromDbPath(string path) {
import std.algorithm : countUntil;
ptrdiff_t idx = countUntil(path, "sitestat-db_");
if (idx == -1) return null;
return path[(idx + 12)..$-7];
}
unittest {
assert(originFromDbPath("sitestat-db__LOCAL_FILESYSTEM_.sqlite") == "_LOCAL_FILESYSTEM_");
}
string[] listAllOrigins(string dir = ".") {
import std.file;
import std.array;
auto app = appender!(string[]);
foreach (DirEntry entry; dirEntries(dir, SpanMode.shallow, false)) {
string origin = originFromDbPath(entry.name);
if (origin !is null) {
app ~= origin;
}
}
return app[];
}