diff --git a/finnow-api/source/analytics/api.d b/finnow-api/source/analytics/api.d index 5fadd5e..8a46ff1 100644 --- a/finnow-api/source/analytics/api.d +++ b/finnow-api/source/analytics/api.d @@ -1,17 +1,16 @@ module analytics.api; import handy_http_primitives; -import handy_http_handlers.path_handler : PathMapping; +import handy_http_handlers.path_handler : PathMapping, GetMapping; +import handy_http_data : writeJsonBody; +import std.datetime; import profile.data; import profile.service; import profile.api : PROFILE_PATH; - -@PathMapping(HttpMethod.GET, PROFILE_PATH ~ "/analytics/balance-time-series") -void handleGetBalanceTimeSeries(ref ServerHttpRequest request, ref ServerHttpResponse response) { - auto ds = getProfileDataSource(request); - serveJsonFromProperty(response, ds, "analytics.balanceTimeSeries"); -} +import analytics.balances; +import util.money; +import util.data; @PathMapping(HttpMethod.GET, PROFILE_PATH ~ "/analytics/category-spend-time-series") void handleGetCategorySpendTimeSeries(ref ServerHttpRequest request, ref ServerHttpResponse response) { @@ -19,6 +18,15 @@ void handleGetCategorySpendTimeSeries(ref ServerHttpRequest request, ref ServerH serveJsonFromProperty(response, ds, "analytics.categorySpendTimeSeries"); } +@GetMapping(PROFILE_PATH ~ "/analytics/balance-time-series") +void handleGetBalanceTimeSeriesV2(ref ServerHttpRequest request, ref ServerHttpResponse response) { + auto ds = getProfileDataSource(request); + Currency currency = Currency.ofCode(request.getParamAs!string("currency", Currencies.USD.code)); + TimeRange timeRange = TimeRange(Optional!(SysTime).empty(), Optional!(SysTime).empty()); + auto data = computeBalanceTimeSeriesV2(ds, currency, timeRange); + writeJsonBody(response, data); +} + /** * Helper method to serve JSON analytics data to a client by fetching it * directly from the user's profile properties table and writing it to the diff --git a/finnow-api/source/analytics/balances.d b/finnow-api/source/analytics/balances.d index 1baa62b..46729ed 100644 --- a/finnow-api/source/analytics/balances.d +++ b/finnow-api/source/analytics/balances.d @@ -5,6 +5,9 @@ import std.datetime; import std.stdio; import std.path; import std.file; +import std.algorithm; +import std.array; +import std.conv; import slf4d; import asdf; @@ -17,6 +20,9 @@ import analytics.util; import transaction.model; import transaction.dto; import util.pagination; +import util.money; +import util.data; +import analytics.data; struct TimeSeriesPoint { /// The millisecond UTC timestamp. @@ -25,64 +31,130 @@ struct TimeSeriesPoint { long y; } -alias CurrencyGroupedTimeSeries = TimeSeriesPoint[][string]; - struct AccountBalanceData { ulong accountId; - string currencyCode; TimeSeriesPoint[] data; } struct BalanceTimeSeriesAnalytics { AccountBalanceData[] accounts; - CurrencyGroupedTimeSeries totals; + TimeSeriesPoint[] totals; } -void computeAccountBalanceTimeSeries(Profile profile, ProfileRepository profileRepo) { - ProfileDataSource ds = profileRepo.getDataSource(profile); - Account[] accounts = ds.getAccountRepository().findAll(); +/** + * Computes a time series tracking the balance of each account (and total of + * all accounts) over the given time range. + * Params: + * ds = The profile data source. + * currency = The currency to report data for. + * timeRange = The time range to generate the time series for. + * Returns: An analytics response containing a "totals" time series, as well + * as a time series for each known account in the given time range. + */ +BalanceTimeSeriesAnalytics computeBalanceTimeSeriesV2( + ProfileDataSource ds, + in Currency currency, + in TimeRange timeRange +) { + SysTime[] timestamps = generateTimeSeriesTimestamps(days(1), timeRange); + AnalyticsRepository repo = ds.getAnalyticsRepository(); + JournalEntryStub[] journalEntries = repo.getJournalEntries(currency, timeRange); + BalanceRecordStub[] balanceRecords = repo.getBalanceRecords(currency, timeRange); + auto accountIds = balanceRecords.map!(br => br.accountId).uniq.array.sort; - // Initialize the data structure that'll store the analytics info. - BalanceTimeSeriesAnalytics data; - foreach (account; accounts) { - AccountBalanceData accountData; - accountData.accountId = account.id; - accountData.currencyCode = account.currency.code.idup; - data.accounts ~= accountData; - } + BalanceTimeSeriesAnalytics result; - foreach (timestamp; generateTimeSeriesTimestamps(days(1), 365)) { - // Compute the balance of each account at this timestamp. - foreach (idx, account; accounts) { - auto balance = getBalance(ds, account.id, timestamp); - if (!balance.isNull) { - data.accounts[idx].data ~= TimeSeriesPoint( - timestamp.toUnixMillis(), - balance.value - ); + foreach (timestamp; timestamps) { + long totalBalance = 0; + foreach (accountId; accountIds) { + Optional!long optionalBalance = deriveBalance(accountId, journalEntries, balanceRecords, timestamp); + if (!optionalBalance.isNull) { + TimeSeriesPoint p = TimeSeriesPoint(timestamp.toUnixMillis(), optionalBalance.value); + bool isAccountDataPresent = false; + foreach (ref accountData; result.accounts) { + if (accountData.accountId == accountId) { + accountData.data ~= p; + isAccountDataPresent = true; + break; + } + } + if (!isAccountDataPresent) { + result.accounts ~= AccountBalanceData(accountId, [p]); + } + totalBalance += optionalBalance.value; } } - - // Compute total balances for this timestamp. - auto totalBalances = getTotalBalanceForAllAccounts(ds, timestamp); - foreach (CurrencyBalance bal; totalBalances) { - data.totals[bal.currency.code.idup] ~= TimeSeriesPoint(timestamp.toUnixMillis(), bal.balance); - } + result.totals ~= TimeSeriesPoint(timestamp.toUnixMillis(), totalBalance); } - ds.doTransaction(() { - ds.getPropertiesRepository().deleteProperty("analytics.balanceTimeSeries"); - ds.getPropertiesRepository().setProperty( - "analytics.balanceTimeSeries", - serializeToJsonPretty(data) - ); - }); - infoF!"Computed account balance analytics for user %s, profile %s."( - profile.username, - profile.name - ); + return result; } +private Optional!long deriveBalance( + ulong accountId, + in JournalEntryStub[] journalEntries, + in BalanceRecordStub[] balanceRecords, + in SysTime timestamp +) { + import core.time : abs; + Optional!BalanceRecordStub nearestBalanceRecord; + foreach (br; balanceRecords) { + if (br.accountId == accountId) { + if (nearestBalanceRecord.isNull) { + nearestBalanceRecord = Optional!(BalanceRecordStub).of(br); + } else { + Duration currentDiff = abs(nearestBalanceRecord.value.timestamp - timestamp); + Duration newDiff = abs(br.timestamp - timestamp); + if (newDiff < currentDiff) { + nearestBalanceRecord = Optional!(BalanceRecordStub).of(br); + } + } + } + } + if (nearestBalanceRecord.isNull) { + return Optional!(long).empty(); + } + if (timestamp == nearestBalanceRecord.value.timestamp) { + return Optional!(long).of(nearestBalanceRecord.value.value); + } + + // Now that we have a balance record, work our way towards the desired + // timestamp, applying journal entry changes. + SysTime startTimestamp; + SysTime endTimestamp; + long balance = nearestBalanceRecord.value.value; + if (timestamp > nearestBalanceRecord.value.timestamp) { + startTimestamp = nearestBalanceRecord.value.timestamp; + endTimestamp = timestamp; + } else { + startTimestamp = timestamp; + endTimestamp = nearestBalanceRecord.value.timestamp; + } + auto relevantJournalEntries = journalEntries + .filter!(je => ( + je.accountId == accountId && + je.timestamp >= startTimestamp && + je.timestamp <= endTimestamp + )); + foreach (je; relevantJournalEntries) { + long entryValue = je.amount; + if (je.type == AccountJournalEntryType.CREDIT) { + entryValue *= -1; + } + if (!je.accountType.debitsPositive) { + entryValue *= -1; + } + if (je.timestamp < nearestBalanceRecord.value.timestamp) { + entryValue *= -1; + } + balance += entryValue; + } + return Optional!(long).of(balance); +} + + +alias CurrencyGroupedTimeSeries = TimeSeriesPoint[][string]; + struct CategorySpendData { ulong categoryId; string categoryName; diff --git a/finnow-api/source/analytics/data.d b/finnow-api/source/analytics/data.d new file mode 100644 index 0000000..d249273 --- /dev/null +++ b/finnow-api/source/analytics/data.d @@ -0,0 +1,37 @@ +module analytics.data; + +import std.datetime; + +import util.money; +import util.data; +import analytics.balances; +import account.model; + +struct JournalEntryStub { + SysTime timestamp; + ulong accountId; + AccountType accountType; + ulong amount; + AccountJournalEntryType type; +} + +struct BalanceRecordStub { + SysTime timestamp; + ulong accountId; + long value; +} + +/** + * Repository that provides various functions for fetching data that's used in + * the calculation of analytics, separate from usual app functionality. + */ +interface AnalyticsRepository { + JournalEntryStub[] getJournalEntries( + in Currency currency, + in TimeRange timeRange + ); + BalanceRecordStub[] getBalanceRecords( + in Currency currency, + in TimeRange timeRange + ); +} \ No newline at end of file diff --git a/finnow-api/source/analytics/data_impl_sqlite.d b/finnow-api/source/analytics/data_impl_sqlite.d new file mode 100644 index 0000000..8ca59e1 --- /dev/null +++ b/finnow-api/source/analytics/data_impl_sqlite.d @@ -0,0 +1,104 @@ +module analytics.data_impl_sqlite; + +import d2sqlite3; +import std.array; +import std.datetime; + +import util.money; +import util.data; +import util.sqlite; +import account.model : AccountJournalEntryType, AccountType; +import analytics.balances; +import analytics.data; + +class SqliteAnalyticsRepository : AnalyticsRepository { + private Database db; + this(Database db) { + this.db = db; + } + + JournalEntryStub[] getJournalEntries( + in Currency currency, + in TimeRange timeRange + ) { + QueryBuilder qb = QueryBuilder("account_journal_entry je") + .select("je.timestamp,je.account_id,je.amount,je.type,account.type") + .join("LEFT JOIN account ON account.id = je.account_id") + .where("UPPER(je.currency) = ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, currency.codeString); + }); + if (timeRange.fromTime) { + qb.where("je.timestamp >= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + if (timeRange.toTime) { + qb.where("je.timestamp <= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + string query = qb.build() ~ " ORDER BY je.timestamp"; + Statement stmt = db.prepare(query); + qb.applyArgBindings(stmt); + ResultRange result = stmt.execute(); + Appender!(JournalEntryStub[]) app; + foreach (row; result) { + auto journalEntryTypeStr = row.peek!(string, PeekMode.slice)(3); + AccountJournalEntryType type; + if (journalEntryTypeStr == AccountJournalEntryType.CREDIT) { + type = AccountJournalEntryType.CREDIT; + } else { + type = AccountJournalEntryType.DEBIT; + } + app ~= JournalEntryStub( + SysTime.fromISOExtString(row.peek!(string, PeekMode.slice)(0)), + row.peek!ulong(1), + AccountType.fromId(row.peek!(string, PeekMode.slice)(4)), + row.peek!ulong(2), + type + ); + } + return app[]; + } + + BalanceRecordStub[] getBalanceRecords( + in Currency currency, + in TimeRange timeRange + ) { + QueryBuilder qb = QueryBuilder("account_value_record") + .select("timestamp,account_id,value,type") + .where("UPPER(currency) = ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, currency.codeString); + }) + .where("UPPER(type) = 'BALANCE'"); + if (timeRange.fromTime) { + qb.where("timestamp >= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + if (timeRange.toTime) { + qb.where("timestamp <= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + string query = qb.build() ~ " ORDER BY timestamp"; + Statement stmt = db.prepare(query); + qb.applyArgBindings(stmt); + ResultRange result = stmt.execute(); + Appender!(BalanceRecordStub[]) app; + foreach (row; result) { + app ~= BalanceRecordStub( + SysTime.fromISOExtString(row.peek!(string, PeekMode.slice)(0)), + row.peek!ulong(1), + row.peek!long(2) + ); + } + return app[]; + } +} \ No newline at end of file diff --git a/finnow-api/source/analytics/util.d b/finnow-api/source/analytics/util.d index e104825..8a2ead1 100644 --- a/finnow-api/source/analytics/util.d +++ b/finnow-api/source/analytics/util.d @@ -1,6 +1,8 @@ module analytics.util; import std.datetime; +import std.array; +import util.data; SysTime[] generateTimeSeriesTimestamps(Duration intervalSize, int intervalCount) { const SysTime now = Clock.currTime(UTC()); @@ -18,6 +20,34 @@ SysTime[] generateTimeSeriesTimestamps(Duration intervalSize, int intervalCount) return timestamps; } +SysTime[] generateTimeSeriesTimestamps(Duration intervalSize, in TimeRange timeRange) { + SysTime endOfRange; + SysTime startOfRange; + if (timeRange.toTime.isNull) { + endOfRange = Clock.currTime(UTC()); + } else { + endOfRange = timeRange.toTime.value; + } + if (timeRange.fromTime.isNull) { + startOfRange = endOfRange; + startOfRange.add!"years"(-5); + } else { + startOfRange = timeRange.fromTime.value; + } + import std.stdio; + writefln!"start = %s, end = %s"(startOfRange, endOfRange); + + Appender!(SysTime[]) app; + app ~= startOfRange; + + SysTime timestamp = startOfRange; + while (timestamp + intervalSize <= endOfRange) { + timestamp += intervalSize; + app ~= timestamp; + } + return app[]; +} + ulong toUnixMillis(in SysTime ts) { return (ts - SysTime(unixTimeToStdTime(0))).total!"msecs"; } diff --git a/finnow-api/source/profile/data.d b/finnow-api/source/profile/data.d index 905b956..6186fc0 100644 --- a/finnow-api/source/profile/data.d +++ b/finnow-api/source/profile/data.d @@ -40,6 +40,7 @@ interface ProfileDataSource { import account.data; import transaction.data; import attachment.data; + import analytics.data; PropertiesRepository getPropertiesRepository(); AttachmentRepository getAttachmentRepository(); @@ -53,6 +54,8 @@ interface ProfileDataSource { TransactionTagRepository getTransactionTagRepository(); TransactionRepository getTransactionRepository(); + AnalyticsRepository getAnalyticsRepository(); + void doTransaction(void delegate () dg); } @@ -61,6 +64,7 @@ version(unittest) { import account.data; import transaction.data; import attachment.data; + import analytics.data; PropertiesRepository getPropertiesRepository() { throw new Exception("Not implemented"); @@ -89,6 +93,9 @@ version(unittest) { TransactionRepository getTransactionRepository() { throw new Exception("Not implemented"); } + AnalyticsRepository getAnalyticsRepository() { + throw new Exception("Not implemented"); + } void doTransaction(void delegate () dg) { throw new Exception("Not implemented"); } diff --git a/finnow-api/source/profile/data_impl_sqlite.d b/finnow-api/source/profile/data_impl_sqlite.d index 65e7ac0..03a367a 100644 --- a/finnow-api/source/profile/data_impl_sqlite.d +++ b/finnow-api/source/profile/data_impl_sqlite.d @@ -181,6 +181,8 @@ class SqliteProfileDataSource : ProfileDataSource { import transaction.data_impl_sqlite; import attachment.data; import attachment.data_impl_sqlite; + import analytics.data; + import analytics.data_impl_sqlite; private const string dbPath; Database db; @@ -195,6 +197,7 @@ class SqliteProfileDataSource : ProfileDataSource { TransactionCategoryRepository transactionCategoryRepo; TransactionTagRepository transactionTagRepo; TransactionRepository transactionRepo; + AnalyticsRepository analyticsRepo; this(string path) { this.dbPath = path; @@ -276,6 +279,13 @@ class SqliteProfileDataSource : ProfileDataSource { return transactionRepo; } + AnalyticsRepository getAnalyticsRepository() { + if (analyticsRepo is null) { + analyticsRepo = new SqliteAnalyticsRepository(db); + } + return analyticsRepo; + } + void doTransaction(void delegate () dg) { util.sqlite.doTransaction(db, dg); } diff --git a/finnow-api/source/scheduled_jobs.d b/finnow-api/source/scheduled_jobs.d index 44a01b5..0bb6bc3 100644 --- a/finnow-api/source/scheduled_jobs.d +++ b/finnow-api/source/scheduled_jobs.d @@ -15,7 +15,6 @@ void startScheduledJobs() { JobScheduler jobScheduler = new TaskPoolScheduler(); jobScheduler.addJob(() { info("Computing account balance time series analytics for all users..."); - doForAllUserProfiles(&computeAccountBalanceTimeSeries); doForAllUserProfiles(&computeCategorySpendTimeSeries); info("Done computing analytics!"); }, analyticsSchedule); diff --git a/finnow-api/source/util/data.d b/finnow-api/source/util/data.d index b28a1b0..b5ed6a3 100644 --- a/finnow-api/source/util/data.d +++ b/finnow-api/source/util/data.d @@ -3,6 +3,7 @@ module util.data; import handy_http_primitives; import handy_http_data.multipart; import std.typecons; +import std.datetime; Optional!T toOptional(T)(Nullable!T value) { if (value.isNull) { @@ -125,3 +126,13 @@ private MultipartFile parseMultipartFile(in MultipartElement e) { cast(ubyte[]) e.content ); } + +/** + * Struct representing a user-provided time range with optional "from" and "to" + * timestamps. If both are empty, it is assumed that the user is requesting all + * data. + */ +struct TimeRange { + Optional!SysTime fromTime; + Optional!SysTime toTime; +} diff --git a/finnow-api/source/util/money.d b/finnow-api/source/util/money.d index f45bf5e..2e16695 100644 --- a/finnow-api/source/util/money.d +++ b/finnow-api/source/util/money.d @@ -27,6 +27,11 @@ struct Currency { } throw new Exception("Unknown currency code: " ~ code); } + + string codeString() const { + import std.conv : to; + return code.to!string; + } } /**