module analytics.balances; import handy_http_primitives : Optional, mapIfPresent; import std.datetime; import std.stdio; import std.path; import std.file; import std.algorithm; import std.array; import std.conv; import slf4d; import asdf; import profile.data; import profile.model; import account.data; import account.model; import account.service; import analytics.util; import transaction.model; import transaction.dto; import util.pagination; import util.money; import util.data; import analytics.data; struct TimeSeriesPoint { /// The millisecond UTC timestamp. ulong x; /// The value at this timestamp. long y; } struct AccountBalanceData { ulong accountId; TimeSeriesPoint[] data; } struct BalanceTimeSeriesAnalytics { AccountBalanceData[] accounts; TimeSeriesPoint[] totals; } /** * Computes a time series tracking the balance of each account (and total of * all accounts) over the given time range. * Params: * ds = The profile data source. * currency = The currency to report data for. * timeRange = The time range to generate the time series for. * Returns: An analytics response containing a "totals" time series, as well * as a time series for each known account in the given time range. */ BalanceTimeSeriesAnalytics computeBalanceTimeSeriesV2( ProfileDataSource ds, in Currency currency, in TimeRange timeRange ) { SysTime[] timestamps = generateTimeSeriesTimestamps(days(1), timeRange); AnalyticsRepository repo = ds.getAnalyticsRepository(); JournalEntryStub[] journalEntries = repo.getJournalEntries(currency, timeRange); BalanceRecordStub[] balanceRecords = repo.getBalanceRecords(currency, timeRange); auto accountIds = balanceRecords.map!(br => br.accountId).uniq.array.sort; BalanceTimeSeriesAnalytics result; foreach (timestamp; timestamps) { long totalBalance = 0; foreach (accountId; accountIds) { Optional!long optionalBalance = deriveBalance(accountId, journalEntries, balanceRecords, timestamp); if (!optionalBalance.isNull) { TimeSeriesPoint p = TimeSeriesPoint(timestamp.toUnixMillis(), optionalBalance.value); bool isAccountDataPresent = false; foreach (ref accountData; result.accounts) { if (accountData.accountId == accountId) { accountData.data ~= p; isAccountDataPresent = true; break; } } if (!isAccountDataPresent) { result.accounts ~= AccountBalanceData(accountId, [p]); } totalBalance += optionalBalance.value; } } result.totals ~= TimeSeriesPoint(timestamp.toUnixMillis(), totalBalance); } return result; } private Optional!long deriveBalance( ulong accountId, in JournalEntryStub[] journalEntries, in BalanceRecordStub[] balanceRecords, in SysTime timestamp ) { import core.time : abs; Optional!BalanceRecordStub nearestBalanceRecord; foreach (br; balanceRecords) { if (br.accountId == accountId) { if (nearestBalanceRecord.isNull) { nearestBalanceRecord = Optional!(BalanceRecordStub).of(br); } else { Duration currentDiff = abs(nearestBalanceRecord.value.timestamp - timestamp); Duration newDiff = abs(br.timestamp - timestamp); if (newDiff < currentDiff) { nearestBalanceRecord = Optional!(BalanceRecordStub).of(br); } } } } if (nearestBalanceRecord.isNull) { return Optional!(long).empty(); } if (timestamp == nearestBalanceRecord.value.timestamp) { return Optional!(long).of(nearestBalanceRecord.value.value); } // Now that we have a balance record, work our way towards the desired // timestamp, applying journal entry changes. SysTime startTimestamp; SysTime endTimestamp; long balance = nearestBalanceRecord.value.value; if (timestamp > nearestBalanceRecord.value.timestamp) { startTimestamp = nearestBalanceRecord.value.timestamp; endTimestamp = timestamp; } else { startTimestamp = timestamp; endTimestamp = nearestBalanceRecord.value.timestamp; } auto relevantJournalEntries = journalEntries .filter!(je => ( je.accountId == accountId && je.timestamp >= startTimestamp && je.timestamp <= endTimestamp )); foreach (je; relevantJournalEntries) { long entryValue = je.amount; if (je.type == AccountJournalEntryType.CREDIT) { entryValue *= -1; } if (!je.accountType.debitsPositive) { entryValue *= -1; } if (je.timestamp < nearestBalanceRecord.value.timestamp) { entryValue *= -1; } balance += entryValue; } return Optional!(long).of(balance); } alias CurrencyGroupedTimeSeries = TimeSeriesPoint[][string]; struct CategorySpendData { ulong categoryId; string categoryName; string categoryColor; CurrencyGroupedTimeSeries dataByCurrency; } struct CategorySpendTimeSeriesAnalytics { CategorySpendData[] categories; } void computeCategorySpendTimeSeries(Profile profile, ProfileRepository profileRepo) { ProfileDataSource ds = profileRepo.getDataSource(profile); TransactionCategory[] rootCategories = ds.getTransactionCategoryRepository() .findAllByParentId(Optional!ulong.empty); CategorySpendTimeSeriesAnalytics data; PageRequest pr = PageRequest(1, 100, [Sort("txn.timestamp", SortDir.ASC)]); Page!TransactionsListItem page = ds.getTransactionRepository().findAll(pr); while (page.items.length > 0) { foreach (TransactionsListItem txn; page.items) { if (!isTypicalSpendingTransaction(txn)) continue; ulong categoryId = txn.category.mapIfPresent!(c => c.id).orElse(0); TimeSeriesPoint dataPoint = TimeSeriesPoint( SysTime.fromISOExtString(txn.timestamp).toUnixMillis(), txn.amount ); bool foundCategory = false; foreach (ref cd; data.categories) { if (cd.categoryId == categoryId) { cd.dataByCurrency[txn.currency.code.idup] ~= dataPoint; foundCategory = true; break; } } if (!foundCategory) { CategorySpendData cd; cd.categoryId = categoryId; cd.categoryName = txn.category.mapIfPresent!(c => c.name).orElse("None"); cd.categoryColor = txn.category.mapIfPresent!(c => c.color).orElse("FFFFFF"); cd.dataByCurrency[txn.currency.code.idup] ~= dataPoint; data.categories ~= cd; } } page = ds.getTransactionRepository().findAll(page.pageRequest.next()); } ds.doTransaction(() { ds.getPropertiesRepository().deleteProperty("analytics.categorySpendTimeSeries"); ds.getPropertiesRepository().setProperty( "analytics.categorySpendTimeSeries", serializeToJsonPretty(data) ); }); infoF!"Computed category spend analytics for user %s, profile %s."( profile.username, profile.name ); } private bool isTypicalSpendingTransaction(in TransactionsListItem txn) { return !txn.creditedAccount.isNull && txn.debitedAccount.isNull && !txn.internalTransfer; }