diff --git a/finnow-api/source/analytics/api.d b/finnow-api/source/analytics/api.d index 8a46ff1..35687da 100644 --- a/finnow-api/source/analytics/api.d +++ b/finnow-api/source/analytics/api.d @@ -8,7 +8,7 @@ import std.datetime; import profile.data; import profile.service; import profile.api : PROFILE_PATH; -import analytics.balances; +import analytics.util; import util.money; import util.data; @@ -19,11 +19,24 @@ void handleGetCategorySpendTimeSeries(ref ServerHttpRequest request, ref ServerH } @GetMapping(PROFILE_PATH ~ "/analytics/balance-time-series") -void handleGetBalanceTimeSeriesV2(ref ServerHttpRequest request, ref ServerHttpResponse response) { +void handleGetBalanceTimeSeries(ref ServerHttpRequest request, ref ServerHttpResponse response) { + import analytics.modules.balances; auto ds = getProfileDataSource(request); Currency currency = Currency.ofCode(request.getParamAs!string("currency", Currencies.USD.code)); - TimeRange timeRange = TimeRange(Optional!(SysTime).empty(), Optional!(SysTime).empty()); - auto data = computeBalanceTimeSeriesV2(ds, currency, timeRange); + auto data = computeBalanceTimeSeries(ds, currency, getTimeRange(request)); + writeJsonBody(response, data); +} + +@GetMapping(PROFILE_PATH ~ "/analytics/category-spend") +void handleGetCategorySpend(ref ServerHttpRequest request, ref ServerHttpResponse response) { + import analytics.modules.category_spend; + auto ds = getProfileDataSource(request); + Currency currency = Currency.ofCode(request.getParamAs!string("currency", Currencies.USD.code)); + long categoryParentId = request.getParamAs!long("parentId", -1); + Optional!ulong categoryParentIdOpt = categoryParentId == -1 + ? Optional!(ulong).empty() + : Optional!(ulong).of(categoryParentId); + auto data = computeCategorySpend(ds, currency, getTimeRange(request), categoryParentIdOpt); writeJsonBody(response, data); } @@ -46,3 +59,16 @@ private void serveJsonFromProperty(ref ServerHttpResponse response, ref ProfileD response.writeBodyString(jsonStr, ContentTypes.APPLICATION_JSON); } } + +private TimeRange getTimeRange(in ServerHttpRequest request) { + long fromTimestampMs = request.getParamAs!long("from", -1); + long toTimestampMs = request.getParamAs!long("to", -1); + return TimeRange( + fromTimestampMs == -1 + ? Optional!(SysTime).empty() + : Optional!(SysTime).of(toSysTime(fromTimestampMs)), + toTimestampMs == -1 + ? Optional!(SysTime).empty() + : Optional!(SysTime).of(toSysTime(toTimestampMs)) + ); +} diff --git a/finnow-api/source/analytics/data.d b/finnow-api/source/analytics/data.d index d249273..6020b29 100644 --- a/finnow-api/source/analytics/data.d +++ b/finnow-api/source/analytics/data.d @@ -1,12 +1,23 @@ module analytics.data; import std.datetime; +import handy_http_primitives : Optional; +import asdf : serdeTransformOut; import util.money; import util.data; -import analytics.balances; import account.model; +/** + * A common time-series data point. + */ +struct TimeSeriesPoint { + /// The millisecond UTC timestamp. + ulong x; + /// The value at this timestamp. + long y; +} + struct JournalEntryStub { SysTime timestamp; ulong accountId; @@ -21,6 +32,15 @@ struct BalanceRecordStub { long value; } +struct CategorySpendData { + ulong categoryId; + string categoryName; + string categoryColor; + @serdeTransformOut!serializeOptional + Optional!ulong parentCategoryId; + long amount; +} + /** * Repository that provides various functions for fetching data that's used in * the calculation of analytics, separate from usual app functionality. @@ -34,4 +54,8 @@ interface AnalyticsRepository { in Currency currency, in TimeRange timeRange ); + CategorySpendData[] getCategorySpendData( + in Currency currency, + in TimeRange timeRange + ); } \ No newline at end of file diff --git a/finnow-api/source/analytics/data_impl_sqlite.d b/finnow-api/source/analytics/data_impl_sqlite.d index 8ca59e1..46da9c7 100644 --- a/finnow-api/source/analytics/data_impl_sqlite.d +++ b/finnow-api/source/analytics/data_impl_sqlite.d @@ -8,7 +8,7 @@ import util.money; import util.data; import util.sqlite; import account.model : AccountJournalEntryType, AccountType; -import analytics.balances; +import analytics.modules.balances; import analytics.data; class SqliteAnalyticsRepository : AnalyticsRepository { @@ -101,4 +101,53 @@ class SqliteAnalyticsRepository : AnalyticsRepository { } return app[]; } + + CategorySpendData[] getCategorySpendData( + in Currency currency, + in TimeRange timeRange + ) { + QueryBuilder qb = QueryBuilder("transaction_category c") + .select("c.id,c.name,c.color,c.parent_id") + .select(` + ( + SUM(CASE WHEN je.type LIKE 'DEBIT' THEN je.amount ELSE 0 END) + - SUM(CASE WHEN je.type LIKE 'CREDIT' THEN je.amount ELSE 0 END) + ) + `) + .join("LEFT JOIN \"transaction\" t ON t.category_id = c.id") + .join("LEFT JOIN account_journal_entry je ON je.transaction_id = t.id"); + qb.where("t.internal_transfer = false"); + qb.where("t.currency = ?"); + qb.withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, currency.codeString); + }); + if (timeRange.fromTime) { + qb.where("t.timestamp >= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + if (timeRange.toTime) { + qb.where("t.timestamp <= ?") + .withArgBinding((ref stmt, ref idx) { + stmt.bind(idx++, timeRange.fromTime.value); + }); + } + string query = qb.build() ~ " GROUP BY c.id ORDER BY c.id"; + Statement stmt = db.prepare(query); + qb.applyArgBindings(stmt); + ResultRange result = stmt.execute(); + Appender!(CategorySpendData[]) app; + foreach (row; result) { + import std.typecons : Nullable; + app ~= CategorySpendData( + row.peek!ulong(0), + row.peek!string(1), + row.peek!string(2), + toOptional(row.peek!(Nullable!ulong)(3)), + row.peek!long(4) + ); + } + return app[]; + } } \ No newline at end of file diff --git a/finnow-api/source/analytics/balances.d b/finnow-api/source/analytics/modules/balances.d similarity index 63% rename from finnow-api/source/analytics/balances.d rename to finnow-api/source/analytics/modules/balances.d index 46729ed..7ab596b 100644 --- a/finnow-api/source/analytics/balances.d +++ b/finnow-api/source/analytics/modules/balances.d @@ -1,4 +1,4 @@ -module analytics.balances; +module analytics.modules.balances; import handy_http_primitives : Optional, mapIfPresent; import std.datetime; @@ -24,13 +24,6 @@ import util.money; import util.data; import analytics.data; -struct TimeSeriesPoint { - /// The millisecond UTC timestamp. - ulong x; - /// The value at this timestamp. - long y; -} - struct AccountBalanceData { ulong accountId; TimeSeriesPoint[] data; @@ -51,7 +44,7 @@ struct BalanceTimeSeriesAnalytics { * Returns: An analytics response containing a "totals" time series, as well * as a time series for each known account in the given time range. */ -BalanceTimeSeriesAnalytics computeBalanceTimeSeriesV2( +BalanceTimeSeriesAnalytics computeBalanceTimeSeries( ProfileDataSource ds, in Currency currency, in TimeRange timeRange @@ -151,75 +144,3 @@ private Optional!long deriveBalance( } return Optional!(long).of(balance); } - - -alias CurrencyGroupedTimeSeries = TimeSeriesPoint[][string]; - -struct CategorySpendData { - ulong categoryId; - string categoryName; - string categoryColor; - CurrencyGroupedTimeSeries dataByCurrency; -} - -struct CategorySpendTimeSeriesAnalytics { - CategorySpendData[] categories; -} - -void computeCategorySpendTimeSeries(Profile profile, ProfileRepository profileRepo) { - ProfileDataSource ds = profileRepo.getDataSource(profile); - TransactionCategory[] rootCategories = ds.getTransactionCategoryRepository() - .findAllByParentId(Optional!ulong.empty); - - CategorySpendTimeSeriesAnalytics data; - - PageRequest pr = PageRequest(1, 100, [Sort("txn.timestamp", SortDir.ASC)]); - Page!TransactionsListItem page = ds.getTransactionRepository().findAll(pr); - while (page.items.length > 0) { - foreach (TransactionsListItem txn; page.items) { - if (!isTypicalSpendingTransaction(txn)) continue; - - ulong categoryId = txn.category.mapIfPresent!(c => c.id).orElse(0); - TimeSeriesPoint dataPoint = TimeSeriesPoint( - SysTime.fromISOExtString(txn.timestamp).toUnixMillis(), - txn.amount - ); - - bool foundCategory = false; - foreach (ref cd; data.categories) { - if (cd.categoryId == categoryId) { - cd.dataByCurrency[txn.currency.code.idup] ~= dataPoint; - foundCategory = true; - break; - } - } - if (!foundCategory) { - CategorySpendData cd; - cd.categoryId = categoryId; - cd.categoryName = txn.category.mapIfPresent!(c => c.name).orElse("None"); - cd.categoryColor = txn.category.mapIfPresent!(c => c.color).orElse("FFFFFF"); - cd.dataByCurrency[txn.currency.code.idup] ~= dataPoint; - data.categories ~= cd; - } - } - page = ds.getTransactionRepository().findAll(page.pageRequest.next()); - } - - ds.doTransaction(() { - ds.getPropertiesRepository().deleteProperty("analytics.categorySpendTimeSeries"); - ds.getPropertiesRepository().setProperty( - "analytics.categorySpendTimeSeries", - serializeToJsonPretty(data) - ); - }); - infoF!"Computed category spend analytics for user %s, profile %s."( - profile.username, - profile.name - ); -} - -private bool isTypicalSpendingTransaction(in TransactionsListItem txn) { - return !txn.creditedAccount.isNull && - txn.debitedAccount.isNull && - !txn.internalTransfer; -} diff --git a/finnow-api/source/analytics/modules/category_spend.d b/finnow-api/source/analytics/modules/category_spend.d new file mode 100644 index 0000000..07597e0 --- /dev/null +++ b/finnow-api/source/analytics/modules/category_spend.d @@ -0,0 +1,49 @@ +module analytics.modules.category_spend; + +import handy_http_primitives : Optional; +import std.algorithm; +import std.array; + +import analytics.data; +import profile.data; +import util.money : Currency; +import util.data : TimeRange; + +CategorySpendData[] computeCategorySpend( + ProfileDataSource ds, + in Currency currency, + in TimeRange timeRange, + in Optional!ulong parentId +) { + AnalyticsRepository repo = ds.getAnalyticsRepository(); + CategorySpendData[] allCategories = repo.getCategorySpendData(currency, timeRange); + return allCategories + .filter!(d => ( + parentId + ? d.parentCategoryId && d.parentCategoryId.value == parentId.value + : d.parentCategoryId.isNull + )) + .map!((category) { + // For each category that we're reporting on, recursively sum up + // the amount of the category and all its children. + long totalRecursiveSum = sumAllChildCategoriesRecursive(category, allCategories); + return CategorySpendData( + category.categoryId, + category.categoryName, + category.categoryColor, + category.parentCategoryId, + totalRecursiveSum + ); + }) + .array(); +} + +private long sumAllChildCategoriesRecursive(in CategorySpendData parentCategory, in CategorySpendData[] data) { + long sum = parentCategory.amount; + foreach (category; data) { + if (category.parentCategoryId && category.parentCategoryId.value == parentCategory.categoryId) { + sum += sumAllChildCategoriesRecursive(category, data); + } + } + return sum; +} \ No newline at end of file diff --git a/finnow-api/source/analytics/package.d b/finnow-api/source/analytics/package.d deleted file mode 100644 index b5ebc45..0000000 --- a/finnow-api/source/analytics/package.d +++ /dev/null @@ -1,28 +0,0 @@ -module analytics; - -public import analytics.balances; - -import profile.data; -import profile.model; - -/** - * Helper function to run a function on each available user profile. - * Params: - * fn = The function to run. - */ -void doForAllUserProfiles( - void function(Profile, ProfileRepository) fn -) { - import auth.data; - import auth.data_impl_fs; - import profile.data; - import profile.data_impl_sqlite; - - UserRepository userRepo = new FileSystemUserRepository(); - foreach (user; userRepo.findAll()) { - ProfileRepository profileRepo = new FileSystemProfileRepository(user.username); - foreach (prof; profileRepo.findAll()) { - fn(prof, profileRepo); - } - } -} \ No newline at end of file diff --git a/finnow-api/source/analytics/util.d b/finnow-api/source/analytics/util.d index 8a2ead1..bab2e90 100644 --- a/finnow-api/source/analytics/util.d +++ b/finnow-api/source/analytics/util.d @@ -34,8 +34,6 @@ SysTime[] generateTimeSeriesTimestamps(Duration intervalSize, in TimeRange timeR } else { startOfRange = timeRange.fromTime.value; } - import std.stdio; - writefln!"start = %s, end = %s"(startOfRange, endOfRange); Appender!(SysTime[]) app; app ~= startOfRange; @@ -51,3 +49,7 @@ SysTime[] generateTimeSeriesTimestamps(Duration intervalSize, in TimeRange timeR ulong toUnixMillis(in SysTime ts) { return (ts - SysTime(unixTimeToStdTime(0))).total!"msecs"; } + +SysTime toSysTime(in ulong unixMillis) { + return SysTime(unixTimeToStdTime(0)) + msecs(unixMillis); +} diff --git a/finnow-api/source/scheduled_jobs.d b/finnow-api/source/scheduled_jobs.d index 0bb6bc3..fe7e04d 100644 --- a/finnow-api/source/scheduled_jobs.d +++ b/finnow-api/source/scheduled_jobs.d @@ -4,8 +4,6 @@ import scheduled; import std.datetime; import slf4d; -import analytics; - void startScheduledJobs() { JobSchedule analyticsSchedule = new FixedIntervalSchedule( hours(1), @@ -13,10 +11,9 @@ void startScheduledJobs() { ); JobScheduler jobScheduler = new TaskPoolScheduler(); - jobScheduler.addJob(() { - info("Computing account balance time series analytics for all users..."); - doForAllUserProfiles(&computeCategorySpendTimeSeries); - info("Done computing analytics!"); - }, analyticsSchedule); - jobScheduler.start(); + // jobScheduler.addJob(() { + // info("Computing account balance time series analytics for all users..."); + // info("Done computing analytics!"); + // }, analyticsSchedule); + // jobScheduler.start(); } \ No newline at end of file diff --git a/web-app/src/api/analytics.ts b/web-app/src/api/analytics.ts index 9963eb2..979b523 100644 --- a/web-app/src/api/analytics.ts +++ b/web-app/src/api/analytics.ts @@ -7,28 +7,22 @@ export interface TimeSeriesPoint { y: number } -export type CurrencyGroupedTimeSeries = Record - export interface AccountBalanceData { accountId: number - currencyCode: string data: TimeSeriesPoint[] } export interface BalanceTimeSeriesAnalytics { accounts: AccountBalanceData[] - totals: CurrencyGroupedTimeSeries + totals: TimeSeriesPoint[] } export interface CategorySpendData { - categoryId: number - categoryName: string - categoryColor: string - dataByCurrency: CurrencyGroupedTimeSeries -} - -export interface CategorySpendTimeSeriesAnalytics { - categories: CategorySpendData[] + categoryId: number | null + categoryName: string | null + categoryColor: string | null + parentCategoryId: number | null + amount: number } export class AnalyticsApiClient extends ApiClient { @@ -41,11 +35,39 @@ export class AnalyticsApiClient extends ApiClient { this.path = `/profiles/${this.profileName}/analytics` } - getBalanceTimeSeries(): Promise { - return super.getJson(this.path + '/balance-time-series') + getBalanceTimeSeries( + currencyCode: string, + fromTimestampMs: number | null, + toTimestampMs: number | null, + ): Promise { + const params = new URLSearchParams() + params.append('currency', currencyCode) + if (fromTimestampMs !== null) { + params.append('from', fromTimestampMs + '') + } + if (toTimestampMs !== null) { + params.append('to', toTimestampMs + '') + } + return super.getJson(this.path + '/balance-time-series?' + params.toString()) } - getCategorySpendTimeSeries(): Promise { - return super.getJson(this.path + '/category-spend-time-series') + getCategorySpend( + currencyCode: string, + fromTimestampMs: number | null, + toTimestampMs: number | null, + parentCategoryId: number | null, + ): Promise { + const params = new URLSearchParams() + params.append('currency', currencyCode) + if (fromTimestampMs !== null) { + params.append('from', fromTimestampMs + '') + } + if (toTimestampMs !== null) { + params.append('to', toTimestampMs + '') + } + if (parentCategoryId !== null) { + params.append('parentId', parentCategoryId + '') + } + return super.getJson(this.path + '/category-spend?' + params.toString()) } } diff --git a/web-app/src/pages/home/AnalyticsModule.vue b/web-app/src/pages/home/AnalyticsModule.vue index b97793c..64e014b 100644 --- a/web-app/src/pages/home/AnalyticsModule.vue +++ b/web-app/src/pages/home/AnalyticsModule.vue @@ -1,29 +1,27 @@ diff --git a/web-app/src/pages/home/analytics/CategorySpendPieChart.vue b/web-app/src/pages/home/analytics/CategorySpendPieChart.vue index 8158300..975e1de 100644 --- a/web-app/src/pages/home/analytics/CategorySpendPieChart.vue +++ b/web-app/src/pages/home/analytics/CategorySpendPieChart.vue @@ -1,19 +1,18 @@