module transaction.data_impl_sqlite; import handy_http_primitives : Optional, ServerHttpRequest; import std.datetime; import std.typecons; import d2sqlite3; import transaction.model; import transaction.data; import transaction.dto; import util.sqlite; import util.money; import util.pagination; import util.data; import account.model; class SqliteTransactionVendorRepository : TransactionVendorRepository { private Database db; this(Database db) { this.db = db; } Optional!TransactionVendor findById(ulong id) { return util.sqlite.findById(db, "transaction_vendor", &parseVendor, id); } TransactionVendor[] findAll() { return util.sqlite.findAll( db, "SELECT * FROM transaction_vendor ORDER BY name ASC", &parseVendor ); } bool existsByName(string name) { return util.sqlite.exists(db, "SELECT id FROM transaction_vendor WHERE name = ?", name); } bool existsById(ulong id) { return util.sqlite.exists(db, "SELECT id FROM transaction_vendor WHERE id = ?", id); } TransactionVendor insert(string name, string description) { util.sqlite.update( db, "INSERT INTO transaction_vendor (name, description) VALUES (?, ?)", name, description ); ulong id = db.lastInsertRowid(); return findById(id).orElseThrow(); } void deleteById(ulong id) { util.sqlite.deleteById(db, "transaction_vendor", id); } TransactionVendor updateById(ulong id, string name, string description) { util.sqlite.update( db, "UPDATE transaction_vendor SET name = ?, description = ? WHERE id = ?", name, description, id ); return findById(id).orElseThrow(); } private static TransactionVendor parseVendor(Row row) { return TransactionVendor( row.peek!ulong(0), row.peek!string(1), row.peek!string(2) ); } } class SqliteTransactionCategoryRepository : TransactionCategoryRepository { private Database db; this(Database db) { this.db = db; } Optional!TransactionCategory findById(ulong id) { return util.sqlite.findById(db, "transaction_category", &parseCategory, id); } Optional!TransactionCategory findByName(string name) { return util.sqlite.findOne(db, "SELECT * FROM transaction_category WHERE name = ?", &parseCategory, name); } bool existsById(ulong id) { return util.sqlite.exists(db, "SELECT id FROM transaction_category WHERE id = ?", id); } bool existsByName(string name) { return util.sqlite.exists(db, "SELECT id FROM transaction_category WHERE name = ?", name); } TransactionCategory[] findAll() { return util.sqlite.findAll( db, "SELECT * FROM transaction_category ORDER BY parent_id, name", &parseCategory ); } TransactionCategory[] findAllByParentId(Optional!ulong parentId) { if (parentId) { return util.sqlite.findAll( db, "SELECT * FROM transaction_category WHERE parent_id = ? ORDER BY name ASC", &parseCategory, parentId.value ); } return util.sqlite.findAll( db, "SELECT * FROM transaction_category WHERE parent_id IS NULL ORDER BY name ASC", &parseCategory ); } TransactionCategory insert(Optional!ulong parentId, string name, string description, string color) { util.sqlite.update( db, "INSERT INTO transaction_category (parent_id, name, description, color) VALUES (?, ?, ?, ?)", toNullable(parentId), name, description, color ); ulong id = db.lastInsertRowid(); return findById(id).orElseThrow(); } void deleteById(ulong id) { util.sqlite.deleteById(db, "transaction_category", id); } TransactionCategory updateById(ulong id, string name, string description, string color, Optional!ulong parentId) { util.sqlite.update( db, "UPDATE transaction_category SET name = ?, description = ?, color = ?, parent_id = ? WHERE id = ?", name, description, color, toNullable!ulong(parentId), id ); return findById(id).orElseThrow(); } TransactionCategoryBalance[] getBalance( ulong id, bool includeChildren, Optional!SysTime afterTimestamp, Optional!SysTime beforeTimestamp ) { import std.algorithm : map; import std.conv : to; import std.string : join; // First collect the list of IDs to include in the query. ulong[] idsForQuery = [id]; if (includeChildren) { import std.range : front, popFront; ulong[] idQueue = [id]; while (idQueue.length > 0) { ulong nextId = idQueue.front; idQueue.popFront; auto children = findAllByParentId(Optional!ulong.of(nextId)); foreach (child; children) { idsForQuery ~= child.id; idQueue ~= child.id; } } } const categoryIdsString = idsForQuery.map!(id => id.to!string).join(","); // Now build the query, taking into account the optional timestamp constraints. QueryBuilder qb = QueryBuilder("\"transaction\" txn") .join("LEFT JOIN account_journal_entry je ON je.transaction_id = txn.id") .select("je.currency") .select("je.type") .select("SUM(je.amount)") .where("txn.category_id IN (" ~ categoryIdsString ~ ")"); if (!afterTimestamp.isNull) { qb.where("txn.timestamp > ?") .withArgBinding((ref stmt, ref idx) { stmt.bind(idx++, afterTimestamp.value.toISOExtString()); }); } if (!beforeTimestamp.isNull) { qb.where("txn.timestamp < ?") .withArgBinding((ref stmt, ref idx) { stmt.bind(idx++, beforeTimestamp.value.toISOExtString()); }); } string query = qb.build() ~ " ORDER BY je.currency ASC, je.type ASC"; Statement stmt = db.prepare(query); qb.applyArgBindings(stmt); ResultRange result = stmt.execute(); // Process the results into a set of category balances for each currency. TransactionCategoryBalance[ushort] balancesGroupedByCurrency; foreach (row; result) { Currency currency = Currency.ofCode(row.peek!(string, PeekMode.slice)(0)); string journalEntryType = row.peek!(string, PeekMode.slice)(1); ulong amountSum = row.peek!ulong(2); TransactionCategoryBalance balance; if (currency.numericCode in balancesGroupedByCurrency) { balance = balancesGroupedByCurrency[currency.numericCode]; } else { balance = TransactionCategoryBalance(0, 0, 0, currency); } if (journalEntryType == AccountJournalEntryType.CREDIT) { balance.credits = amountSum; } else if (journalEntryType == AccountJournalEntryType.DEBIT) { balance.debits = amountSum; } balancesGroupedByCurrency[currency.numericCode] = balance; } // Post-process into a list of balances for returning: TransactionCategoryBalance[] balances = balancesGroupedByCurrency.values; foreach (ref bal; balances) { bal.balance = cast(long) bal.debits - cast(long) bal.credits; } return balances; } private static TransactionCategory parseCategory(Row row) { import std.typecons; return TransactionCategory( row.peek!ulong(0), toOptional(row.peek!(Nullable!ulong)(1)), row.peek!string(2), row.peek!string(3), row.peek!string(4) ); } } class SqliteTransactionTagRepository : TransactionTagRepository { private Database db; this(Database db) { this.db = db; } string[] findAllByTransactionId(ulong transactionId) { return util.sqlite.findAll( db, "SELECT tag FROM transaction_tag WHERE transaction_id = ? ORDER BY tag", r => r.peek!string(0), transactionId ); } void updateTags(ulong transactionId, in string[] tags) { util.sqlite.update( db, "DELETE FROM transaction_tag WHERE transaction_id = ?", transactionId ); foreach (tag; tags) { util.sqlite.update( db, "INSERT INTO transaction_tag (transaction_id, tag) VALUES (?, ?)", transactionId, tag ); } } string[] findAll() { return util.sqlite.findAll( db, "SELECT DISTINCT tag FROM transaction_tag ORDER BY tag", r => r.peek!string(0) ); } } class SqliteTransactionRepository : TransactionRepository { private const TABLE_NAME = "\"transaction\""; private Database db; this(Database db) { this.db = db; } Page!TransactionsListItem findAll(in PageRequest pr) { const pageIdsQuery = "SELECT DISTINCT txn.id FROM \"transaction\" txn " ~ pr.toSql(); QueryBuilder qb = getBuilderForTransactionsList(); addSelectsForTransactionsList(qb); qb.where("txn.id IN (" ~ pageIdsQuery ~ ")"); string query = qb.build() ~ "\n" ~ pr.toOrderClause(); TransactionsListItem[] results = util.sqlite.findAllDirect(db, query, &parseListItems); ulong totalCount = util.sqlite.count(db, "SELECT COUNT(DISTINCT id) FROM \"transaction\""); return Page!(TransactionsListItem).of(results, pr, totalCount); } Page!TransactionsListItem search(in PageRequest pr, in ServerHttpRequest request) { import std.algorithm; import std.conv; import std.string : join; import transaction.search_filters; import std.stdio; QueryBuilder qb = getBuilderForTransactionsList(); // 1. Get the total count of transactions that match the search filters. qb.select("COUNT (DISTINCT txn.id)"); applyFilters(qb, request, new SqliteTransactionCategoryRepository(db)); string countQuery = qb.build(); Statement countStmt = db.prepare(countQuery); qb.applyArgBindings(countStmt); auto countResult = countStmt.execute(); ulong count = countResult.empty ? 0 : countResult.front.peek!ulong(0); if (count == 0) return Page!TransactionsListItem.of([], pr, 0); // 2. Select the ordered list of IDs of transactions to include in the page of results. // This is because one transaction may be split over multiple db rows. qb.selections = []; qb.select("DISTINCT txn.id"); string idsQuery = qb.build() ~ "\n" ~ pr.toSql(); Statement idsStmt = db.prepare(idsQuery); qb.applyArgBindings(idsStmt); string idsStr = idsStmt.execute() .map!(r => r.peek!ulong(0)) .map!(id => id.to!string) .join(","); // 3. Select the list of transactions for the results based on the IDs found in step 2. qb.selections = []; qb.conditions = []; qb.argBinders = []; addSelectsForTransactionsList(qb); qb.where("txn.id IN (" ~ idsStr ~ ")"); string query = qb.build() ~ "\n" ~ pr.toOrderClause(); Statement stmt = db.prepare(query); auto results = parseListItems(stmt.execute()); return Page!TransactionsListItem.of(results, pr, count); } Optional!TransactionDetail findById(ulong id) { Optional!TransactionDetail item = util.sqlite.findOne( db, import("sql/get_transaction.sql"), (row) { TransactionDetail item; item.id = row.peek!ulong(0); item.timestamp = row.peek!string(1); item.addedAt = row.peek!string(2); item.amount = row.peek!ulong(3); item.currency = Currency.ofCode(row.peek!(string, PeekMode.slice)(4)); item.description = row.peek!string(5); item.internalTransfer = row.peek!bool(6); Nullable!ulong vendorId = row.peek!(Nullable!ulong)(7); if (!vendorId.isNull) { item.vendor = Optional!(TransactionDetail.Vendor).of( TransactionDetail.Vendor( vendorId.get, row.peek!string(8), row.peek!string(9) )).toNullable; } Nullable!ulong categoryId = row.peek!(Nullable!ulong)(10); if (!categoryId.isNull) { item.category = Optional!(TransactionDetail.Category).of( TransactionDetail.Category( categoryId.get, row.peek!(Nullable!ulong)(11), row.peek!string(12), row.peek!string(13), row.peek!string(14) )).toNullable; } Nullable!ulong creditedAccountId = row.peek!(Nullable!ulong)(15); if (!creditedAccountId.isNull) { item.creditedAccount = Optional!(TransactionDetail.Account).of( TransactionDetail.Account( creditedAccountId.get, row.peek!string(16), row.peek!string(17), row.peek!string(18) )).toNullable; } Nullable!ulong debitedAccountId = row.peek!(Nullable!ulong)(19); if (!debitedAccountId.isNull) { item.debitedAccount = Optional!(TransactionDetail.Account).of( TransactionDetail.Account( debitedAccountId.get, row.peek!string(20), row.peek!string(21), row.peek!string(22) )).toNullable; } string tagsStr = row.peek!string(23); if (tagsStr !is null && tagsStr.length > 0) { import std.string : split; item.tags = tagsStr.split(","); } else { item.tags = []; } return item; }, id ); if (item.isNull) return item; item.value.lineItems = util.sqlite.findAll( db, import("sql/get_line_items.sql"), (row) { TransactionDetail.LineItem li; li.idx = row.peek!uint(0); li.valuePerItem = row.peek!long(1); li.quantity = row.peek!ulong(2); li.description = row.peek!string(3); Nullable!ulong categoryId = row.peek!(Nullable!ulong)(4); if (!categoryId.isNull) { li.category = Optional!(TransactionDetail.Category).of( TransactionDetail.Category( categoryId.get, row.peek!(Nullable!ulong)(5), row.peek!string(6), row.peek!string(7), row.peek!string(8) )).toNullable; } return li; }, id ); return item; } TransactionDetail insert(in AddTransactionPayload data) { util.sqlite.update( db, import("sql/insert_transaction.sql"), data.timestamp, Clock.currTime(UTC()).toISOExtString(), data.amount, data.currencyCode, data.description, data.internalTransfer, data.vendorId, data.categoryId ); ulong transactionId = db.lastInsertRowid(); insertLineItems(transactionId, data); return findById(transactionId).orElseThrow(); } void linkAttachment(ulong transactionId, ulong attachmentId) { util.sqlite.update( db, "INSERT INTO transaction_attachment (transaction_id, attachment_id) VALUES (?, ?)", transactionId, attachmentId ); } TransactionDetail update(ulong transactionId, in AddTransactionPayload data) { util.sqlite.update( db, import("sql/update_transaction.sql"), data.timestamp, data.amount, data.currencyCode, data.description, data.internalTransfer, data.vendorId, data.categoryId, transactionId ); // Re-write all line items: util.sqlite.update( db, "DELETE FROM transaction_line_item WHERE transaction_id = ?", transactionId ); insertLineItems(transactionId, data); return findById(transactionId).orElseThrow(); } void deleteById(ulong id) { util.sqlite.deleteById(db, TABLE_NAME, id); // Delete all history items for journal entries that have been removed by the deletion. util.sqlite.update( db, "DELETE FROM account_history_item WHERE type LIKE 'JOURNAL_ENTRY' AND id NOT IN ( SELECT item_id FROM history_item_linked_journal_entry )" ); } /** * Function to parse a list of transaction list items as obtained from the * `get_transactions.sql` query. Because there are possibly multiple rows * per transaction, we have to deal with the result range as a whole. * Params: * r = The result range to read. * Returns: The list of transaction list items. */ private static TransactionsListItem[] parseListItems(ResultRange r) { import std.array : appender; auto app = appender!(TransactionsListItem[]); TransactionsListItem item; /// Helper function that appends the current item to the list, and resets state. void appendItem() { import std.algorithm : sort; sort(item.tags); app ~= item; item.id = 0; item.tags = []; item.vendor = Optional!(TransactionsListItem.Vendor).empty(); item.category = Optional!(TransactionsListItem.Category).empty(); item.creditedAccount = Optional!(TransactionsListItem.Account).empty(); item.debitedAccount = Optional!(TransactionsListItem.Account).empty(); } size_t rowCount = 0; foreach (Row row; r) { rowCount++; ulong txnId = row.peek!ulong(0); if (item.id != txnId) { // We're parsing a new item. First, append the current one if there is one. if (item.id != 0) { appendItem(); } item.id = txnId; item.timestamp = row.peek!string(1); item.addedAt = row.peek!string(2); item.amount = row.peek!ulong(3); item.currency = Currency.ofCode(row.peek!(string, PeekMode.slice)(4)); item.description = row.peek!string(5); item.internalTransfer = row.peek!bool(6); // Read the nullable Vendor information. Nullable!ulong vendorId = row.peek!(Nullable!ulong)(7); if (!vendorId.isNull) { string vendorName = row.peek!string(8); item.vendor = Optional!(TransactionsListItem.Vendor).of( TransactionsListItem.Vendor(vendorId.get, vendorName)); } // Read the nullable Category information. Nullable!ulong categoryId = row.peek!(Nullable!ulong)(9); if (!categoryId.isNull) { string categoryName = row.peek!string(10); string categoryColor = row.peek!string(11); item.category = Optional!(TransactionsListItem.Category).of( TransactionsListItem.Category(categoryId.get, categoryName, categoryColor)); } // Read the nullable creditedAccount. Nullable!ulong creditedAccountId = row.peek!(Nullable!ulong)(12); if (!creditedAccountId.isNull) { ulong id = creditedAccountId.get; string name = row.peek!string(13); string type = row.peek!string(14); string suffix = row.peek!string(15); item.creditedAccount = Optional!(TransactionsListItem.Account).of( TransactionsListItem.Account(id, name, type, suffix)); } // Read the nullable debitedAccount. Nullable!ulong debitedAccountId = row.peek!(Nullable!ulong)(16); if (!debitedAccountId.isNull) { ulong id = debitedAccountId.get; string name = row.peek!string(17); string type = row.peek!string(18); string suffix = row.peek!string(19); item.debitedAccount = Optional!(TransactionsListItem.Account).of( TransactionsListItem.Account(id, name, type, suffix)); } } // Read multi-row properties, like tags, to the current item. string tag = row.peek!string(20); if (tag !is null) { item.tags ~= tag; } } // If there's one last cached item, append it to the list. if (item.id != 0) { appendItem(); } return app[]; } private void insertLineItems(ulong transactionId, in AddTransactionPayload data) { foreach (size_t idx, lineItem; data.lineItems) { util.sqlite.update( db, import("sql/insert_line_item.sql"), transactionId, idx, lineItem.valuePerItem, lineItem.quantity, lineItem.description, lineItem.categoryId ); } } private QueryBuilder getBuilderForTransactionsList() { return QueryBuilder("\"transaction\" txn") .join("LEFT JOIN transaction_vendor vendor ON vendor.id = txn.vendor_id") .join("LEFT JOIN transaction_category category ON category.id = txn.category_id") .join("LEFT JOIN account_journal_entry j_credit " ~ "ON j_credit.transaction_id = txn.id AND UPPER(j_credit.type) = 'CREDIT'") .join("LEFT JOIN account account_credit ON account_credit.id = j_credit.account_id") .join("LEFT JOIN account_journal_entry j_debit " ~ "ON j_debit.transaction_id = txn.id AND UPPER(j_debit.type) = 'DEBIT'") .join("LEFT JOIN account account_debit ON account_debit.id = j_debit.account_id") .join("LEFT JOIN transaction_tag tags ON tags.transaction_id = txn.id"); } private void addSelectsForTransactionsList(ref QueryBuilder qb) { qb .select("txn.id") .select("txn.timestamp") .select("txn.added_at") .select("txn.amount") .select("txn.currency") .select("txn.description") .select("txn.internal_transfer") .select("txn.vendor_id") .select("vendor.name") .select("txn.category_id") .select("category.name") .select("category.color") .select("account_credit.id") .select("account_credit.name") .select("account_credit.type") .select("account_credit.number_suffix") .select("account_debit.id") .select("account_debit.name") .select("account_debit.type") .select("account_debit.number_suffix") .select("tags.tag"); } }