Added improvements to recurring transactions modal, and implemented scheduled job to process recurring transactions.

finnow-api/source/scheduled_jobs.d

@@ -45,25 +45,18 @@ void startScheduledJobs() {
 		import profile.data;
 		import profile.data_impl_sqlite;
 		import profile.model;
-		import transaction.dto;
+		import transaction.service;
-		import transaction.data;
+		import std.datetime.stopwatch;
-		import util.pagination;
+		StopWatch sw = StopWatch(AutoStart.yes);
-		import std.stdio;
-		import std.datetime;
-		import cronexp;
 		FileSystemProfileRepository.doForAllUserProfiles((Profile profile, ProfileRepository profileRepo) {
-			writefln!"Recurring transaction check: %s / %s"(profile.username, profile.name);
 			ProfileDataSource ds = profileRepo.getDataSource(profile);
-			RecurringTransactionRepository rtRepo = ds.getRecurringTransactionRepository();
+			updateRecurringTransactionScheduling(ds);
-			Page!(RecurringTransactionResponse) result = rtRepo.findAll(PageRequest.unpaged());
-			foreach (rt; result.items) {
-				writeln(rt.scheduleExpr);
-				DateTime now = cast(DateTime) Clock.currTime();
-				auto cron = CronExpr(rt.scheduleExpr);
-				writefln!"scheduleExpr = %s -> %s"(rt.scheduleExpr, cron.getNext(now));
-				// TODO: Figure out how to actually create the transactions!
-			}
 		});
+		sw.stop();
+		long elapsedTimeMs = sw.peek.total!"msecs";
+		if (elapsedTimeMs > 1000) {
+			warnF!"Took more than 1 second to process recurring transactions: %d ms."(elapsedTimeMs);
+		}
 	}, new FixedIntervalSchedule(minutes(1), Clock.currTime(UTC())));
 
 	jobScheduler.start();

finnow-api/source/transaction/data.d

@@ -74,4 +74,9 @@ interface RecurringTransactionRepository {
     Optional!RecurringTransactionResponse findById(ulong id);
     RecurringTransactionResponse insert(in RecurringTransactionPayload data);
     void deleteById(ulong id);
+    void deleteAllByDraftId(ulong draftId);
+
+    void addToQueue(ulong id, string timestamp);
+    Optional!string findInQueue(ulong id);
+    void removeFromQueue(ulong id);
 }

finnow-api/source/transaction/data_impl_sqlite.d

@@ -770,6 +770,11 @@ class SqliteTransactionDraftRepository : TransactionDraftRepository {
             draftId
         );
         insertLineItems(draftId, data);
+        // If template name is empty / NULL, clear any recurring transactions that may exist.
+        if (data.templateName.isNull || data.templateName.value.length == 0) {
+            new SqliteRecurringTransactionRepository(this.db)
+                .deleteAllByDraftId(draftId);
+        }
         return findById(draftId).orElseThrow();
     }
 
@@ -957,6 +962,40 @@ class SqliteRecurringTransactionRepository : RecurringTransactionRepository {
         util.sqlite.deleteById(db, "recurring_transaction", id);
     }
 
+    void deleteAllByDraftId(ulong draftId) {
+        util.sqlite.update(
+            db,
+            "DELETE FROM recurring_transaction WHERE draft_id = ?",
+            draftId
+        );
+    }
+
+    void addToQueue(ulong id, string timestamp) {
+        util.sqlite.update(
+            db,
+            "INSERT INTO recurring_transaction_queue (recurring_transaction_id, scheduled_timestamp) VALUES (?, ?)",
+            id,
+            timestamp
+        );
+    }
+
+    Optional!string findInQueue(ulong id) {
+        return util.sqlite.findOne(
+            db,
+            "SELECT scheduled_timestamp FROM recurring_transaction_queue WHERE recurring_transaction_id = ?",
+            row => row.peek!string(0),
+            id
+        );
+    }
+
+    void removeFromQueue(ulong id) {
+        util.sqlite.update(
+            db,
+            "DELETE FROM recurring_transaction_queue WHERE recurring_transaction_id = ?",
+            id
+        );
+    }
+
     private static RecurringTransactionResponse parseRecurringTransaction(Row row) {
         return RecurringTransactionResponse(
             row.peek!ulong(0),

finnow-api/source/transaction/service.d

@@ -551,3 +551,104 @@ RecurringTransactionResponse[] getRecurringTransactionsForDraft(ProfileDataSourc
     return ds.getRecurringTransactionRepository()
         .findAllByDraftId(draftId);
 }
+
+/**
+ * A routine used by a scheduled job that runs periodically for all users, and
+ * all those users' profiles. It iterates over every one of the user's recurring
+ * transactions and does the following:
+ *  - Adds it to the persistent queue with the next timestamp if it's not
+ *    already there.
+ *  - Creates a new draft transaction from the recurring transaction's linked
+ *    template, if its queued timestamp has elapsed. Then requeues that
+ *    recurring transaction for its next timestamp.
+ * Params:
+ *   ds = The profile data source to operate on.
+ */
+void updateRecurringTransactionScheduling(ProfileDataSource ds) {
+    import cronexp;
+
+    RecurringTransactionRepository rtRepo = ds.getRecurringTransactionRepository();
+    Page!(RecurringTransactionResponse) result = rtRepo.findAll(PageRequest.unpaged());
+    const DateTime NOW = cast(DateTime) Clock.currTime(UTC());
+    ds.doTransaction(() {
+        foreach (RecurringTransactionResponse rt; result.items) {
+            auto cron = CronExpr(rt.scheduleExpr);
+            auto nextScheduledOccurrence = cron.getNext(NOW);
+            if (nextScheduledOccurrence.isNull) {
+                rtRepo.deleteById(rt.id);
+            } else {
+                // First, check if it's already in queue to execute (and execute it if it's time).
+                Optional!string queuedTimestampStr = rtRepo.findInQueue(rt.id);
+                bool shouldRequeue = false;
+                if (queuedTimestampStr) {
+                    DateTime queuedTimestamp = DateTime.fromISOExtString(queuedTimestampStr.value);
+                    if (queuedTimestamp < NOW) {
+                        createDraftForRecurringTransaction(ds, rt, queuedTimestamp);
+                        rtRepo.removeFromQueue(rt.id);
+                        shouldRequeue = true;
+                    } else if (queuedTimestamp != nextScheduledOccurrence) {
+                        // The queued timestamp doesn't match the next scheduled occurrence.
+                        // This shouldn't really happen at all, but if it does, we remove it
+                        // so the proper scheduled timestamp can be added.
+                        warnF!("Found a recurring transaction whose queued timestamp " ~ 
+                            "doesn't match the next scheduled occurrence as per its " ~
+                            "Cron schedule expression. Removing that queued transaction " ~
+                            "and requeueing it using the Cron expression's next date. " ~
+                            "Recurring Transaction ID: %d; Cron Expression: %s")(rt.id, rt.scheduleExpr);
+                        rtRepo.removeFromQueue(rt.id);
+                        shouldRequeue = true;
+                    }
+                } else {
+                    shouldRequeue = true;
+                }
+
+                if (shouldRequeue) {
+                    const nextTimestamp = nextScheduledOccurrence.get().toISOExtString();
+                    infoF!"Requeueing recurring transaction %d to occur next at %s."(rt.id, nextTimestamp);
+                    rtRepo.addToQueue(rt.id, nextTimestamp);
+                }
+            }
+        }
+    });
+}
+
+private void createDraftForRecurringTransaction(
+    ProfileDataSource ds,
+    in RecurringTransactionResponse rt,
+    DateTime queuedTimestamp
+) {
+    TransactionDraftRepository draftRepo = ds.getTransactionDraftRepository();
+    Optional!TransactionDraftResponse draftOpt = draftRepo.findById(rt.draftId);
+    if (!draftOpt) {
+        warnF!"Couldn't create draft for recurring transaction %d because template draft %d doesn't exist."(
+            rt.id, rt.draftId
+        );
+        return;
+    }
+    TransactionDraftResponse templateDraft = draftOpt.value;
+    TransactionDraftPayload payload;
+    payload.timestamp = queuedTimestamp.toISOExtString().toOptional();
+    payload.amount = templateDraft.amount;
+    payload.currencyCode = templateDraft.currency.mapIfPresent!(c => c.code.idup);
+    string description = templateDraft.description.orElse("")
+        ~ "\n- Created automatically as a recurring transaction from template: \"" ~
+        templateDraft.templateName.orElse("Unknown") ~ "\".";
+    payload.description = description.toOptional;
+    payload.internalTransfer = templateDraft.internalTransfer;
+    payload.vendorId = templateDraft.vendor.mapIfPresent!(v => v.id);
+    payload.categoryId = templateDraft.category.mapIfPresent!(c => c.id);
+    payload.creditedAccountId = templateDraft.creditedAccount.mapIfPresent!(a => a.id);
+    payload.debitedAccountId = templateDraft.debitedAccount.mapIfPresent!(a => a.id);
+    payload.tags = templateDraft.tags;
+    import std.algorithm : map;
+    import std.array : array;
+    payload.lineItems = templateDraft.lineItems.map!(li => TransactionDraftPayload.LineItemPayload(
+        li.valuePerItem,
+        li.quantity,
+        li.description,
+        li.category.mapIfPresent!(c => c.id)
+    )).array;
+    auto createdDraft = draftRepo.insert(payload);
+    draftRepo.updateTags(createdDraft.id, payload.tags);
+    infoF!"Inserted draft %d for scheduled recurring transaction %d."(createdDraft.id, rt.id);
+}

finnow-api/sql/migrations/2.sql

@@ -72,3 +72,13 @@ CREATE TABLE recurring_transaction (
         FOREIGN KEY (draft_id) REFERENCES transaction_draft(id)
             ON UPDATE CASCADE ON DELETE CASCADE
 );
+
+-- A "queue" table where upcoming recurring transactions are posted until it's
+-- time to add them for real. This ensures proper handling of recurring transactions.
+CREATE TABLE recurring_transaction_queue (
+    recurring_transaction_id INTEGER PRIMARY KEY,
+    scheduled_timestamp TEXT NOT NULL,
+    CONSTRAINT fk_recurring_transaction_queue_id
+        FOREIGN KEY (recurring_transaction_id) REFERENCES recurring_transaction(id)
+            ON UPDATE CASCADE ON DELETE CASCADE
+);

finnow-api/sql/schema.sql

@@ -302,3 +302,13 @@ CREATE TABLE recurring_transaction (
         FOREIGN KEY (draft_id) REFERENCES transaction_draft(id)
             ON UPDATE CASCADE ON DELETE CASCADE
 );
+
+-- A "queue" table where upcoming recurring transactions are posted until it's
+-- time to add them for real. This ensures proper handling of recurring transactions.
+CREATE TABLE recurring_transaction_queue (
+    recurring_transaction_id INTEGER PRIMARY KEY,
+    scheduled_timestamp TEXT NOT NULL,
+    CONSTRAINT fk_recurring_transaction_queue_id
+        FOREIGN KEY (recurring_transaction_id) REFERENCES recurring_transaction(id)
+            ON UPDATE CASCADE ON DELETE CASCADE
+);

web-app/package.json

@@ -26,6 +26,7 @@
     "chart.js": "^4.5.1",
     "chartjs-adapter-date-fns": "^3.0.0",
     "cron-parser": "^5.6.1",
+    "cronstrue": "^3.24.0",
     "date-fns": "^4.1.0",
     "date-fns-tz": "^3.2.0",
     "pinia": "^3.0.4",

web-app/src/components/AddRecurringTransactionModal.vue

@@ -5,13 +5,14 @@ import {
   type RecurringTransactionResponse,
 } from '@/api/transaction'
 import ModalWrapper from './common/ModalWrapper.vue'
-import { ref, useTemplateRef, type Ref } from 'vue'
+import { computed, ref, useTemplateRef, watch, type Ref } from 'vue'
 import { useRoute } from 'vue-router'
 import AppForm from './common/form/AppForm.vue'
 import FormGroup from './common/form/FormGroup.vue'
 import FormControl from './common/form/FormControl.vue'
 import AppButton from './common/AppButton.vue'
 import { getSelectedProfile } from '@/api/profile.ts'
+import { CronExpressionParser } from 'cron-parser'
 
 const route = useRoute()
 const props = defineProps<{ draftId: number }>()
@@ -19,7 +20,34 @@ const modal = useTemplateRef('modal')
 const savedTxn: Ref<RecurringTransactionResponse | undefined> = ref()
 
 // Form Data:
+enum ScheduleType {
+  Monthly,
+  Custom,
+}
+
+const chosenScheduleType: Ref<ScheduleType> = ref(ScheduleType.Monthly)
+const inputIsValid = ref(true)
 const dayOfMonth: Ref<number> = ref(1)
+const customScheduleText = ref('')
+const customCronErrorMsg = ref<string | null>(null)
+
+// Reset form data when the user chooses a different schedule type.
+watch(chosenScheduleType, () => {
+  dayOfMonth.value = 1
+  customScheduleText.value = ''
+  customCronErrorMsg.value = null
+})
+
+watch([chosenScheduleType, dayOfMonth, customScheduleText], () => {
+  inputIsValid.value = checkInputValid()
+})
+
+const upcomingDates = computed(() => {
+  if (!inputIsValid.value) return []
+  const cronExpr = getScheduleExpression()
+  const cron = CronExpressionParser.parse(cronExpr)
+  return cron.take(3).map((d) => d.toString())
+})
 
 async function show(): Promise<RecurringTransactionResponse | undefined> {
   if (!modal.value) return undefined
@@ -34,7 +62,7 @@ async function show(): Promise<RecurringTransactionResponse | undefined> {
 async function addRecurringTransaction() {
   const payload: RecurringTransactionPayload = {
     draftId: props.draftId,
-    scheduleExpr: `0 0 0 ${dayOfMonth.value} * *`,
+    scheduleExpr: getScheduleExpression(),
   }
   const api = new TransactionApiClient(getSelectedProfile(route))
   try {
@@ -46,6 +74,48 @@ async function addRecurringTransaction() {
   }
 }
 
+function getScheduleExpression(): string {
+  if (chosenScheduleType.value === ScheduleType.Custom) {
+    try {
+      CronExpressionParser.parse(customScheduleText.value)
+      return customScheduleText.value.trim()
+    } catch (err) {
+      console.warn('Failed to parse schedule Cron expression.', err)
+      throw err
+    }
+  } else if (chosenScheduleType.value === ScheduleType.Monthly) {
+    return `0 0 12 ${dayOfMonth.value} * *`
+  }
+  throw new Error('Unsupported schedule type.')
+}
+
+function checkInputValid(): boolean {
+  if (chosenScheduleType.value === ScheduleType.Custom) {
+    customCronErrorMsg.value = null
+    if (customScheduleText.value.trim().length === 0) {
+      customCronErrorMsg.value = 'Missing cron expression.'
+      return false
+    }
+    try {
+      const expr = CronExpressionParser.parse(customScheduleText.value)
+      if (expr.hasNext()) {
+        return true
+      } else {
+        customCronErrorMsg.value = 'This cron expression has no discernable next date.'
+        return false
+      }
+    } catch (err) {
+      if (err && typeof err === 'object' && 'message' in err && typeof err.message === 'string') {
+        customCronErrorMsg.value = err.message
+      }
+      return false
+    }
+  } else if (chosenScheduleType.value === ScheduleType.Monthly) {
+    return dayOfMonth.value >= 1 && dayOfMonth.value <= 28
+  }
+  return false
+}
+
 defineExpose({ show })
 </script>
 <template>
@@ -58,11 +128,30 @@ defineExpose({ show })
         then review and submit.
       </p>
       <p>
-        Each time a new draft is created from this template for recurring transactions, the new
+        Each time a new draft is created from this template, the new draft's timestamp will be set
-        draft's timestamp will be set to the current date and time.
+        to the current date and time.
+      </p>
+      <p class="font-italic font-size-small text-muted">
+        Note that Finnow processes recurring transactions at a regular interval, but to avoid
+        issues, please don't create recurring transactions whose schedule is less than hourly, as
+        these may be too frequent for the system to process reliably without skipping.
       </p>
       <AppForm>
+        <FormGroup>
+          <FormControl label="Type of Schedule">
+            <select v-model="chosenScheduleType">
+              <option :value="ScheduleType.Monthly">Monthly</option>
+              <option :value="ScheduleType.Custom">Custom</option>
+            </select>
+          </FormControl>
+        </FormGroup>
+
+        <div v-if="chosenScheduleType === ScheduleType.Monthly">
           <h3>Monthly Schedule</h3>
+          <p>
+            Schedule a recurring transaction to occur on the same calendar day every month. To
+            ensure that this schedule works all year, you must pick a day between the 1st and 28th.
+          </p>
           <FormGroup>
             <FormControl
               label="Day of Month"
@@ -72,15 +161,72 @@ defineExpose({ show })
                 type="number"
                 step="1"
                 min="1"
-              max="31"
+                max="28"
                 v-model="dayOfMonth"
+                style="max-width: 100px"
               />
             </FormControl>
           </FormGroup>
+        </div>
+
+        <div v-if="chosenScheduleType === ScheduleType.Custom">
+          <h3>Custom Schedule</h3>
+          <p>
+            Use a custom
+            <a
+              href="https://en.wikipedia.org/wiki/Cron"
+              target="_blank"
+              >Cron</a
+            >
+            expression to define when this recurring transaction should happen.
+          </p>
+          <FormGroup>
+            <FormControl label="Schedule Expression">
+              <input
+                type="text"
+                maxlength="64"
+                v-model="customScheduleText"
+                style="max-width: 300px"
+              />
+            </FormControl>
+          </FormGroup>
+          <p
+            v-if="customCronErrorMsg"
+            class="text-negative font-size-xsmall m0"
+          >
+            Failed to parse cron expression: {{ customCronErrorMsg }}
+          </p>
+          <p
+            v-if="customCronErrorMsg === null"
+            class="text-positive font-size-xsmall m0"
+          >
+            Cron expression is valid.
+          </p>
+        </div>
+
+        <!-- A little area at the bottom to show hypothetical upcoming dates with the chosen schedule: -->
+        <div v-if="inputIsValid">
+          <p class="font-size-small mb-0 font-italic">
+            Using the schedule you've chosen above, here are some upcoming dates at which this
+            recurring transaction will create a new draft.
+          </p>
+          <ul class="font-size-small mt-1">
+            <li
+              v-for="d in upcomingDates"
+              :key="d"
+            >
+              {{ d }}
+            </li>
+          </ul>
+        </div>
       </AppForm>
     </template>
     <template v-slot:buttons>
-      <AppButton @click="addRecurringTransaction()">Add</AppButton>
+      <AppButton
+        :disabled="!inputIsValid"
+        @click="addRecurringTransaction()"
+        >Add</AppButton
+      >
       <AppButton
         button-style="secondary"
         @click="modal?.close()"

web-app/src/pages/TransactionDraftPage.vue

@@ -20,7 +20,7 @@ import AppBadge from '@/components/common/AppBadge.vue'
 import ButtonBar from '@/components/common/ButtonBar.vue'
 import { useRoute, useRouter } from 'vue-router'
 import AddRecurringTransactionModal from '@/components/AddRecurringTransactionModal.vue'
-import { CronExpressionParser } from 'cron-parser'
+import cronstrue from 'cronstrue'
 
 const route = useRoute()
 const router = useRouter()
@@ -91,17 +91,7 @@ async function deleteRecurringTransaction(rt: RecurringTransactionResponse) {
 }
 
 function formatRecurringTransactionScheduleExpr(rt: RecurringTransactionResponse) {
-  const interval = CronExpressionParser.parse(rt.scheduleExpr)
+  return cronstrue.toString(rt.scheduleExpr)
-  const dayOfMonth = interval.fields.dayOfMonth.serialize().values[0] as number
-  return `The ${dayOfMonth}${getNumberAdjectiveSuffix(dayOfMonth)} day of every month`
-}
-
-function getNumberAdjectiveSuffix(n: number) {
-  const digit = n % 10
-  if (digit === 1) return 'st'
-  if (digit === 2) return 'nd'
-  if (digit === 3) return 'rd'
-  return 'th'
 }
 </script>
 <template>
@@ -151,7 +141,7 @@ function getNumberAdjectiveSuffix(n: number) {
       />
     </div>
 
-    <p>{{ draft.description }}</p>
+    <p style="white-space: pre-wrap">{{ draft.description }}</p>
 
     <div
       v-if="draft.creditedAccount"

web-app/src/pages/TransactionPage.vue

@@ -140,7 +140,7 @@ async function onVendorClicked() {
       />
     </div>
 
-    <p>{{ transaction.description }}</p>
+    <p style="white-space: pre-wrap">{{ transaction.description }}</p>
 
     <div
       v-if="transaction.creditedAccount"
@@ -229,7 +229,7 @@ async function onVendorClicked() {
       <AppButton
         icon="wrench"
         @click="
-          router.push(`/profiles/${getSelectedProfile(route)}/transactions/${transaction.id}/edit`)
+          router.push(`/profiles/${getSelectedProfile(route)}/transactions/${transaction?.id}/edit`)
         "
       >
         Edit

web-app/yarn.lock

@@ -1136,6 +1136,11 @@ cron-parser@^5.6.1:
   dependencies:
     luxon "^3.7.2"
 
+cronstrue@^3.24.0:
+  version "3.24.0"
+  resolved "https://registry.yarnpkg.com/cronstrue/-/cronstrue-3.24.0.tgz#24b8732897ddb08f533b997832ebec6ecaa5cc97"
+  integrity sha512-t/Ji3Ur2c/pzhIAWNwC0ftl3JAE4dLfCjAdZoTZXmPDZwcispnS1PaMcMS4OmIIXyIVouAz+yw+mfQiE3hz5OQ==
+
 cross-spawn@^7.0.6:
   version "7.0.6"
   resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz"