From 1080624e48ee87bfe2d17cb472f777720fa653f9 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 15:55:00 -0500 Subject: [PATCH 01/30] Fix last day missing in each interval --- src/shared/lib/accounts.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 09b25e3..50d47da 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -169,11 +169,7 @@ const fetchDailyBalancesForAccount = async ({ dateFilter: { type: 'CUSTOM', startDate: start.toISODate(), - // end is really the start of the next month, so subtract one day - endDate: - end < DateTime.now() - ? end.minus({ day: 1 }).toISODate() - : DateTime.now().toISODate(), + endDate: end < DateTime.now() ? end.toISODate() : DateTime.now().toISODate(), }, overrideApiKey, }) From e5165b5cf3c39e1ce6d88236d4687218c304f1ad Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 16:27:15 -0500 Subject: [PATCH 02/30] Use a generic for Message payload --- src/shared/hooks/useMessage.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/shared/hooks/useMessage.ts b/src/shared/hooks/useMessage.ts index 9392067..a30d0dd 100644 --- a/src/shared/hooks/useMessage.ts +++ b/src/shared/hooks/useMessage.ts @@ -13,13 +13,16 @@ export enum Action { DebugThrowError = 'DEBUG_THROW_ERROR', } -type Message = { action: Action; payload?: Record }; +export type Message> = { + action: Action; + payload?: TPayload +}; export const useMessageListener = >( action: Action, callback: (payload: TPayload) => void | Promise, ) => { - const listenerRef = useRef<(message: Message, sender: unknown, sendResponse: unknown) => void>(); + const listenerRef = useRef<(message: Message, sender: unknown, sendResponse: unknown) => void>(); useEffect(() => { if (listenerRef.current) { @@ -31,9 +34,9 @@ export const useMessageListener = >( if (message.action === action) { // eslint-disable-next-line no-prototype-builtins if (callback.hasOwnProperty('then')) { - await callback(message.payload as TPayload); + await callback(message.payload); } else { - callback(message.payload as TPayload); + callback(message.payload); } } From ad597ca3cafa35f817208fa97443d76082a92029 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 16:37:18 -0500 Subject: [PATCH 03/30] Improved typing for Mint API --- src/shared/lib/accounts.ts | 57 ++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 50d47da..1391d3e 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -15,12 +15,37 @@ import { withRateLimit, } from '@root/src/shared/lib/promises'; +export type AccountCategory = 'DEBT' | 'ASSET'; + +export type TrendType = 'DEBT' | 'ASSET' | 'INCOME' | 'EXPENSE'; + +export type ReportType = + | 'ASSETS_TIME' + | 'DEBTS_TIME' + | 'SPENDING_TIME' + | 'INCOME_TIME' + | 'NET_INCOME' + | 'NET_WORTH'; + +export type FixedDateFilter = + | 'LAST_7_DAYS' + | 'LAST_14_DAYS' + | 'THIS_MONTH' + | 'LAST_MONTH' + | 'LAST_3_MONTHS' + | 'LAST_6_MONTHS' + | 'LAST_12_MONTHS' + | 'THIS_YEAR' + | 'LAST_YEAR' + | 'ALL_TIME' + | 'CUSTOM'; + type TrendEntry = { amount: number; date: string; // this is determined by the type of report we fetch (DEBTS_TIME/ASSETS_TIME) // it will return different values if we decide to fetch more types of reports (e.g., SPENDING_TIME) - type: 'DEBT' | 'ASSET' | string; + type: TrendType; }; type TrendsResponse = { @@ -38,6 +63,28 @@ export type BalanceHistoryCallbackProgress = Parameters void | Promise; +const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { + BankAccount: 'ASSET', + CashAccount: 'ASSET', + CreditAccount: 'DEBT', + InsuranceAccount: 'ASSET', + InvestmentAccount: 'ASSET', + LoanAccount: 'DEBT', + RealEstateAccount: 'ASSET', + VehicleAccount: 'ASSET', + OtherPropertyAccount: 'ASSET', +} satisfies Record; + +type AccountType = keyof typeof ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE; + +type AccountsResponse = { + Account: { + type: AccountType; + id: string; + name: string; + }[]; +}; + /** * Use internal Mint "Trends" API to fetch account balance by month * for all time. @@ -327,14 +374,6 @@ const fetchTrends = ({ overrideApiKey, ); -type AccountsResponse = { - Account: { - type: string; - id: string; - name: string; - }[]; -}; - /** * Use internal Mint API to fetch all of user's accounts. */ From 40debad291295e22a5be6fe5b528ac23b7f893cc Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 16:42:45 -0500 Subject: [PATCH 04/30] Add fetchTrendAccounts to make sense of Mint trend state --- src/shared/lib/__tests__/accounts.test.ts | 27 +++++++++ src/shared/lib/accounts.ts | 74 +++++++++++++++++++++-- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index a96868f..4b3a935 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -5,6 +5,7 @@ import { fetchDailyBalancesForAllAccounts, fetchMonthlyBalancesForAccount, fetchNetWorthBalances, + fetchTrendAccounts, formatBalancesAsCSV, } from '../accounts'; import { DateTime } from 'luxon'; @@ -43,6 +44,32 @@ describe('fetchAccounts', () => { }); }); +describe('fetchTrendAccounts', () => { + it('excludes deselected accounts', async () => { + const allDebtAccounts = await fetchTrendAccounts({ + trend: { + reportType: 'DEBTS_TIME', + deselectedAccountIds: [], + fixedFilter: 'CUSTOM', + fromDate: '2020-01-01', + toDate: '2020-01-01', + }, + overrideApiKey: TEST_MINT_API_KEY, + }); + const accounts = await fetchTrendAccounts({ + trend: { + reportType: 'DEBTS_TIME', + deselectedAccountIds: ['43237333_2630847'], // a debt account + fixedFilter: 'CUSTOM', + fromDate: '2020-01-01', + toDate: '2020-01-01', + }, + overrideApiKey: TEST_MINT_API_KEY, + }); + expect(accounts.length).toEqual(allDebtAccounts.length - 1); + }); +}); + describe('formatBalancesAsCSV', () => { it('includes account name if provied', () => { const result = formatBalancesAsCSV( diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 1391d3e..4d2234d 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -53,6 +53,20 @@ type TrendsResponse = { // there's more here... }; +/** State of user selections on the Mint Trends page */ +export type TrendState = { + /** Use with {@link deselectedAccountIds } to figure out which accounts to include in the trend */ + reportType: ReportType; + /** All accounts eligible for the {@link reportType} that are NOT selected */ + deselectedAccountIds?: string[]; + /** Semantic representation of the {@link fromDate} {@link toDate} range */ + fixedFilter: FixedDateFilter; + /** ISO start date */ + fromDate: string; + /** ISO enddate */ + toDate: string; +}; + export type BalanceHistoryProgressCallback = (progress: { completedAccounts: number; totalAccounts: number; @@ -77,6 +91,8 @@ const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { type AccountType = keyof typeof ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE; +type AccountTypeFilter = (accountType: AccountType) => boolean; + type AccountsResponse = { Account: { type: AccountType; @@ -85,6 +101,41 @@ type AccountsResponse = { }[]; }; +type FetchAccountsOptions = { + offset?: number; + limit?: number; + overrideApiKey?: string; +}; + +/** + * Allows filtering accounts since the API does not seem to allow negated account ID queries, yet + * also does not expose the selected accounts (see {@link deselectedAccountIds}). + */ +export const getAccountTypeFilterForTrend = (trend: TrendState): AccountTypeFilter => { + switch (trend.reportType) { + case 'INCOME_TIME': + case 'ASSETS_TIME': + return (type) => ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE[type] === 'ASSET'; + case 'DEBTS_TIME': + case 'SPENDING_TIME': + return (type) => ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE[type] === 'DEBT'; + case 'NET_INCOME': + return (type) => + type !== 'VehicleAccount' && + type !== 'RealEstateAccount' && + type !== 'OtherPropertyAccount' && + type !== 'InsuranceAccount' && + // Cash account does not appear in the dropdown; it is included only when All Accounts are + // selected not when specific accounts are selected + // TODO: verify this isn't just on idpaterson's account + (type !== 'CashAccount' || !trend.deselectedAccountIds?.length); + case 'NET_WORTH': + return () => true; + default: + throw new Error(`Unsupported report type: ${trend.reportType}`); + } +}; + /** * Use internal Mint "Trends" API to fetch account balance by month * for all time. @@ -381,11 +432,7 @@ export const fetchAccounts = async ({ offset = 0, limit = 1000, // mint default overrideApiKey, -}: { - offset?: number; - limit?: number; - overrideApiKey?: string; -}) => { +}: FetchAccountsOptions) => { const response = await makeMintApiRequest( `/pfm/v1/accounts?offset=${offset}&limit=${limit}`, { @@ -398,6 +445,23 @@ export const fetchAccounts = async ({ return accounts; }; +/** + * Make sense of the {@link TrendState} by determining which accounts are selected. + */ +export const fetchTrendAccounts = async ({ + trend, + ...options +}: { + trend: TrendState; +} & FetchAccountsOptions) => { + const allAccounts = await fetchAccounts({ offset: 0, ...options }); + const accountTypeFilter = getAccountTypeFilterForTrend(trend); + + return allAccounts.filter( + ({ id, type }) => accountTypeFilter(type) && !trend.deselectedAccountIds?.includes(id), + ); +}; + export const formatBalancesAsCSV = (balances: TrendEntry[], accountName?: string) => { const header = ['Date', 'Amount', accountName && 'Account Name'].filter(Boolean); const maybeAccountColumn: [string?] = accountName ? [accountName] : []; From a342b1aa94770dddafeaf26ae7cac24951bd1742 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 16:47:32 -0500 Subject: [PATCH 05/30] Added fetchDailyBalancesForTrend --- src/shared/lib/accounts.ts | 61 +++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 4d2234d..ee229df 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -75,8 +75,19 @@ export type BalanceHistoryProgressCallback = (progress: { export type BalanceHistoryCallbackProgress = Parameters[0]; +export type TrendBalanceHistoryProgressCallback = (progress: { + completePercentage: number; +}) => void | Promise; + +export type TrendBalanceHistoryCallbackProgress = Parameters[0]; + type ProgressCallback = (progress: { complete: number; total: number }) => void | Promise; +type AccountIdFilter = { + type: 'AccountIdFilter'; + accountId: string; +}; + const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { BankAccount: 'ASSET', CashAccount: 'ASSET', @@ -233,9 +244,9 @@ const fetchIntervalsForAccountHistory = async ({ }; /** - * Fetch balance history for each month for an account. + * Fetch balance history for each month for one or more accounts. */ -const fetchDailyBalancesForAccount = async ({ +const fetchDailyBalances = async ({ periods, accountId, reportType, @@ -243,8 +254,9 @@ const fetchDailyBalancesForAccount = async ({ onProgress, }: { periods: Interval[]; - accountId: string; + accountId: string | string[]; reportType: string; + excludeSpecifiedAccounts?: boolean; overrideApiKey?: string; onProgress?: ProgressCallback; }) => { @@ -252,6 +264,7 @@ const fetchDailyBalancesForAccount = async ({ throw new Error('Invalid report type.'); } + const accountIds = Array.isArray(accountId) ? accountId : [accountId]; const counter = { count: 0, }; @@ -263,7 +276,7 @@ const fetchDailyBalancesForAccount = async ({ withRetry(() => fetchTrends({ reportType, - filters: [makeAccountIdFilter(accountId)], + filters: accountIds.map(makeAccountIdFilter), dateFilter: { type: 'CUSTOM', startDate: start.toISODate(), @@ -327,7 +340,7 @@ export const fetchDailyBalancesForAllAccounts = async ({ ({ accountId, accountName, periods, reportType }, accountIndex) => async () => { const balances = await withDefaultOnError([])( - fetchDailyBalancesForAccount({ + fetchDailyBalances({ accountId, periods, reportType, @@ -363,6 +376,40 @@ export const fetchDailyBalancesForAllAccounts = async ({ return balancesByAccount; }; +export const fetchDailyBalancesForTrend = async ({ + trend, + onProgress, + overrideApiKey, +}: { + trend: TrendState; + onProgress?: TrendBalanceHistoryProgressCallback; + overrideApiKey?: string; +}) => { + const accounts = await withRetry(() => fetchTrendAccounts({ trend, overrideApiKey })); + const { reportType, fromDate, toDate } = trend; + const interval = Interval.fromDateTimes(DateTime.fromISO(fromDate), DateTime.fromISO(toDate)); + const periods = interval.splitBy({ + days: MINT_DAILY_TRENDS_MAX_DAYS, + }) as Interval[]; + + // fetch one account at a time so we don't hit the rate limit + const balances = await withDefaultOnError([])( + fetchDailyBalances({ + accountId: accounts.map(({ id }) => id), + periods, + reportType, + overrideApiKey, + onProgress: ({ complete }) => { + onProgress?.({ + completePercentage: complete / periods.length, + }); + }, + }), + ); + + return balances; +}; + /** * Use internal Mint API to fetch net worth history. Return list of * balances for type: "ASSET" and type: "DEBT" for each month. @@ -396,7 +443,7 @@ const fetchTrends = ({ overrideApiKey, }: { reportType: string; - filters?: Record[]; + filters?: AccountIdFilter[]; dateFilter?: Record; offset?: number; limit?: number; @@ -479,7 +526,7 @@ export const formatBalancesAsCSV = (balances: TrendEntry[], accountName?: string return formatCSV([header, ...rows]); }; -const makeAccountIdFilter = (accountId: string) => ({ +const makeAccountIdFilter = (accountId: string): AccountIdFilter => ({ type: 'AccountIdFilter', accountId, }); From 5cf7bfd18f72443de0dcd06ffb053b401784c1e8 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 16:48:37 -0500 Subject: [PATCH 06/30] Support CSV export of net worth and income --- src/pages/background/index.ts | 5 ++- src/shared/lib/accounts.ts | 77 +++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index a6d10d0..c2a5990 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -170,7 +170,10 @@ const handleDownloadAllAccountBalances = async (sendResponse: () => void) => { const seenCount = (seenAccountNames[accountName] = (seenAccountNames[accountName] || 0) + 1); // If there are multiple accounts with the same name, export both with distinct filenames const disambiguation = seenCount > 1 ? ` (${seenCount - 1})` : ''; - zip.file(`${accountName}${disambiguation}.csv`, formatBalancesAsCSV(balances, accountName)); + zip.file( + `${accountName}${disambiguation}.csv`, + formatBalancesAsCSV({ balances, accountName }), + ); }); const zipFile = await zip.generateAsync({ type: 'base64' }); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index ee229df..3e78bba 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -46,6 +46,8 @@ type TrendEntry = { // this is determined by the type of report we fetch (DEBTS_TIME/ASSETS_TIME) // it will return different values if we decide to fetch more types of reports (e.g., SPENDING_TIME) type: TrendType; + /** Represents the negative amount in net income/worth trends. Calculated, not from Mint */ + inverseAmount?: number; }; type TrendsResponse = { @@ -509,18 +511,77 @@ export const fetchTrendAccounts = async ({ ); }; -export const formatBalancesAsCSV = (balances: TrendEntry[], accountName?: string) => { - const header = ['Date', 'Amount', accountName && 'Account Name'].filter(Boolean); - const maybeAccountColumn: [string?] = accountName ? [accountName] : []; +/** + * Merges paired API response into a single amount/inverseAmount entry. + * + * The API response does not include an entry for zero inverse amounts, but there is always a + * positive amount for each date in the trend. + */ +const zipTrendEntries = (trendEntries: TrendEntry[]) => { + const mergedTrendEntries: TrendEntry[] = []; + for (let i = 0; i < trendEntries.length; i += 1) { + const trendEntry = trendEntries[i]; + const nextTrendEntry = trendEntries[i + 1]; + let inverseAmount = 0; + // If the next entry is the inverse of this one, consume it + if (nextTrendEntry && nextTrendEntry.type !== trendEntry.type) { + i += 1; + inverseAmount = nextTrendEntry.amount; + } + mergedTrendEntries.push({ + ...trendEntry, + inverseAmount, + }); + } + return mergedTrendEntries; +}; + +export const formatBalancesAsCSV = ({ + balances, + accountName, + reportType, +}: { + balances: TrendEntry[]; + accountName?: string; + reportType?: ReportType; +}) => { + const header = ['Date']; + const columns: (keyof TrendEntry | ((trendEntry: TrendEntry) => string | number))[] = ['date']; + let trendEntries = balances; + // net income/worth reports have two rows per date, CSV needs one row with two columns + if (reportType?.startsWith('NET_')) { + // merge the positive and negative balances into one row + trendEntries = zipTrendEntries(balances); + if (reportType === 'NET_INCOME') { + header.push('Income', 'Expenses'); + } else { + header.push('Assts', 'Debts'); + } + header.push('Net'); + columns.push('amount', 'inverseAmount', (trendEntry) => + (trendEntry.amount - trendEntry.inverseAmount).toFixed(2), + ); + } else { + header.push('Amount'); + columns.push('amount'); + } + const maybeAccountColumn: [string?] = []; + if (accountName) { + header.push('Account Name'); + maybeAccountColumn.push(accountName); + } // remove zero balances from the end of the report leaving just the first row if all are zero - const rows = balances.reduceRight( - (acc, { date, amount }, index) => { - if (acc.length || amount !== 0 || index === 0) { - acc.unshift([date, amount, ...maybeAccountColumn]); + const rows = trendEntries.reduceRight( + (acc, trendEntry, index) => { + if (acc.length || trendEntry.amount !== 0 || trendEntry.inverseAmount || index === 0) { + acc.unshift([ + ...columns.map((col) => (typeof col === 'function' ? col(trendEntry) : trendEntry[col])), + ...maybeAccountColumn, + ]); } return acc; }, - [] as [string, number, string?][], + [] as (string | number)[][], ); return formatCSV([header, ...rows]); From 44d1ceb7dbaf38005a71d4b10ed6d4e7ee5f4723 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 15:22:46 -0500 Subject: [PATCH 07/30] Add trendStorage --- src/shared/storages/trendStorage.ts | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/shared/storages/trendStorage.ts diff --git a/src/shared/storages/trendStorage.ts b/src/shared/storages/trendStorage.ts new file mode 100644 index 0000000..df03b56 --- /dev/null +++ b/src/shared/storages/trendStorage.ts @@ -0,0 +1,31 @@ +import { TrendBalanceHistoryCallbackProgress, TrendState } from '@root/src/shared/lib/accounts'; +import { createStorage, StorageType } from '@src/shared/storages/base'; + +export enum TrendDownloadStatus { + Idle = 'idle', + Loading = 'loading', + Success = 'success', + Error = 'error', +} + +type State = { + status: TrendDownloadStatus; + trend: TrendState; + progress?: TrendBalanceHistoryCallbackProgress; +}; + +const trendStorage = createStorage( + 'trend', + { + status: TrendDownloadStatus.Idle, + trend: null, + progress: { + completePercentage: 0, + }, + }, + { + storageType: StorageType.Local, + }, +); + +export default trendStorage; From ab37de4285f8e5109a6096fa4531a9012c77feaf Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 16:25:21 -0500 Subject: [PATCH 08/30] Update unit tests for new call signatures --- src/shared/lib/__tests__/accounts.test.ts | 156 ++++++++++++---------- 1 file changed, 82 insertions(+), 74 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index 4b3a935..a065d50 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -72,13 +72,13 @@ describe('fetchTrendAccounts', () => { describe('formatBalancesAsCSV', () => { it('includes account name if provied', () => { - const result = formatBalancesAsCSV( - [ - { amount: 123.45, date: '2021-01-01', type: '' }, - { amount: 234.56, date: '2021-02-01', type: '' }, + const result = formatBalancesAsCSV({ + balances: [ + { amount: 123.45, date: '2021-01-01', type: 'ASSET' }, + { amount: 234.56, date: '2021-02-01', type: 'ASSET' }, ], - `Mason's Account`, - ); + accountName: `Mason's Account`, + }); expect(result).toEqual(`"Date","Amount","Account Name" "2021-01-01","123.45","Mason's Account" "2021-02-01","234.56","Mason's Account" @@ -86,10 +86,12 @@ describe('formatBalancesAsCSV', () => { }); it('does not include account name if not provided', () => { - const result = formatBalancesAsCSV([ - { amount: 123.45, date: '2021-01-01', type: '' }, - { amount: 234.56, date: '2021-02-01', type: '' }, - ]); + const result = formatBalancesAsCSV({ + balances: [ + { amount: 123.45, date: '2021-01-01', type: 'ASSET' }, + { amount: 234.56, date: '2021-02-01', type: 'ASSET' }, + ], + }); expect(result).toEqual(`"Date","Amount" "2021-01-01","123.45" "2021-02-01","234.56" @@ -97,41 +99,45 @@ describe('formatBalancesAsCSV', () => { }); it('converts undefined balances to empty string', () => { - const result = formatBalancesAsCSV([ - { - amount: undefined, - date: '2020-01-01', - type: '', - }, - ]); + const result = formatBalancesAsCSV({ + balances: [ + { + amount: undefined, + date: '2020-01-01', + type: 'ASSET', + }, + ], + }); expect(result).toEqual(`"Date","Amount" "2020-01-01","" `); }); it('trims trailing zero balances', () => { - const result = formatBalancesAsCSV([ - { - amount: 123.45, - date: '2020-01-01', - type: '', - }, - { - amount: 234.56, - date: '2020-01-02', - type: '', - }, - { - amount: 0, - date: '2020-01-03', - type: '', - }, - { - amount: 0, - date: '2020-01-04', - type: '', - }, - ]); + const result = formatBalancesAsCSV({ + balances: [ + { + amount: 123.45, + date: '2020-01-01', + type: 'ASSET', + }, + { + amount: 234.56, + date: '2020-01-02', + type: 'ASSET', + }, + { + amount: 0, + date: '2020-01-03', + type: 'ASSET', + }, + { + amount: 0, + date: '2020-01-04', + type: 'ASSET', + }, + ], + }); expect(result).toEqual(`"Date","Amount" "2020-01-01","123.45" "2020-01-02","234.56" @@ -139,23 +145,25 @@ describe('formatBalancesAsCSV', () => { }); it('leaves one row if all balances are zero', () => { - const result = formatBalancesAsCSV([ - { - amount: 0, - date: '2020-01-01', - type: '', - }, - { - amount: 0, - date: '2020-01-02', - type: '', - }, - { - amount: 0, - date: '2020-01-03', - type: '', - }, - ]); + const result = formatBalancesAsCSV({ + balances: [ + { + amount: 0, + date: '2020-01-01', + type: 'ASSET', + }, + { + amount: 0, + date: '2020-01-02', + type: 'ASSET', + }, + { + amount: 0, + date: '2020-01-03', + type: 'ASSET', + }, + ], + }); expect(result).toEqual(`"Date","Amount" "2020-01-01","0" `); @@ -175,16 +183,16 @@ describe('fetchDailyBalancesForAllAccounts', () => { describe('calculateIntervalForAccountHistory', () => { it('starts at the first day of the first month with history', () => { const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 5, type: '' }, - { date: '2023-02-28', amount: 10, type: '' }, + { date: '2023-01-31', amount: 5, type: 'ASSET' }, + { date: '2023-02-28', amount: 10, type: 'ASSET' }, ]); expect(result.start.toISODate()).toBe('2023-01-01'); }); it('ends today for nonzero balances', () => { const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 5, type: '' }, - { date: '2023-02-28', amount: 10, type: '' }, + { date: '2023-01-31', amount: 5, type: 'ASSET' }, + { date: '2023-02-28', amount: 10, type: 'ASSET' }, ]); expect(result.end.toISODate()).toBe(DateTime.now().toISODate()); }); @@ -192,28 +200,28 @@ describe('calculateIntervalForAccountHistory', () => { it('ends today even if the data goes beyond today', () => { const nextMonth = DateTime.now().plus({ month: 1 }).endOf('month').toISODate(); const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 5, type: '' }, - { date: nextMonth, amount: 10, type: '' }, + { date: '2023-01-31', amount: 5, type: 'ASSET' }, + { date: nextMonth, amount: 10, type: 'ASSET' }, ]); expect(result.end.toISODate()).toBe(DateTime.now().toISODate()); }); it('ends 1 month after the last historic nonzero monthly balance', () => { const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 5, type: '' }, - { date: '2023-02-28', amount: 10, type: '' }, - { date: '2023-03-31', amount: 0, type: '' }, + { date: '2023-01-31', amount: 5, type: 'ASSET' }, + { date: '2023-02-28', amount: 10, type: 'ASSET' }, + { date: '2023-03-31', amount: 0, type: 'ASSET' }, ]); expect(result.end.toISODate()).toBe('2023-03-31'); }); it('ends 1 month after the last historic nonzero monthly balance', () => { const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 5, type: '' }, - { date: '2023-02-28', amount: 10, type: '' }, - { date: '2023-03-31', amount: 0, type: '' }, - { date: '2023-04-30', amount: 0, type: '' }, - { date: '2023-05-31', amount: 0, type: '' }, + { date: '2023-01-31', amount: 5, type: 'ASSET' }, + { date: '2023-02-28', amount: 10, type: 'ASSET' }, + { date: '2023-03-31', amount: 0, type: 'ASSET' }, + { date: '2023-04-30', amount: 0, type: 'ASSET' }, + { date: '2023-05-31', amount: 0, type: 'ASSET' }, ]); expect(result.end.toISODate()).toBe('2023-03-31'); }); @@ -222,10 +230,10 @@ describe('calculateIntervalForAccountHistory', () => { // No need for a special case here, the interval is 2 months because we always add 1 month for // safety to the last month worth including in the report. const result = calculateIntervalForAccountHistory([ - { date: '2023-01-31', amount: 0, type: '' }, - { date: '2023-02-28', amount: 0, type: '' }, - { date: '2023-03-31', amount: 0, type: '' }, - { date: '2023-04-30', amount: 0, type: '' }, + { date: '2023-01-31', amount: 0, type: 'ASSET' }, + { date: '2023-02-28', amount: 0, type: 'ASSET' }, + { date: '2023-03-31', amount: 0, type: 'ASSET' }, + { date: '2023-04-30', amount: 0, type: 'ASSET' }, ]); expect(result.start.toISODate()).toBe('2023-01-01'); expect(result.end.toISODate()).toBe('2023-02-28'); From 276a4ca47d4109e9180ec2aa7e41735500e7d8bb Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 16:36:07 -0500 Subject: [PATCH 09/30] Add GetTrendState action to expose current trend --- src/pages/background/index.ts | 37 ++++++++++++++++++++++++++-- src/shared/constants/error.ts | 1 + src/shared/hooks/useMessage.ts | 1 + src/shared/lib/trends.ts | 44 ++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/shared/lib/trends.ts diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index c2a5990..bb1b117 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -1,6 +1,6 @@ import { ResponseStatus } from '@root/src/pages/popup/Popup'; import { ErrorCode } from '@root/src/shared/constants/error'; -import { Action } from '@root/src/shared/hooks/useMessage'; +import { Action, Message } from '@root/src/shared/hooks/useMessage'; import { fetchDailyBalancesForAllAccounts, formatBalancesAsCSV, @@ -20,6 +20,8 @@ import reloadOnUpdate from 'virtual:reload-on-update-in-background-script'; import 'webextension-polyfill'; import * as Sentry from '@sentry/browser'; +import trendStorage from '../../shared/storages/trendStorage'; +import { getCurrentTrendState } from '../../shared/lib/trends'; // @ts-ignore - https://github.com/getsentry/sentry-javascript/issues/5289#issuecomment-1368705821 Sentry.WINDOW.document = { @@ -49,7 +51,7 @@ reloadOnUpdate('pages/background'); const THROTTLE_INTERVAL_MS = 200; -chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { +chrome.runtime.onMessage.addListener((message: Message, sender, sendResponse) => { if (sender.tab?.url.startsWith('chrome://')) { return true; } @@ -65,6 +67,8 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { handlePopupOpened(sendResponse); } else if (message.action === Action.GetMintApiKey) { handleMintAuthentication(sendResponse); + } else if (message.action === Action.GetTrendState) { + handleGetTrendState(sendResponse); } else if (message.action === Action.DownloadTransactions) { handleTransactionsDownload(sendResponse); } else if (message.action === Action.DownloadAllAccountBalances) { @@ -125,6 +129,35 @@ function getMintApiKey() { return window.__shellInternal?.appExperience?.appApiKey; } +const handleGetTrendState = async (sendResponse: (args: unknown) => void) => { + const [activeMintTab] = await chrome.tabs.query({ + active: true, + url: 'https://mint.intuit.com/*', + }); + + // No active Mint tab, return early + if (!activeMintTab) { + sendResponse({ success: false, error: ErrorCode.MintTabNotFound }); + return; + } + + // Get the trend state from the page + const response = await chrome.scripting.executeScript({ + target: { tabId: activeMintTab.id }, + world: 'MAIN', + func: getCurrentTrendState, + }); + + const [{ result: trend }] = response; + + await trendStorage.patch({ trend }); + if (trend) { + sendResponse({ success: true, trend }); + } else { + sendResponse({ success: false, error: ErrorCode.MintTrendStateNotFound }); + } +}; + const handleTransactionsDownload = async (sendResponse: (args: unknown) => void) => { const totalTransactionCount = await fetchTransactionsTotalCount(); diff --git a/src/shared/constants/error.ts b/src/shared/constants/error.ts index 2d0b3d4..108a29d 100644 --- a/src/shared/constants/error.ts +++ b/src/shared/constants/error.ts @@ -1,4 +1,5 @@ export enum ErrorCode { MintTabNotFound = 'ERR_MINT_NOT_FOUND', MintApiKeyNotFound = 'ERR_MINT_API_KEY_NOT_FOUND', + MintTrendStateNotFound = 'ERR_MINT_TREND_STATE_NOT_FOUND', } diff --git a/src/shared/hooks/useMessage.ts b/src/shared/hooks/useMessage.ts index a30d0dd..ff59c8b 100644 --- a/src/shared/hooks/useMessage.ts +++ b/src/shared/hooks/useMessage.ts @@ -3,6 +3,7 @@ import { useRef, useEffect, useCallback } from 'react'; export enum Action { PopupOpened = 'POPUP_OPENED', GetMintApiKey = 'GET_MINT_API_KEY', + GetTrendState = 'GET_TREND_STATE', // Sent by the button in the popup to start downloading transactions RequestTransactionsDownload = 'REQUEST_TRANSACTIONS_DOWNLOAD', DownloadTransactions = 'DOWNLOAD_TRANSACTIONS', diff --git a/src/shared/lib/trends.ts b/src/shared/lib/trends.ts new file mode 100644 index 0000000..f0b4071 --- /dev/null +++ b/src/shared/lib/trends.ts @@ -0,0 +1,44 @@ +import { TrendType, TrendState, ReportType } from '../../shared/lib/accounts'; + +/** + * State for the most recent trend according to localStorage, does not need to be visible + * + * This function must be executed in the context of the Mint tab and therefore must be + * self-contained. + */ +export const getCurrentTrendState = () => { + if (window.location.pathname.startsWith('/trends')) { + try { + const CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY = 'trends-state'; + const currentTrendType = localStorage.getItem( + CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY, + ) as TrendType; + const trendState = JSON.parse( + localStorage.getItem(`${CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY}-${currentTrendType}`) || + 'null', + ) as TrendState; + console.log('trendState', trendState); + + return trendState; + } catch (e) { + // ignore + } + } + return null; +}; + +/** Whether the extension can generate daily balances for the active trend. */ +export const isSupportedTrendReport = (type: ReportType) => { + switch (type) { + case 'ASSETS_TIME': + case 'DEBTS_TIME': + case 'INCOME_TIME': + case 'SPENDING_TIME': + case 'NET_INCOME': + case 'NET_WORTH': + return true; + default: + // by tag, by category, by type, etc. + return false; + } +}; From 627778b6c5752b9389cebf84ed145abc1eafe364 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 16:47:57 -0500 Subject: [PATCH 10/30] Add DownloadTrendBalances action --- src/pages/background/index.ts | 44 +++++++++++++++++++++++++++++++++- src/shared/hooks/useMessage.ts | 7 ++++-- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index bb1b117..0dcffdc 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -5,6 +5,8 @@ import { fetchDailyBalancesForAllAccounts, formatBalancesAsCSV, BalanceHistoryCallbackProgress, + fetchDailyBalancesForTrend, + TrendBalanceHistoryCallbackProgress, } from '@root/src/shared/lib/accounts'; import { throttle } from '@root/src/shared/lib/events'; import stateStorage from '@root/src/shared/storages/stateStorage'; @@ -20,7 +22,7 @@ import reloadOnUpdate from 'virtual:reload-on-update-in-background-script'; import 'webextension-polyfill'; import * as Sentry from '@sentry/browser'; -import trendStorage from '../../shared/storages/trendStorage'; +import trendStorage, { TrendDownloadStatus } from '../../shared/storages/trendStorage'; import { getCurrentTrendState } from '../../shared/lib/trends'; // @ts-ignore - https://github.com/getsentry/sentry-javascript/issues/5289#issuecomment-1368705821 @@ -73,6 +75,8 @@ chrome.runtime.onMessage.addListener((message: Message, sender, sendResponse) => handleTransactionsDownload(sendResponse); } else if (message.action === Action.DownloadAllAccountBalances) { handleDownloadAllAccountBalances(sendResponse); + } else if (message.action === Action.DownloadTrendBalances) { + handleDownloadTrendBalances(sendResponse); } else if (message.action === Action.DebugThrowError) { throw new Error('Debug error'); } else { @@ -228,6 +232,40 @@ const handleDownloadAllAccountBalances = async (sendResponse: () => void) => { } }; +/** Download daily balances for the specified trend. */ +const handleDownloadTrendBalances = async (sendResponse: () => void) => { + try { + const throttledSendDownloadTrendBalancesProgress = throttle( + sendDownloadTrendBalancesProgress, + THROTTLE_INTERVAL_MS, + ); + + const { trend } = await trendStorage.get(); + await trendStorage.set({ + trend, + status: TrendDownloadStatus.Loading, + progress: { completePercentage: 0 }, + }); + const balances = await fetchDailyBalancesForTrend({ + trend, + onProgress: throttledSendDownloadTrendBalancesProgress, + }); + const { reportType } = trend; + const csv = formatBalancesAsCSV({ balances, reportType }); + + chrome.downloads.download({ + url: `data:text/csv,${csv}`, + filename: 'mint-trend-daily-balances.csv', + }); + + await trendStorage.patch({ status: TrendDownloadStatus.Success }); + } catch (e) { + await trendStorage.patch({ status: TrendDownloadStatus.Error }); + } finally { + sendResponse(); + } +}; + /** * Updates both the state storage and sends a message with the current progress, * so the popup can update the UI and we have a state to restore from if the @@ -236,3 +274,7 @@ const handleDownloadAllAccountBalances = async (sendResponse: () => void) => { const sendDownloadBalancesProgress = async (payload: BalanceHistoryCallbackProgress) => { await accountStorage.patch({ progress: payload }); }; + +const sendDownloadTrendBalancesProgress = async (payload: TrendBalanceHistoryCallbackProgress) => { + await trendStorage.patch({ progress: payload }); +}; diff --git a/src/shared/hooks/useMessage.ts b/src/shared/hooks/useMessage.ts index ff59c8b..19ac3eb 100644 --- a/src/shared/hooks/useMessage.ts +++ b/src/shared/hooks/useMessage.ts @@ -10,20 +10,23 @@ export enum Action { DownloadAllAccountBalances = 'DOWNLOAD_ALL_ACCOUNT_BALANCES', DownloadBalancesProgress = 'DOWNLOAD_BALANCES_PROGRESS', DownloadBalancesComplete = 'DOWNLOAD_BALANCES_COMPLETE', + DownloadTrendBalances = 'DOWNLOAD_TREND_BALANCES', + DownloadTrendBalancesProgress = 'DOWNLOAD_TREND_BALANCES_PROGRESS', // Debug actions DebugThrowError = 'DEBUG_THROW_ERROR', } export type Message> = { action: Action; - payload?: TPayload + payload?: TPayload; }; export const useMessageListener = >( action: Action, callback: (payload: TPayload) => void | Promise, ) => { - const listenerRef = useRef<(message: Message, sender: unknown, sendResponse: unknown) => void>(); + const listenerRef = + useRef<(message: Message, sender: unknown, sendResponse: unknown) => void>(); useEffect(() => { if (listenerRef.current) { From 4953eb1394f73c9364ce7a33d295c78acef701c7 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 16:48:25 -0500 Subject: [PATCH 11/30] Disable export of filtered trends --- src/shared/lib/trends.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/shared/lib/trends.ts b/src/shared/lib/trends.ts index f0b4071..4f6c1a4 100644 --- a/src/shared/lib/trends.ts +++ b/src/shared/lib/trends.ts @@ -7,7 +7,12 @@ import { TrendType, TrendState, ReportType } from '../../shared/lib/accounts'; * self-contained. */ export const getCurrentTrendState = () => { - if (window.location.pathname.startsWith('/trends')) { + if ( + // disable when not viewing the Trends page + window.location.pathname.startsWith('/trends') && + // disable when filtered by category, tag, etc. because this filter is not in the trend state + !document.querySelector('[data-automation-id="filter-chip"]') + ) { try { const CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY = 'trends-state'; const currentTrendType = localStorage.getItem( @@ -17,7 +22,6 @@ export const getCurrentTrendState = () => { localStorage.getItem(`${CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY}-${currentTrendType}`) || 'null', ) as TrendState; - console.log('trendState', trendState); return trendState; } catch (e) { From 5212d7fcab1ceda079e4b145067e7cf7800840ac Mon Sep 17 00:00:00 2001 From: idpaterson Date: Fri, 17 Nov 2023 19:37:36 -0500 Subject: [PATCH 12/30] formatBalancesAsCSV unit tests for net income --- src/shared/lib/__tests__/accounts.test.ts | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index a065d50..8c6ff7f 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -166,6 +166,110 @@ describe('formatBalancesAsCSV', () => { }); expect(result).toEqual(`"Date","Amount" "2020-01-01","0" +`); + }); + + it('handles net income balances', () => { + const result = formatBalancesAsCSV({ + reportType: 'NET_INCOME', + balances: [ + { + amount: 0, + date: '2020-01-01', + type: 'INCOME', + }, + { + amount: 123.45, + date: '2020-01-01', + type: 'EXPENSE', + }, + { + amount: 43.21, + date: '2020-01-02', + type: 'INCOME', + }, + { + amount: 234.56, + date: '2020-01-03', + type: 'INCOME', + }, + ], + }); + expect(result).toEqual(`"Date","Income","Expenses","Net" +"2020-01-01","0","123.45","-123.45" +"2020-01-02","43.21","0","43.21" +"2020-01-03","234.56","0","234.56" +`); + }); + + it('trims trailing zero net income balances', () => { + const result = formatBalancesAsCSV({ + reportType: 'NET_INCOME', + balances: [ + { + amount: 0, + date: '2020-01-01', + type: 'INCOME', + }, + { + amount: 123.45, + date: '2020-01-01', + type: 'EXPENSE', + }, + { + amount: 0, + date: '2020-01-02', + type: 'INCOME', + }, + { + amount: 0, + date: '2020-01-03', + type: 'INCOME', + }, + ], + }); + expect(result).toEqual(`"Date","Income","Expenses","Net" +"2020-01-01","0","123.45","-123.45" +`); + }); + + it('fixes floating point subtraction in net income balances', () => { + const result = formatBalancesAsCSV({ + reportType: 'NET_INCOME', + balances: [ + { + amount: 123.45, + date: '2020-01-01', + type: 'INCOME', + }, + { + amount: 12.35, + date: '2020-01-01', + type: 'EXPENSE', + }, + ], + }); + // Not 111.10000000000001 + expect(result).toEqual(`"Date","Income","Expenses","Net" +"2020-01-01","123.45","12.35","111.10" +`); + }); + + it('represents zero in the Net column as 0.00', () => { + // as a consequence of toFixed(2) + const result = formatBalancesAsCSV({ + reportType: 'NET_INCOME', + balances: [ + { + amount: 0, + date: '2020-01-01', + type: 'INCOME', + }, + ], + }); + // Not 111.10000000000001 + expect(result).toEqual(`"Date","Income","Expenses","Net" +"2020-01-01","0","0","0.00" `); }); }); From 595fcd30a8631ba2bafca3d73b67939f905da66e Mon Sep 17 00:00:00 2001 From: idpaterson Date: Sat, 18 Nov 2023 09:01:54 -0500 Subject: [PATCH 13/30] Added tests for getAccountTypeFilterForTrend() --- src/shared/lib/__tests__/accounts.test.ts | 70 +++++++++++++++++++++++ src/shared/lib/accounts.ts | 17 ++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index 8c6ff7f..618ae9c 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -1,5 +1,10 @@ import { TEST_MINT_API_KEY } from '@root/src/shared/lib/constants'; import { + ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE, + AccountCategory, + AccountType, + PROPERTY_ACCOUNT_TYPES, + TrendState, calculateIntervalForAccountHistory, fetchAccounts, fetchDailyBalancesForAllAccounts, @@ -7,6 +12,7 @@ import { fetchNetWorthBalances, fetchTrendAccounts, formatBalancesAsCSV, + getAccountTypeFilterForTrend, } from '../accounts'; import { DateTime } from 'luxon'; @@ -343,3 +349,67 @@ describe('calculateIntervalForAccountHistory', () => { expect(result.end.toISODate()).toBe('2023-02-28'); }); }); + +describe('getAccountTypeFilterForTrend', () => { + const allAccountTypes: AccountType[] = []; + const accountTypesByCatgeory = Object.entries(ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE).reduce( + (acc, [accountType, category]: [AccountType, AccountCategory]) => { + allAccountTypes.push(accountType); + acc[category] = acc[category] || []; + acc[category].push(accountType); + return acc; + }, + {} as Record, + ); + const baseTrend: TrendState = { + reportType: 'ASSETS_TIME', + fixedFilter: 'CUSTOM', + fromDate: '2020-01-01', + toDate: '2020-01-01', + }; + + it('should exclude debt accounts for ASSETS_TIME', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'ASSETS_TIME' }); + for (const accountType of accountTypesByCatgeory.ASSET) { + expect(filter(accountType)).toBe(true); + } + for (const accountType of accountTypesByCatgeory.DEBT) { + expect(filter(accountType)).toBe(false); + } + }); + + it('should exclude asset accounts for DEBTS_TIME', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'DEBTS_TIME' }); + for (const accountType of accountTypesByCatgeory.ASSET) { + expect(filter(accountType)).toBe(false); + } + for (const accountType of accountTypesByCatgeory.DEBT) { + expect(filter(accountType)).toBe(true); + } + }); + + it('should include all accounts for NET_WORTH', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'NET_WORTH' }); + for (const accountType of allAccountTypes) { + expect(filter(accountType)).toBe(true); + } + }); + + it('should exclude property and insurance accounts for NET_INCOME', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'NET_INCOME' }); + for (const accountType of allAccountTypes) { + expect(filter(accountType)).toBe( + !(accountType === 'InsuranceAccount' || PROPERTY_ACCOUNT_TYPES.includes(accountType)), + ); + } + }); + + it('should exclude cash accounts for NET_INCOME when any accounts are deselected', () => { + const filter = getAccountTypeFilterForTrend({ + ...baseTrend, + reportType: 'NET_INCOME', + deselectedAccountIds: ['43237333_1544498'], + }); + expect(filter('CashAccount')).toBe(false); + }); +}); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 3e78bba..ee9ded4 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -81,7 +81,8 @@ export type TrendBalanceHistoryProgressCallback = (progress: { completePercentage: number; }) => void | Promise; -export type TrendBalanceHistoryCallbackProgress = Parameters[0]; +export type TrendBalanceHistoryCallbackProgress = + Parameters[0]; type ProgressCallback = (progress: { complete: number; total: number }) => void | Promise; @@ -90,7 +91,7 @@ type AccountIdFilter = { accountId: string; }; -const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { +export const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { BankAccount: 'ASSET', CashAccount: 'ASSET', CreditAccount: 'DEBT', @@ -102,7 +103,13 @@ const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { OtherPropertyAccount: 'ASSET', } satisfies Record; -type AccountType = keyof typeof ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE; +export type AccountType = keyof typeof ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE; + +export const PROPERTY_ACCOUNT_TYPES: AccountType[] = [ + 'OtherPropertyAccount', + 'RealEstateAccount', + 'VehicleAccount', +]; type AccountTypeFilter = (accountType: AccountType) => boolean; @@ -134,10 +141,8 @@ export const getAccountTypeFilterForTrend = (trend: TrendState): AccountTypeFilt return (type) => ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE[type] === 'DEBT'; case 'NET_INCOME': return (type) => - type !== 'VehicleAccount' && - type !== 'RealEstateAccount' && - type !== 'OtherPropertyAccount' && type !== 'InsuranceAccount' && + !PROPERTY_ACCOUNT_TYPES.includes(type) && // Cash account does not appear in the dropdown; it is included only when All Accounts are // selected not when specific accounts are selected // TODO: verify this isn't just on idpaterson's account From 2c3488b25d7402d6a0195f9e14dece65564926a2 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Sat, 18 Nov 2023 10:01:42 -0500 Subject: [PATCH 14/30] No account filter if all accounts selected --- src/shared/lib/accounts.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index ee9ded4..a94c046 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -508,6 +508,10 @@ export const fetchTrendAccounts = async ({ }: { trend: TrendState; } & FetchAccountsOptions) => { + // Mint knows best which accounts are eligible for the trend if nothing is selected + if (!trend.deselectedAccountIds?.length) { + return []; + } const allAccounts = await fetchAccounts({ offset: 0, ...options }); const accountTypeFilter = getAccountTypeFilterForTrend(trend); From 97742cb7736037b4c93fb2ea83e232c914df4b3f Mon Sep 17 00:00:00 2001 From: idpaterson Date: Sat, 18 Nov 2023 10:52:09 -0500 Subject: [PATCH 15/30] Match account filtering to Mint logic --- src/shared/lib/__tests__/accounts.test.ts | 96 ++++++++++++----------- src/shared/lib/accounts.ts | 52 ++++++------ 2 files changed, 73 insertions(+), 75 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index 618ae9c..306be87 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -1,9 +1,6 @@ import { TEST_MINT_API_KEY } from '@root/src/shared/lib/constants'; import { - ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE, - AccountCategory, - AccountType, - PROPERTY_ACCOUNT_TYPES, + ReportType, TrendState, calculateIntervalForAccountHistory, fetchAccounts, @@ -351,16 +348,6 @@ describe('calculateIntervalForAccountHistory', () => { }); describe('getAccountTypeFilterForTrend', () => { - const allAccountTypes: AccountType[] = []; - const accountTypesByCatgeory = Object.entries(ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE).reduce( - (acc, [accountType, category]: [AccountType, AccountCategory]) => { - allAccountTypes.push(accountType); - acc[category] = acc[category] || []; - acc[category].push(accountType); - return acc; - }, - {} as Record, - ); const baseTrend: TrendState = { reportType: 'ASSETS_TIME', fixedFilter: 'CUSTOM', @@ -368,48 +355,67 @@ describe('getAccountTypeFilterForTrend', () => { toDate: '2020-01-01', }; - it('should exclude debt accounts for ASSETS_TIME', () => { + it('should exclude certain debt accounts for ASSETS_TIME', () => { const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'ASSETS_TIME' }); - for (const accountType of accountTypesByCatgeory.ASSET) { - expect(filter(accountType)).toBe(true); - } - for (const accountType of accountTypesByCatgeory.DEBT) { - expect(filter(accountType)).toBe(false); - } + expect(filter('BankAccount')).toBe(true); + expect(filter('VehicleAccount')).toBe(true); + expect(filter('CreditAccount')).toBe(false); + expect(filter('LoanAccount')).toBe(false); }); - it('should exclude asset accounts for DEBTS_TIME', () => { + it('should exclude certain asset accounts for DEBTS_TIME', () => { const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'DEBTS_TIME' }); - for (const accountType of accountTypesByCatgeory.ASSET) { - expect(filter(accountType)).toBe(false); - } - for (const accountType of accountTypesByCatgeory.DEBT) { - expect(filter(accountType)).toBe(true); - } + expect(filter('CreditAccount')).toBe(true); + expect(filter('LoanAccount')).toBe(true); + expect(filter('BankAccount')).toBe(false); + expect(filter('InvestmentAccount')).toBe(false); }); - it('should include all accounts for NET_WORTH', () => { + it('should exclude property accounts for SPENDING_TIME', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'SPENDING_TIME' }); + expect(filter('BankAccount')).toBe(true); + expect(filter('CreditAccount')).toBe(true); + expect(filter('RealEstateAccount')).toBe(false); + expect(filter('VehicleAccount')).toBe(false); + }); + + it('should exclude property accounts for INCOME_TIME', () => { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'INCOME_TIME' }); + expect(filter('BankAccount')).toBe(true); + expect(filter('CreditAccount')).toBe(true); + expect(filter('RealEstateAccount')).toBe(false); + expect(filter('VehicleAccount')).toBe(false); + }); + + it('should include all accounts except insurance and cash for NET_WORTH', () => { const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'NET_WORTH' }); - for (const accountType of allAccountTypes) { - expect(filter(accountType)).toBe(true); - } + expect(filter('BankAccount')).toBe(true); + expect(filter('CreditAccount')).toBe(true); + expect(filter('RealEstateAccount')).toBe(true); + expect(filter('VehicleAccount')).toBe(true); }); - it('should exclude property and insurance accounts for NET_INCOME', () => { + it('should exclude property accounts for NET_INCOME', () => { const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType: 'NET_INCOME' }); - for (const accountType of allAccountTypes) { - expect(filter(accountType)).toBe( - !(accountType === 'InsuranceAccount' || PROPERTY_ACCOUNT_TYPES.includes(accountType)), - ); - } + expect(filter('BankAccount')).toBe(true); + expect(filter('CreditAccount')).toBe(true); + expect(filter('RealEstateAccount')).toBe(false); + expect(filter('VehicleAccount')).toBe(false); }); - it('should exclude cash accounts for NET_INCOME when any accounts are deselected', () => { - const filter = getAccountTypeFilterForTrend({ - ...baseTrend, - reportType: 'NET_INCOME', - deselectedAccountIds: ['43237333_1544498'], - }); - expect(filter('CashAccount')).toBe(false); + it('should exclude cash and insurance accounts for all types', () => { + const reportTypes: ReportType[] = [ + 'ASSETS_TIME', + 'DEBTS_TIME', + 'SPENDING_TIME', + 'INCOME_TIME', + 'NET_WORTH', + 'NET_INCOME', + ]; + for (const reportType of reportTypes) { + const filter = getAccountTypeFilterForTrend({ ...baseTrend, reportType }); + expect(filter('CashAccount')).toBe(false); + expect(filter('InsuranceAccount')).toBe(false); + } }); }); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index a94c046..150ac5a 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -15,7 +15,7 @@ import { withRateLimit, } from '@root/src/shared/lib/promises'; -export type AccountCategory = 'DEBT' | 'ASSET'; +export type AccountCategory = 'DEBT' | 'ASSET' | 'SPECIAL'; export type TrendType = 'DEBT' | 'ASSET' | 'INCOME' | 'EXPENSE'; @@ -91,25 +91,16 @@ type AccountIdFilter = { accountId: string; }; -export const ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE = { - BankAccount: 'ASSET', - CashAccount: 'ASSET', - CreditAccount: 'DEBT', - InsuranceAccount: 'ASSET', - InvestmentAccount: 'ASSET', - LoanAccount: 'DEBT', - RealEstateAccount: 'ASSET', - VehicleAccount: 'ASSET', - OtherPropertyAccount: 'ASSET', -} satisfies Record; - -export type AccountType = keyof typeof ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE; - -export const PROPERTY_ACCOUNT_TYPES: AccountType[] = [ - 'OtherPropertyAccount', - 'RealEstateAccount', - 'VehicleAccount', -]; +export type AccountType = + | 'BankAccount' + | 'CashAccount' + | 'CreditAccount' + | 'InsuranceAccount' + | 'InvestmentAccount' + | 'LoanAccount' + | 'RealEstateAccount' + | 'VehicleAccount' + | 'OtherPropertyAccount'; type AccountTypeFilter = (accountType: AccountType) => boolean; @@ -130,25 +121,26 @@ type FetchAccountsOptions = { /** * Allows filtering accounts since the API does not seem to allow negated account ID queries, yet * also does not expose the selected accounts (see {@link deselectedAccountIds}). + * + * Logic from `accountsToFilter` data in Mint trends ui module. */ export const getAccountTypeFilterForTrend = (trend: TrendState): AccountTypeFilter => { + const defaultFilter = (type) => type !== 'CashAccount' && type !== 'InsuranceAccount'; switch (trend.reportType) { case 'INCOME_TIME': + case 'SPENDING_TIME': + return (type) => + type !== 'RealEstateAccount' && type !== 'VehicleAccount' && defaultFilter(type); case 'ASSETS_TIME': - return (type) => ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE[type] === 'ASSET'; + return (type) => type !== 'LoanAccount' && type !== 'CreditAccount' && defaultFilter(type); case 'DEBTS_TIME': - case 'SPENDING_TIME': - return (type) => ACCOUNT_CATEGORY_BY_ACCOUNT_TYPE[type] === 'DEBT'; + return (type) => + type !== 'BankAccount' && type !== 'InvestmentAccount' && defaultFilter(type); case 'NET_INCOME': return (type) => - type !== 'InsuranceAccount' && - !PROPERTY_ACCOUNT_TYPES.includes(type) && - // Cash account does not appear in the dropdown; it is included only when All Accounts are - // selected not when specific accounts are selected - // TODO: verify this isn't just on idpaterson's account - (type !== 'CashAccount' || !trend.deselectedAccountIds?.length); + type !== 'RealEstateAccount' && type !== 'VehicleAccount' && defaultFilter(type); case 'NET_WORTH': - return () => true; + return defaultFilter; default: throw new Error(`Unsupported report type: ${trend.reportType}`); } From c836d9c5697522fee1851f6cf9b5043b70759dcc Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 16:53:12 -0500 Subject: [PATCH 16/30] Add popup button to download current trend --- src/components/popup/DownloadTrend.tsx | 67 +++++++++++++++++++++++++ src/components/popup/PopupContainer.tsx | 30 ++++++++++- src/pages/popup/Popup.tsx | 4 ++ src/shared/storages/stateStorage.ts | 2 +- 4 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 src/components/popup/DownloadTrend.tsx diff --git a/src/components/popup/DownloadTrend.tsx b/src/components/popup/DownloadTrend.tsx new file mode 100644 index 0000000..25f4d48 --- /dev/null +++ b/src/components/popup/DownloadTrend.tsx @@ -0,0 +1,67 @@ +import ErrorBoundary from '@root/src/components/ErrorBoundary'; +import Progress from '@root/src/components/Progress'; +import SpinnerWithText from '@root/src/components/SpinnerWithText'; +import Text from '@root/src/components/Text'; +import DefaultButton from '@root/src/components/button/DefaultButton'; + +import useStorage from '@root/src/shared/hooks/useStorage'; +import { useMemo } from 'react'; +import trendStorage, { TrendDownloadStatus } from '../../shared/storages/trendStorage'; + +const DownloadTrend = () => { + const trendStateValue = useStorage(trendStorage); + const isSuccess = trendStateValue.status === TrendDownloadStatus.Success; + + const content = useMemo(() => { + const { progress } = trendStateValue ?? {}; + const { completePercentage = 0 } = progress ?? {}; + + if (isSuccess) { + return ( +
+ Download complete! + + Balance history for the trend downloaded to your computer. + + + Import into Monarch + +
+ ); + } else if (completePercentage > 0) { + return ( +
+ + Downloading balance history for the trend. This may take a minute. + + {completePercentage > 0 && ( +
+ +
+ )} +
+ ); + } else { + return Getting your balance information...; + } + }, [isSuccess, trendStateValue]); + + if (trendStateValue.status === TrendDownloadStatus.Error) { + return ( + + Sorry, there was an error downloading the trend. We've been notified and are working on + a fix. + + ); + } + + return ( +
+ +
{content}
+ +
+ ); +}; + +export default DownloadTrend; diff --git a/src/components/popup/PopupContainer.tsx b/src/components/popup/PopupContainer.tsx index e8be5ee..b81d90e 100644 --- a/src/components/popup/PopupContainer.tsx +++ b/src/components/popup/PopupContainer.tsx @@ -14,6 +14,9 @@ import OtherResources from '@root/src/components/popup/OtherResources'; import { fetchAccounts } from '@root/src/shared/lib/accounts'; import DownloadBalances from '@root/src/components/popup/DownloadBalances'; import accountStorage, { AccountsDownloadStatus } from '@root/src/shared/storages/accountStorage'; +import DownloadTrend from './DownloadTrend'; +import { isSupportedTrendReport } from '../../shared/lib/trends'; +import trendStorage from '../../shared/storages/trendStorage'; interface Page { title: string; @@ -29,10 +32,15 @@ const PAGE_TO_COMPONENT: Record = { title: 'Mint Account Balance History', component: DownloadBalances, }, + downloadTrend: { + title: 'Current Trend Balance History', + component: DownloadTrend, + }, }; const PopupContainer = ({ children }: React.PropsWithChildren) => { const { currentPage, downloadTransactionsStatus } = useStorage(stateStorage); + const { trend } = useStorage(trendStorage); const { status, userData } = usePopupContext(); const sendMessage = useMessageSender(); @@ -84,6 +92,15 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { await sendMessage({ action: Action.DownloadAllAccountBalances }); }, [sendMessage]); + const onDownloadTrend = useCallback(async () => { + await stateStorage.patch({ + currentPage: 'downloadTrend', + downloadTransactionsStatus: undefined, + totalTransactionsCount: undefined, + }); + await sendMessage({ action: Action.DownloadTrendBalances }); + }, [sendMessage]); + const content = useMemo(() => { switch (status) { case ResponseStatus.Loading: @@ -115,6 +132,11 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { Download Mint account balance history + + Download current trend daily balances + ); default: @@ -124,7 +146,13 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { ); } - }, [status, userData?.userName, onDownloadTransactions, onDownloadAccountBalanceHistory]); + }, [ + status, + userData?.userName, + trend?.reportType, + onDownloadTransactions, + onDownloadAccountBalanceHistory, + ]); const { component: PageComponent, title: pageTitle } = PAGE_TO_COMPONENT[currentPage] ?? {}; diff --git a/src/pages/popup/Popup.tsx b/src/pages/popup/Popup.tsx index 6761125..1a1fe8b 100644 --- a/src/pages/popup/Popup.tsx +++ b/src/pages/popup/Popup.tsx @@ -90,6 +90,10 @@ const Popup = () => { } else { await authenticateUser(apiKey); } + + await sendMessage({ + action: Action.GetTrendState, + }); }; const authenticateOnDashboard = async () => { diff --git a/src/shared/storages/stateStorage.ts b/src/shared/storages/stateStorage.ts index 41b2dba..f26b118 100644 --- a/src/shared/storages/stateStorage.ts +++ b/src/shared/storages/stateStorage.ts @@ -1,6 +1,6 @@ import { createStorage, StorageType } from '@src/shared/storages/base'; -export type PageKey = 'downloadTransactions' | 'downloadBalances'; +export type PageKey = 'downloadTransactions' | 'downloadBalances' | 'downloadTrend'; type State = { currentPage: PageKey | undefined; From e192a53bd7336fe5fb5c18cd9ab6d137498c100a Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 18:33:36 -0500 Subject: [PATCH 17/30] Count retried requests toward progress only once --- src/shared/lib/accounts.ts | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 150ac5a..add3ded 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -15,7 +15,7 @@ import { withRateLimit, } from '@root/src/shared/lib/promises'; -export type AccountCategory = 'DEBT' | 'ASSET' | 'SPECIAL'; +export type AccountCategory = 'DEBT' | 'ASSET'; export type TrendType = 'DEBT' | 'ASSET' | 'INCOME' | 'EXPENSE'; @@ -283,20 +283,22 @@ const fetchDailyBalances = async ({ }, overrideApiKey, }) - .then((response) => - response.json().then(({ Trend }) => - Trend.map(({ amount, type, ...rest }) => ({ - ...rest, - type, - amount: type === 'DEBT' ? -amount : amount, - })), - ), - ) - .finally(() => { - counter.count += 1; - onProgress?.({ complete: counter.count, total: periods.length }); + .then((response) => response.json()) + .then(({ Trend }) => { + if (!Trend) { + // Trend is omitted when request times out + throw new Error('Trend timeout'); + } + return Trend.map(({ amount, type, ...rest }) => ({ + ...rest, + type, + amount: type === 'DEBT' ? -amount : amount, + })); }), - ), + ).finally(() => { + counter.count += 1; + onProgress?.({ complete: counter.count, total: periods.length }); + }), ), ); From 297fd02e188135ecc1db46d84741e09104e8b99d Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 19:02:55 -0500 Subject: [PATCH 18/30] Use p-ratelimit to limit concurrent requests --- package.json | 1 + pnpm-lock.yaml | 10 +++++- src/shared/lib/__tests__/promises.test.ts | 2 +- src/shared/lib/accounts.ts | 3 +- src/shared/lib/constants.ts | 7 ++-- src/shared/lib/promises.ts | 44 +++++++++++++++-------- 6 files changed, 47 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index fcb5e2e..7800c7f 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "construct-style-sheets-polyfill": "^3.1.0", "jszip": "3.10.1", "luxon": "3.4.3", + "p-ratelimit": "^1.0.1", "pluralize": "^8.0.0", "postcss": "^8.4.31", "react": "18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f28de10..da14a73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -29,6 +29,9 @@ dependencies: luxon: specifier: 3.4.3 version: 3.4.3 + p-ratelimit: + specifier: ^1.0.1 + version: 1.0.1 pluralize: specifier: ^8.0.0 version: 8.0.0 @@ -6346,6 +6349,11 @@ packages: p-limit: 4.0.0 dev: true + /p-ratelimit@1.0.1: + resolution: {integrity: sha512-tKBGoow6aWRH68K2eQx+qc1gSegjd5VLirZYc1Yms9pPFsYQ9TFI6aMn0vJH2vmvzjNpjlWZOFft4aPUen2w0A==} + engines: {node: '>=10.23.0'} + dev: false + /p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} diff --git a/src/shared/lib/__tests__/promises.test.ts b/src/shared/lib/__tests__/promises.test.ts index 6877bd4..92e591d 100644 --- a/src/shared/lib/__tests__/promises.test.ts +++ b/src/shared/lib/__tests__/promises.test.ts @@ -9,7 +9,7 @@ describe('withRateLimit', () => { }; const startTime = new Date().getTime(); - await withRateLimit({ delayMs: 500 })([request, request, request]); + await withRateLimit({ rate: 2 })([request, request, request]); const endTime = new Date().getTime(); expect(endTime - startTime).toBeGreaterThan(1000); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index add3ded..37e1d9b 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -5,7 +5,6 @@ import { DATE_FILTER_ALL_TIME, MINT_DAILY_TRENDS_MAX_DAYS, MINT_HEADERS, - MINT_RATE_LIMIT_DELAY_MS, } from '@root/src/shared/lib/constants'; import { formatCSV } from '@root/src/shared/lib/csv'; import { withRetry } from '@root/src/shared/lib/retry'; @@ -268,7 +267,7 @@ const fetchDailyBalances = async ({ count: 0, }; - const dailyBalancesByPeriod = await withRateLimit({ delayMs: MINT_RATE_LIMIT_DELAY_MS })( + const dailyBalancesByPeriod = await withRateLimit()( periods.map( ({ start, end }) => () => diff --git a/src/shared/lib/constants.ts b/src/shared/lib/constants.ts index d4fa176..b58ac92 100644 --- a/src/shared/lib/constants.ts +++ b/src/shared/lib/constants.ts @@ -12,8 +12,11 @@ export const UTM_URL_PARAMETERS = { utm_source: 'mint_export_extension', }; -// we may need to increase this, need to test more -export const MINT_RATE_LIMIT_DELAY_MS = 50; +/** Default number of API requests that can run at the same time */ +export const MINT_RATE_LIMIT_REQUESTS_PER_SECOND = 20; + +/** Default number of API requests that can run at the same time */ +export const MINT_RATE_LIMIT_CONCURRENT_REQUESTS = 4; // The Mint API returns daily activity when the date range is 43 days or fewer. export const MINT_DAILY_TRENDS_MAX_DAYS = 43; diff --git a/src/shared/lib/promises.ts b/src/shared/lib/promises.ts index 782303e..9aae4da 100644 --- a/src/shared/lib/promises.ts +++ b/src/shared/lib/promises.ts @@ -1,30 +1,46 @@ -import delay from '@root/src/shared/lib/delay'; +import { pRateLimit, Quota } from 'p-ratelimit'; +import { + MINT_RATE_LIMIT_CONCURRENT_REQUESTS, + MINT_RATE_LIMIT_REQUESTS_PER_SECOND, +} from './constants'; -type RateLimitOptions = { - /** Delay between when each request is started. Does not wait for request to finish. */ - delayMs: number; -}; +type RateLimitOptions = Quota; /** * Like Promise.all, except with requests spaced out. * - * Usage: - * await withRateLimit({ delayMs: 50 })([ + * The default interval is 1 second, so {@link RateLimitOptions.rate} is in terms of requests per + * second. The default {@link RateLimitOptions.rate} is {@link MINT_RATE_LIMIT_REQUESTS_PER_SECOND} + * and the default {@link RateLimitOptions.concurrency} option is + * {@link MINT_RATE_LIMIT_CONCURRENT_REQUESTS}. + * + * The concurrency limit once reached ensures that the next request does not begin until a previous + * request has completed. + * + * Usage for 5 requests per second: + * await withRateLimit({ rate: 5 })([ * () => request(), * () => request(), * ]) */ export const withRateLimit = - (options: RateLimitOptions) => + (options?: RateLimitOptions) => async (requests: (() => Promise)[]): Promise => { - const { delayMs } = options; + const limit = pRateLimit({ + interval: 1000, + rate: MINT_RATE_LIMIT_REQUESTS_PER_SECOND, + concurrency: MINT_RATE_LIMIT_CONCURRENT_REQUESTS, + ...options, + }); + let cancelled = false; return Promise.all( - requests.map(async (request, i) => { - await delay(i * delayMs); - return request(); - }), - ); + // if Promise.all rejects, stop all remaining requests + requests.map((request) => limit(() => (cancelled ? Promise.reject() : request()))), + ).catch((e) => { + cancelled = true; + throw e; + }); }; /** From ccec2f4c3639331dd29c22749f7e94445c294c69 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 19:05:45 -0500 Subject: [PATCH 19/30] Fix empty trend CSV on error --- src/shared/lib/accounts.ts | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 37e1d9b..3e11fc5 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -392,20 +392,17 @@ export const fetchDailyBalancesForTrend = async ({ days: MINT_DAILY_TRENDS_MAX_DAYS, }) as Interval[]; - // fetch one account at a time so we don't hit the rate limit - const balances = await withDefaultOnError([])( - fetchDailyBalances({ - accountId: accounts.map(({ id }) => id), - periods, - reportType, - overrideApiKey, - onProgress: ({ complete }) => { - onProgress?.({ - completePercentage: complete / periods.length, - }); - }, - }), - ); + const balances = await fetchDailyBalances({ + accountId: accounts.map(({ id }) => id), + periods, + reportType, + overrideApiKey, + onProgress: ({ complete }) => { + onProgress?.({ + completePercentage: complete / periods.length, + }); + }, + }); return balances; }; From 778326e59d27f3afb2380aa04e6dd5e933ad2e09 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 19:12:07 -0500 Subject: [PATCH 20/30] Acknowledge causes of daily trend balance failure --- src/components/popup/DownloadTrend.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/popup/DownloadTrend.tsx b/src/components/popup/DownloadTrend.tsx index 25f4d48..8d9ddd3 100644 --- a/src/components/popup/DownloadTrend.tsx +++ b/src/components/popup/DownloadTrend.tsx @@ -49,8 +49,9 @@ const DownloadTrend = () => { if (trendStateValue.status === TrendDownloadStatus.Error) { return ( - Sorry, there was an error downloading the trend. We've been notified and are working on - a fix. + Sorry, there was an error downloading the trend balances. Note that daily trend data is less + likely to be available for trends that include very old transactions across a large number + of accounts. Please try again later or refine the trend. ); } From 4d5d46f84c5600d02e41b7574711c454e0bfeed0 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 20:04:59 -0500 Subject: [PATCH 21/30] Limit to one trend export at a time --- src/pages/background/index.ts | 56 ++++++++++++++++++++--------------- src/shared/lib/accounts.ts | 2 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index 0dcffdc..ecd305d 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -7,6 +7,7 @@ import { BalanceHistoryCallbackProgress, fetchDailyBalancesForTrend, TrendBalanceHistoryCallbackProgress, + TrendEntry, } from '@root/src/shared/lib/accounts'; import { throttle } from '@root/src/shared/lib/events'; import stateStorage from '@root/src/shared/storages/stateStorage'; @@ -232,33 +233,40 @@ const handleDownloadAllAccountBalances = async (sendResponse: () => void) => { } }; +let pendingTrendBalances: Promise; + /** Download daily balances for the specified trend. */ const handleDownloadTrendBalances = async (sendResponse: () => void) => { try { - const throttledSendDownloadTrendBalancesProgress = throttle( - sendDownloadTrendBalancesProgress, - THROTTLE_INTERVAL_MS, - ); - - const { trend } = await trendStorage.get(); - await trendStorage.set({ - trend, - status: TrendDownloadStatus.Loading, - progress: { completePercentage: 0 }, - }); - const balances = await fetchDailyBalancesForTrend({ - trend, - onProgress: throttledSendDownloadTrendBalancesProgress, - }); - const { reportType } = trend; - const csv = formatBalancesAsCSV({ balances, reportType }); - - chrome.downloads.download({ - url: `data:text/csv,${csv}`, - filename: 'mint-trend-daily-balances.csv', - }); - - await trendStorage.patch({ status: TrendDownloadStatus.Success }); + if (pendingTrendBalances) { + // already downloading + await pendingTrendBalances; + } else { + const throttledSendDownloadTrendBalancesProgress = throttle( + sendDownloadTrendBalancesProgress, + THROTTLE_INTERVAL_MS, + ); + const { trend } = await trendStorage.get(); + await trendStorage.set({ + trend, + status: TrendDownloadStatus.Loading, + progress: { completePercentage: 0 }, + }); + pendingTrendBalances = fetchDailyBalancesForTrend({ + trend, + onProgress: throttledSendDownloadTrendBalancesProgress, + }); + const balances = await pendingTrendBalances; + const { reportType } = trend; + const csv = formatBalancesAsCSV({ balances, reportType }); + + chrome.downloads.download({ + url: `data:text/csv,${csv}`, + filename: 'mint-trend-daily-balances.csv', + }); + + await trendStorage.patch({ status: TrendDownloadStatus.Success }); + } } catch (e) { await trendStorage.patch({ status: TrendDownloadStatus.Error }); } finally { diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 3e11fc5..d00ec11 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -39,7 +39,7 @@ export type FixedDateFilter = | 'ALL_TIME' | 'CUSTOM'; -type TrendEntry = { +export type TrendEntry = { amount: number; date: string; // this is determined by the type of report we fetch (DEBTS_TIME/ASSETS_TIME) From d8a3659cfc3cf56adf1d429cf5df4e31692c0b50 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 20:21:57 -0500 Subject: [PATCH 22/30] Clean up pending trend when finished --- src/pages/background/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pages/background/index.ts b/src/pages/background/index.ts index ecd305d..fc9fdd6 100644 --- a/src/pages/background/index.ts +++ b/src/pages/background/index.ts @@ -270,6 +270,7 @@ const handleDownloadTrendBalances = async (sendResponse: () => void) => { } catch (e) { await trendStorage.patch({ status: TrendDownloadStatus.Error }); } finally { + pendingTrendBalances = null; sendResponse(); } }; From c71f4f938f8a81242c448a64b9d2cc0b0acebba7 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 22 Nov 2023 20:36:05 -0500 Subject: [PATCH 23/30] Rate limit the account history interval lookup --- src/shared/lib/accounts.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index d00ec11..069f346 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -316,8 +316,8 @@ export const fetchDailyBalancesForAllAccounts = async ({ const accounts = await withRetry(() => fetchAccounts({ overrideApiKey })); // first, fetch the range of dates we need to fetch for each account - const accountsWithPeriodsToFetch = await Promise.all( - accounts.map(async ({ id: accountId, name: accountName }) => { + const accountsWithPeriodsToFetch = await withRateLimit()( + accounts.map(({ id: accountId, name: accountName }) => async () => { const { periods, reportType } = await withDefaultOnError({ periods: [], reportType: '' })( fetchIntervalsForAccountHistory({ accountId, From 759a09f7d6fdecfc3bfadadb11135e32be90a1e1 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Wed, 13 Dec 2023 19:52:29 -0500 Subject: [PATCH 24/30] Do not trust fromDate for ALL_TIME trends --- src/shared/lib/__tests__/accounts.test.ts | 8 ++--- src/shared/lib/accounts.ts | 37 +++++++++++++++-------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index 306be87..046edc8 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -5,7 +5,7 @@ import { calculateIntervalForAccountHistory, fetchAccounts, fetchDailyBalancesForAllAccounts, - fetchMonthlyBalancesForAccount, + fetchMonthlyBalances, fetchNetWorthBalances, fetchTrendAccounts, formatBalancesAsCSV, @@ -13,9 +13,9 @@ import { } from '../accounts'; import { DateTime } from 'luxon'; -describe('fetchMonthlyBalancesForAccount', () => { +describe('fetchMonthlyBalances', () => { it('fetches balances by date for asset account', async () => { - const { balancesByDate } = await fetchMonthlyBalancesForAccount({ + const { balancesByDate } = await fetchMonthlyBalances({ accountId: '43237333_1544498', overrideApiKey: TEST_MINT_API_KEY, }); @@ -23,7 +23,7 @@ describe('fetchMonthlyBalancesForAccount', () => { }); it('fetches balances for debt account', async () => { - const { balancesByDate } = await fetchMonthlyBalancesForAccount({ + const { balancesByDate } = await fetchMonthlyBalances({ accountId: '43237333_2630847', overrideApiKey: TEST_MINT_API_KEY, }); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 069f346..272076e 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -152,25 +152,28 @@ export const getAccountTypeFilterForTrend = (trend: TrendState): AccountTypeFilt * This is technically a paginated API, but since the limit is 1000 (> 83 years) * we probably don't need to worry about pagination. */ -export const fetchMonthlyBalancesForAccount = async ({ +export const fetchMonthlyBalances = async ({ accountId, + reportType, offset, limit, overrideApiKey, }: { - accountId: string; + accountId: string | string[]; + reportType?: ReportType; offset?: number; limit?: number; overrideApiKey?: string; -}): Promise<{ balancesByDate: TrendEntry[]; reportType: string } | undefined> => { +}): Promise<{ balancesByDate: TrendEntry[]; reportType: ReportType } | undefined> => { // we don't have a good way to know if an account is "asset" or "debt", so we just try both reports // the Mint API returns undefined if the report type doesn't match the account type - const tryReportTypes = ['ASSETS_TIME', 'DEBTS_TIME']; + const tryReportTypes: ReportType[] = reportType ? [reportType] : ['ASSETS_TIME', 'DEBTS_TIME']; + const accountIds = Array.isArray(accountId) ? accountId : [accountId]; for (const reportType of tryReportTypes) { const response = await fetchTrends({ reportType, - filters: [makeAccountIdFilter(accountId)], + filters: accountIds.map(makeAccountIdFilter), offset, limit, overrideApiKey, @@ -224,9 +227,7 @@ const fetchIntervalsForAccountHistory = async ({ overrideApiKey?: string; }) => { // fetch monthly balances so we can get start date - const balanceInfo = await withRetry(() => - fetchMonthlyBalancesForAccount({ accountId, overrideApiKey }), - ); + const balanceInfo = await withRetry(() => fetchMonthlyBalances({ accountId, overrideApiKey })); if (!balanceInfo) { throw new Error('Unable to fetch account history.'); @@ -254,7 +255,6 @@ const fetchDailyBalances = async ({ periods: Interval[]; accountId: string | string[]; reportType: string; - excludeSpecifiedAccounts?: boolean; overrideApiKey?: string; onProgress?: ProgressCallback; }) => { @@ -386,14 +386,27 @@ export const fetchDailyBalancesForTrend = async ({ overrideApiKey?: string; }) => { const accounts = await withRetry(() => fetchTrendAccounts({ trend, overrideApiKey })); - const { reportType, fromDate, toDate } = trend; - const interval = Interval.fromDateTimes(DateTime.fromISO(fromDate), DateTime.fromISO(toDate)); + const accountId = accounts.map(({ id }) => id); + const { reportType, fromDate, toDate, fixedFilter } = trend; + let interval: Interval; + // ALL_TIME may report a fromDate that is inaccurate by several years (e.g. 2007 when the trend + // data begins in 2015) so use the monthly trend to find an accurate start date + if (fixedFilter === 'ALL_TIME') { + const { balancesByDate } = await fetchMonthlyBalances({ + accountId, + reportType, + overrideApiKey, + }); + interval = calculateIntervalForAccountHistory(balancesByDate); + } else { + interval = Interval.fromDateTimes(DateTime.fromISO(fromDate), DateTime.fromISO(toDate)); + } const periods = interval.splitBy({ days: MINT_DAILY_TRENDS_MAX_DAYS, }) as Interval[]; const balances = await fetchDailyBalances({ - accountId: accounts.map(({ id }) => id), + accountId, periods, reportType, overrideApiKey, From 9045e5c2c50bbbc20e9daf2b6142e3957d5effd2 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Thu, 14 Dec 2023 07:58:46 -0500 Subject: [PATCH 25/30] Get trend state from React not localStorage --- src/shared/lib/__tests__/accounts.test.ts | 27 -------- src/shared/lib/accounts.ts | 28 +-------- src/shared/lib/trends.ts | 76 +++++++++++++++++++---- 3 files changed, 67 insertions(+), 64 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index 046edc8..c570fb8 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -7,7 +7,6 @@ import { fetchDailyBalancesForAllAccounts, fetchMonthlyBalances, fetchNetWorthBalances, - fetchTrendAccounts, formatBalancesAsCSV, getAccountTypeFilterForTrend, } from '../accounts'; @@ -47,32 +46,6 @@ describe('fetchAccounts', () => { }); }); -describe('fetchTrendAccounts', () => { - it('excludes deselected accounts', async () => { - const allDebtAccounts = await fetchTrendAccounts({ - trend: { - reportType: 'DEBTS_TIME', - deselectedAccountIds: [], - fixedFilter: 'CUSTOM', - fromDate: '2020-01-01', - toDate: '2020-01-01', - }, - overrideApiKey: TEST_MINT_API_KEY, - }); - const accounts = await fetchTrendAccounts({ - trend: { - reportType: 'DEBTS_TIME', - deselectedAccountIds: ['43237333_2630847'], // a debt account - fixedFilter: 'CUSTOM', - fromDate: '2020-01-01', - toDate: '2020-01-01', - }, - overrideApiKey: TEST_MINT_API_KEY, - }); - expect(accounts.length).toEqual(allDebtAccounts.length - 1); - }); -}); - describe('formatBalancesAsCSV', () => { it('includes account name if provied', () => { const result = formatBalancesAsCSV({ diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 272076e..7896167 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -56,10 +56,10 @@ type TrendsResponse = { /** State of user selections on the Mint Trends page */ export type TrendState = { + /** Selected accounts */ + accountIds?: string[]; /** Use with {@link deselectedAccountIds } to figure out which accounts to include in the trend */ reportType: ReportType; - /** All accounts eligible for the {@link reportType} that are NOT selected */ - deselectedAccountIds?: string[]; /** Semantic representation of the {@link fromDate} {@link toDate} range */ fixedFilter: FixedDateFilter; /** ISO start date */ @@ -385,8 +385,7 @@ export const fetchDailyBalancesForTrend = async ({ onProgress?: TrendBalanceHistoryProgressCallback; overrideApiKey?: string; }) => { - const accounts = await withRetry(() => fetchTrendAccounts({ trend, overrideApiKey })); - const accountId = accounts.map(({ id }) => id); + const accountId = trend.accountIds; const { reportType, fromDate, toDate, fixedFilter } = trend; let interval: Interval; // ALL_TIME may report a fromDate that is inaccurate by several years (e.g. 2007 when the trend @@ -502,27 +501,6 @@ export const fetchAccounts = async ({ return accounts; }; -/** - * Make sense of the {@link TrendState} by determining which accounts are selected. - */ -export const fetchTrendAccounts = async ({ - trend, - ...options -}: { - trend: TrendState; -} & FetchAccountsOptions) => { - // Mint knows best which accounts are eligible for the trend if nothing is selected - if (!trend.deselectedAccountIds?.length) { - return []; - } - const allAccounts = await fetchAccounts({ offset: 0, ...options }); - const accountTypeFilter = getAccountTypeFilterForTrend(trend); - - return allAccounts.filter( - ({ id, type }) => accountTypeFilter(type) && !trend.deselectedAccountIds?.includes(id), - ); -}; - /** * Merges paired API response into a single amount/inverseAmount entry. * diff --git a/src/shared/lib/trends.ts b/src/shared/lib/trends.ts index 4f6c1a4..1d98925 100644 --- a/src/shared/lib/trends.ts +++ b/src/shared/lib/trends.ts @@ -1,7 +1,35 @@ -import { TrendType, TrendState, ReportType } from '../../shared/lib/accounts'; +import { TrendState, ReportType, FixedDateFilter } from '../../shared/lib/accounts'; + +type AccountFilterReactProps = { + children: { + props: { + /** Selected account IDs and categories */ + value: string[]; + }; + }; +}; + +type DatePickerReactProps = { + props: { + children: [ + unknown, + { + props: { + /** The ISO date string */ + value: string; + }; + }, + unknown, + ]; + }; +}; + +type TimeFilterReactProps = { + children: [DatePickerReactProps, DatePickerReactProps]; +}; /** - * State for the most recent trend according to localStorage, does not need to be visible + * State for the current visible trend. * * This function must be executed in the context of the Mint tab and therefore must be * self-contained. @@ -10,19 +38,43 @@ export const getCurrentTrendState = () => { if ( // disable when not viewing the Trends page window.location.pathname.startsWith('/trends') && - // disable when filtered by category, tag, etc. because this filter is not in the trend state + // disable when filtered by category, tag, etc. because the extension does not support these !document.querySelector('[data-automation-id="filter-chip"]') ) { try { - const CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY = 'trends-state'; - const currentTrendType = localStorage.getItem( - CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY, - ) as TrendType; - const trendState = JSON.parse( - localStorage.getItem(`${CURRENT_TREND_TYPE_LOCAL_STORAGE_KEY}-${currentTrendType}`) || - 'null', - ) as TrendState; - + // Return React data backing HTML elements + const getReactProps = (selector: string) => { + const el = document.querySelector(selector); + return el?.[Object.keys(el).find((key) => key.startsWith('__reactProps'))] as Props; + }; + const accountState = getReactProps( + '[data-automation-id="filter-accounts"]', + ); + // For ALL_TIME charts this time range may be inaccurate (e.g. 2007 when the data only begins + // in 2021) but the extension is better equipped to choose the correct date with the API. + const timeFilterState = getReactProps( + '[data-automation-id="filter-time-custom"]', + ); + // ReportType can also be found in react props but it does not update reliably + const reportType = document.querySelector('.trends-sidebar-report-selected-list-item a') + ?.id as ReportType; + const fixedFilter = (document.getElementById('select-timeframe') as HTMLSelectElement) + .value as FixedDateFilter; + const accountIds = accountState.children.props.value.filter( + // Only numeric account IDs (ignore selected categories like AllAccounts and BankAccounts + // that will evaluate to NaN) + (id) => +id[0] === +id[0], + ) as string[]; + // This is a bit much, but can't seem to get the value reliably from child elements + const fromDate = timeFilterState.children[0].props.children[1].props.value; + const toDate = timeFilterState.children[1].props.children[1].props.value; + const trendState: TrendState = { + accountIds, + reportType, + fixedFilter, + fromDate, + toDate, + }; return trendState; } catch (e) { // ignore From 29c7e6368553a3741f8134cadcb99c5f43ec2e78 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Thu, 14 Dec 2023 08:44:18 -0500 Subject: [PATCH 26/30] Easier trend state retrieval from Redux store --- src/shared/lib/trends.ts | 76 +++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/src/shared/lib/trends.ts b/src/shared/lib/trends.ts index 1d98925..a3b01e8 100644 --- a/src/shared/lib/trends.ts +++ b/src/shared/lib/trends.ts @@ -1,33 +1,38 @@ -import { TrendState, ReportType, FixedDateFilter } from '../../shared/lib/accounts'; +import { TrendState, ReportType, FixedDateFilter, TrendType } from '../../shared/lib/accounts'; -type AccountFilterReactProps = { - children: { - props: { - /** Selected account IDs and categories */ - value: string[]; - }; - }; +type TrendsUiState = { + reportCategory: TrendType; + reportType: ReportType; + fromDate: string; + toDate: string; + fixedFilter: FixedDateFilter; + accounts: { + id: string; + isSelected: boolean; + }[]; }; -type DatePickerReactProps = { - props: { - children: [ - unknown, - { +type TrendsUiReactProps = { + children: { + props: { + children: { props: { - /** The ISO date string */ - value: string; + children: [ + unknown, + { + props: { + store: { + getState: () => { Trends: TrendsUiState }; + }; + }; + }, + ]; }; - }, - unknown, - ]; + }; + }; }; }; -type TimeFilterReactProps = { - children: [DatePickerReactProps, DatePickerReactProps]; -}; - /** * State for the current visible trend. * @@ -47,27 +52,12 @@ export const getCurrentTrendState = () => { const el = document.querySelector(selector); return el?.[Object.keys(el).find((key) => key.startsWith('__reactProps'))] as Props; }; - const accountState = getReactProps( - '[data-automation-id="filter-accounts"]', - ); - // For ALL_TIME charts this time range may be inaccurate (e.g. 2007 when the data only begins - // in 2021) but the extension is better equipped to choose the correct date with the API. - const timeFilterState = getReactProps( - '[data-automation-id="filter-time-custom"]', - ); - // ReportType can also be found in react props but it does not update reliably - const reportType = document.querySelector('.trends-sidebar-report-selected-list-item a') - ?.id as ReportType; - const fixedFilter = (document.getElementById('select-timeframe') as HTMLSelectElement) - .value as FixedDateFilter; - const accountIds = accountState.children.props.value.filter( - // Only numeric account IDs (ignore selected categories like AllAccounts and BankAccounts - // that will evaluate to NaN) - (id) => +id[0] === +id[0], - ) as string[]; - // This is a bit much, but can't seem to get the value reliably from child elements - const fromDate = timeFilterState.children[0].props.children[1].props.value; - const toDate = timeFilterState.children[1].props.children[1].props.value; + const trendsUiState = + getReactProps( + '.cg-pfm-trends-ui', + ).children.props.children.props.children[1].props.store.getState(); + const { reportType, fromDate, toDate, fixedFilter, accounts } = trendsUiState.Trends; + const accountIds = accounts.flatMap((a) => (a.isSelected ? a.id : [])); const trendState: TrendState = { accountIds, reportType, From db1b734dbaff5ef12e4efd07fbe48b6bc310c6e9 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Thu, 14 Dec 2023 11:22:17 -0500 Subject: [PATCH 27/30] Export trends filtered by tag, merchant, category --- src/shared/lib/__tests__/accounts.test.ts | 5 +- src/shared/lib/accounts.ts | 146 ++++++++++++++++++---- src/shared/lib/trends.ts | 20 ++- 3 files changed, 141 insertions(+), 30 deletions(-) diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts index c570fb8..4c44108 100644 --- a/src/shared/lib/__tests__/accounts.test.ts +++ b/src/shared/lib/__tests__/accounts.test.ts @@ -9,13 +9,14 @@ import { fetchNetWorthBalances, formatBalancesAsCSV, getAccountTypeFilterForTrend, + makeAccountIdFilter, } from '../accounts'; import { DateTime } from 'luxon'; describe('fetchMonthlyBalances', () => { it('fetches balances by date for asset account', async () => { const { balancesByDate } = await fetchMonthlyBalances({ - accountId: '43237333_1544498', + matchAllFilters: [makeAccountIdFilter('43237333_1544498')], overrideApiKey: TEST_MINT_API_KEY, }); expect(balancesByDate.length).toEqual(141); @@ -23,7 +24,7 @@ describe('fetchMonthlyBalances', () => { it('fetches balances for debt account', async () => { const { balancesByDate } = await fetchMonthlyBalances({ - accountId: '43237333_2630847', + matchAllFilters: [makeAccountIdFilter('43237333_2630847')], overrideApiKey: TEST_MINT_API_KEY, }); expect(balancesByDate.length).toEqual(141); diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index 7896167..d49d09a 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -54,17 +54,46 @@ type TrendsResponse = { // there's more here... }; +type CategoryFilterData = { + type: 'CATEGORY'; + includeChildCategories: boolean; + categoryId: string; + categoryName: string; +}; + +type DescriptionFilterData = { + type: 'DESCRIPTION'; + description: string; +}; + +type TagFilterData = { + type: 'TAG'; + tagId: string; + tagName: string; +}; + +export type FilterData = CategoryFilterData | DescriptionFilterData | TagFilterData; + +export type MatchType = 'all' | 'any'; + /** State of user selections on the Mint Trends page */ export type TrendState = { /** Selected accounts */ accountIds?: string[]; - /** Use with {@link deselectedAccountIds } to figure out which accounts to include in the trend */ reportType: ReportType; + /** Spending and income filters, transform with {@link apiFilterForFilterData} for the API */ + otherFilters?: FilterData[]; + /** + * Whether transactions must match any or all {@link otherFilters}. + * + * Account filters always match all. + */ + matchType?: MatchType; /** Semantic representation of the {@link fromDate} {@link toDate} range */ fixedFilter: FixedDateFilter; /** ISO start date */ fromDate: string; - /** ISO enddate */ + /** ISO end date */ toDate: string; }; @@ -90,6 +119,24 @@ type AccountIdFilter = { accountId: string; }; +type CategoryIdFilter = { + type: 'CategoryIdFilter'; + categoryId: string; + includeChildCategories: boolean; +}; + +type DescriptionNameFilter = { + type: 'DescriptionNameFilter'; + description: string; +}; + +type TagIdFilter = { + type: 'TagIdFilter'; + tagId: string; +}; + +type ApiFilter = AccountIdFilter | CategoryIdFilter | DescriptionNameFilter | TagIdFilter; + export type AccountType = | 'BankAccount' | 'CashAccount' @@ -153,13 +200,15 @@ export const getAccountTypeFilterForTrend = (trend: TrendState): AccountTypeFilt * we probably don't need to worry about pagination. */ export const fetchMonthlyBalances = async ({ - accountId, + matchAllFilters, + matchAnyFilters, reportType, offset, limit, overrideApiKey, }: { - accountId: string | string[]; + matchAllFilters?: ApiFilter[]; + matchAnyFilters?: ApiFilter[]; reportType?: ReportType; offset?: number; limit?: number; @@ -168,12 +217,12 @@ export const fetchMonthlyBalances = async ({ // we don't have a good way to know if an account is "asset" or "debt", so we just try both reports // the Mint API returns undefined if the report type doesn't match the account type const tryReportTypes: ReportType[] = reportType ? [reportType] : ['ASSETS_TIME', 'DEBTS_TIME']; - const accountIds = Array.isArray(accountId) ? accountId : [accountId]; for (const reportType of tryReportTypes) { const response = await fetchTrends({ reportType, - filters: accountIds.map(makeAccountIdFilter), + matchAllFilters, + matchAnyFilters, offset, limit, overrideApiKey, @@ -227,7 +276,12 @@ const fetchIntervalsForAccountHistory = async ({ overrideApiKey?: string; }) => { // fetch monthly balances so we can get start date - const balanceInfo = await withRetry(() => fetchMonthlyBalances({ accountId, overrideApiKey })); + const balanceInfo = await withRetry(() => + fetchMonthlyBalances({ + matchAllFilters: [makeAccountIdFilter(accountId)], + overrideApiKey, + }), + ); if (!balanceInfo) { throw new Error('Unable to fetch account history.'); @@ -247,22 +301,22 @@ const fetchIntervalsForAccountHistory = async ({ */ const fetchDailyBalances = async ({ periods, - accountId, reportType, + matchAllFilters, + matchAnyFilters, overrideApiKey, onProgress, }: { periods: Interval[]; - accountId: string | string[]; - reportType: string; + reportType: ReportType; + matchAllFilters?: ApiFilter[]; + matchAnyFilters?: ApiFilter[]; overrideApiKey?: string; onProgress?: ProgressCallback; }) => { if (!reportType) { throw new Error('Invalid report type.'); } - - const accountIds = Array.isArray(accountId) ? accountId : [accountId]; const counter = { count: 0, }; @@ -274,7 +328,8 @@ const fetchDailyBalances = async ({ withRetry(() => fetchTrends({ reportType, - filters: accountIds.map(makeAccountIdFilter), + matchAllFilters, + matchAnyFilters, dateFilter: { type: 'CUSTOM', startDate: start.toISODate(), @@ -318,7 +373,10 @@ export const fetchDailyBalancesForAllAccounts = async ({ // first, fetch the range of dates we need to fetch for each account const accountsWithPeriodsToFetch = await withRateLimit()( accounts.map(({ id: accountId, name: accountName }) => async () => { - const { periods, reportType } = await withDefaultOnError({ periods: [], reportType: '' })( + const { periods, reportType } = await withDefaultOnError({ + periods: [] as Interval[], + reportType: null as ReportType, + })( fetchIntervalsForAccountHistory({ accountId, overrideApiKey, @@ -341,7 +399,7 @@ export const fetchDailyBalancesForAllAccounts = async ({ async () => { const balances = await withDefaultOnError([])( fetchDailyBalances({ - accountId, + matchAllFilters: [makeAccountIdFilter(accountId)], periods, reportType, overrideApiKey, @@ -385,14 +443,25 @@ export const fetchDailyBalancesForTrend = async ({ onProgress?: TrendBalanceHistoryProgressCallback; overrideApiKey?: string; }) => { - const accountId = trend.accountIds; - const { reportType, fromDate, toDate, fixedFilter } = trend; + const { accountIds, matchType, otherFilters, reportType, fromDate, toDate, fixedFilter } = trend; + const matchAllFilters: ApiFilter[] = accountIds.map(makeAccountIdFilter); + const matchAnyFilters: ApiFilter[] = []; + + if (otherFilters?.length) { + if (matchType === 'any') { + matchAnyFilters.push(...otherFilters.map(apiFilterForFilterData)); + } else { + matchAllFilters.push(...otherFilters.map(apiFilterForFilterData)); + } + } + let interval: Interval; // ALL_TIME may report a fromDate that is inaccurate by several years (e.g. 2007 when the trend // data begins in 2015) so use the monthly trend to find an accurate start date if (fixedFilter === 'ALL_TIME') { const { balancesByDate } = await fetchMonthlyBalances({ - accountId, + matchAllFilters, + matchAnyFilters, reportType, overrideApiKey, }); @@ -405,7 +474,8 @@ export const fetchDailyBalancesForTrend = async ({ }) as Interval[]; const balances = await fetchDailyBalances({ - accountId, + matchAllFilters, + matchAnyFilters, periods, reportType, overrideApiKey, @@ -445,14 +515,16 @@ export const fetchNetWorthBalances = async ({ const fetchTrends = ({ reportType, - filters = [], + matchAllFilters = [], + matchAnyFilters = [], dateFilter = DATE_FILTER_ALL_TIME, offset = 0, limit = 1000, // mint default overrideApiKey, }: { reportType: string; - filters?: AccountIdFilter[]; + matchAllFilters?: (AccountIdFilter | ApiFilter)[]; + matchAnyFilters?: ApiFilter[]; dateFilter?: Record; offset?: number; limit?: number; @@ -471,7 +543,11 @@ const fetchTrends = ({ searchFilters: [ { matchAll: true, - filters, + filters: matchAllFilters, + }, + { + matchAll: false, + filters: matchAnyFilters, }, ], offset, @@ -577,7 +653,31 @@ export const formatBalancesAsCSV = ({ return formatCSV([header, ...rows]); }; -const makeAccountIdFilter = (accountId: string): AccountIdFilter => ({ +export const makeAccountIdFilter = (accountId: string): AccountIdFilter => ({ type: 'AccountIdFilter', accountId, }); + +/** Convert trend state filter data to API request filters */ +const apiFilterForFilterData = (data: FilterData): ApiFilter => { + switch (data.type) { + case 'CATEGORY': + return { + type: 'CategoryIdFilter', + categoryId: data.categoryId, + includeChildCategories: data.includeChildCategories, + }; + case 'DESCRIPTION': + return { + type: 'DescriptionNameFilter', + description: data.description, + }; + case 'TAG': + return { + type: 'TagIdFilter', + tagId: data.tagId, + }; + default: + throw new Error('Unsupported filter type'); + } +}; diff --git a/src/shared/lib/trends.ts b/src/shared/lib/trends.ts index a3b01e8..5bd69b0 100644 --- a/src/shared/lib/trends.ts +++ b/src/shared/lib/trends.ts @@ -1,4 +1,11 @@ -import { TrendState, ReportType, FixedDateFilter, TrendType } from '../../shared/lib/accounts'; +import { + TrendState, + ReportType, + FixedDateFilter, + TrendType, + FilterData, + MatchType, +} from '../../shared/lib/accounts'; type TrendsUiState = { reportCategory: TrendType; @@ -10,6 +17,8 @@ type TrendsUiState = { id: string; isSelected: boolean; }[]; + filterCategories: FilterData[]; + matchType: MatchType; }; type TrendsUiReactProps = { @@ -42,9 +51,7 @@ type TrendsUiReactProps = { export const getCurrentTrendState = () => { if ( // disable when not viewing the Trends page - window.location.pathname.startsWith('/trends') && - // disable when filtered by category, tag, etc. because the extension does not support these - !document.querySelector('[data-automation-id="filter-chip"]') + window.location.pathname.startsWith('/trends') ) { try { // Return React data backing HTML elements @@ -56,11 +63,14 @@ export const getCurrentTrendState = () => { getReactProps( '.cg-pfm-trends-ui', ).children.props.children.props.children[1].props.store.getState(); - const { reportType, fromDate, toDate, fixedFilter, accounts } = trendsUiState.Trends; + const { reportType, fromDate, toDate, fixedFilter, accounts, filterCategories, matchType } = + trendsUiState.Trends; const accountIds = accounts.flatMap((a) => (a.isSelected ? a.id : [])); const trendState: TrendState = { accountIds, reportType, + otherFilters: filterCategories, + matchType, fixedFilter, fromDate, toDate, From f544eccfb888ed017385eb48147fa144bcce6a4f Mon Sep 17 00:00:00 2001 From: idpaterson Date: Thu, 14 Dec 2023 13:09:05 -0500 Subject: [PATCH 28/30] Zero fill responses with no Trend data --- src/shared/lib/accounts.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts index d49d09a..111bf8e 100644 --- a/src/shared/lib/accounts.ts +++ b/src/shared/lib/accounts.ts @@ -51,7 +51,7 @@ export type TrendEntry = { type TrendsResponse = { Trend: TrendEntry[]; - // there's more here... + metaData: unknown; }; type CategoryFilterData = { @@ -338,10 +338,18 @@ const fetchDailyBalances = async ({ overrideApiKey, }) .then((response) => response.json()) - .then(({ Trend }) => { + .then(({ Trend, metaData }) => { + if (!Trend && !metaData) { + throw new Error('Unexpected response'); + } if (!Trend) { - // Trend is omitted when request times out - throw new Error('Trend timeout'); + // Trend is omitted when all balances are zero in this period, so build the rows + const dates = Interval.fromDateTimes(start, end).splitBy({ day: 1 }); + return dates.slice(1).map((date: Interval) => ({ + date: date.start.toISODate(), + amount: 0, + type: null, + })); } return Trend.map(({ amount, type, ...rest }) => ({ ...rest, From 93706a54215dae4af93614a96deb9c3d6017b7e7 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Thu, 14 Dec 2023 20:01:36 -0500 Subject: [PATCH 29/30] Open popup to main screen after trend download --- src/components/popup/PopupContainer.tsx | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/components/popup/PopupContainer.tsx b/src/components/popup/PopupContainer.tsx index b81d90e..01dbcd2 100644 --- a/src/components/popup/PopupContainer.tsx +++ b/src/components/popup/PopupContainer.tsx @@ -16,7 +16,7 @@ import DownloadBalances from '@root/src/components/popup/DownloadBalances'; import accountStorage, { AccountsDownloadStatus } from '@root/src/shared/storages/accountStorage'; import DownloadTrend from './DownloadTrend'; import { isSupportedTrendReport } from '../../shared/lib/trends'; -import trendStorage from '../../shared/storages/trendStorage'; +import trendStorage, { TrendDownloadStatus } from '../../shared/storages/trendStorage'; interface Page { title: string; @@ -40,9 +40,18 @@ const PAGE_TO_COMPONENT: Record = { const PopupContainer = ({ children }: React.PropsWithChildren) => { const { currentPage, downloadTransactionsStatus } = useStorage(stateStorage); - const { trend } = useStorage(trendStorage); + const { trend, status: trendStatus } = useStorage(trendStorage); const { status, userData } = usePopupContext(); const sendMessage = useMessageSender(); + let showPage = currentPage; + + // Trend download is likely to be completed multiple times in a row; do not require the user to + // click the back arrow after every download if the popup was closed. + if (currentPage === 'downloadTrend' && trendStatus === TrendDownloadStatus.Success) { + showPage = undefined; + stateStorage.patch({ currentPage: undefined }); + trendStorage.patch({ status: TrendDownloadStatus.Idle }); + } const onDownloadTransactions = useCallback(async () => { await stateStorage.patch({ @@ -154,15 +163,15 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { onDownloadAccountBalanceHistory, ]); - const { component: PageComponent, title: pageTitle } = PAGE_TO_COMPONENT[currentPage] ?? {}; + const { component: PageComponent, title: pageTitle } = PAGE_TO_COMPONENT[showPage] ?? {}; // 💀 const showBackArrow = - currentPage === 'downloadTransactions' + showPage === 'downloadTransactions' ? downloadTransactionsStatus !== ResponseStatus.Loading - : currentPage === 'downloadBalances' + : showPage === 'downloadBalances' ? downloadTransactionsStatus !== ResponseStatus.Loading - : !!currentPage; // there's a page that's not index (index is undefined) + : !!showPage; // there's a page that's not index (index is undefined) return (
From d41ca8b55986d3908cd90bfebe36deda6bc61546 Mon Sep 17 00:00:00 2001 From: idpaterson Date: Sat, 23 Dec 2023 07:45:53 -0500 Subject: [PATCH 30/30] Error out if trend export is not actually running --- src/components/popup/PopupContainer.tsx | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/components/popup/PopupContainer.tsx b/src/components/popup/PopupContainer.tsx index 9ad7a86..fd5f9be 100644 --- a/src/components/popup/PopupContainer.tsx +++ b/src/components/popup/PopupContainer.tsx @@ -195,8 +195,11 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { } = accountStorage.getSnapshot(); if (status === AccountsDownloadStatus.Loading) { setTimeout(async () => { - const { progress } = await accountStorage.get(); - if (completePercentage === progress.completePercentage) { + const { status, progress } = await accountStorage.get(); + if ( + status === AccountsDownloadStatus.Loading && + completePercentage === progress.completePercentage + ) { await accountStorage.patch({ status: AccountsDownloadStatus.Error, }); @@ -212,6 +215,24 @@ const PopupContainer = ({ children }: React.PropsWithChildren) => { }); } }, 30_000); + } else if (currentPage === 'downloadTrend') { + const { + status, + progress: { completePercentage }, + } = trendStorage.getSnapshot(); + if (status === TrendDownloadStatus.Loading) { + setTimeout(async () => { + const { status, progress } = await trendStorage.get(); + if ( + status === TrendDownloadStatus.Loading && + completePercentage === progress.completePercentage + ) { + await trendStorage.patch({ + status: TrendDownloadStatus.Error, + }); + } + }, 5_000); // 5 seconds is enough since there is no networking before progress begins + } } return (