Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Breaking: Add support for multiple YNAB budgets #647

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Breaking: Add support for multiple YNAB budgets #647

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
005bb0a
Skip unmapped YNAB accounts
whatuserever Feb 9, 2025
06658f1
Add budget id to ynab mapping table
whatuserever Feb 10, 2025
215583e
Update accountNumbersToYnabAccountIds type to contain budgetId
whatuserever Feb 10, 2025
674f25b
Filter ynab accounts by row budget
whatuserever Feb 10, 2025
9c942ce
Store ynab categories per budget
whatuserever Feb 10, 2025
caa4631
Add method for getting budget by account number
whatuserever Feb 10, 2025
5f24135
Use account-specific budgetId
whatuserever Feb 10, 2025
1a3dc7b
get category from relevant budget
whatuserever Feb 10, 2025
e85bc5e
get ynab transactions by budget
whatuserever Feb 10, 2025
04a8232
get ynab categories by budget
whatuserever Feb 10, 2025
c05bdb6
Merge remote-tracking branch 'origin/master' into ynab/multiple_budgets
baruchiro Feb 11, 2025
bb3a7e7
cleanly await each iteration of initCategories
whatuserever Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion packages/main/src/backend/commonTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ export interface GoogleSheetsConfig extends OutputVendorConfigBase {
export interface YnabConfig extends OutputVendorConfigBase {
options: {
accessToken: string;
accountNumbersToYnabAccountIds: Record<string, string>;
accountNumbersToYnabAccountIds: Record<string, { ynabBudgetId: string; ynabAccountId: string }>;
budgetId: string;
maxPayeeNameLength?: number;
};
Expand Down Expand Up @@ -100,6 +100,15 @@ export type ExportTransactionsFunction = (
eventPublisher: EventPublisher,
) => Promise<ExportTransactionsResult>;

export interface ExportTransactionsForAccountParams extends ExportTransactionsParams {
accountNumber: string;
}

export type ExportTransactionsForAccountFunction = (
exportTransactionsParams: ExportTransactionsForAccountParams,
eventPublisher: EventPublisher,
) => Promise<ExportTransactionsResult>;

export interface OutputVendor {
name: OutputVendorName;
init?: (outputVendorsConfig: Config['outputVendors']) => Promise<void>;
Expand Down
124 changes: 87 additions & 37 deletions packages/main/src/backend/export/outputVendors/ynab/ynab.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
OutputVendorName,
type Config,
type EnrichedTransaction,
type ExportTransactionsForAccountFunction,
type ExportTransactionsFunction,
type OutputVendor,
type YnabAccountDetails,
Expand All @@ -18,7 +19,9 @@ const NOW = moment();
const MIN_YNAB_ACCESS_TOKEN_LENGTH = 43;
const MAX_YNAB_IMPORT_ID_LENGTH = 36;

const categoriesMap = new Map<string, Pick<ynab.Category, 'id' | 'name' | 'category_group_id'>>();
type YnabCategory = Pick<ynab.Category, 'id' | 'name' | 'category_group_id'>;

const budgetCategoriesMap = new Map<string, Map<string, YnabCategory>>();
const transactionsFromYnab = new Map<Date, ynab.TransactionDetail[]>();

let ynabConfig: YnabConfig | undefined;
Expand All @@ -37,15 +40,54 @@ async function initFromToken(accessToken?: string) {
}
}

const createTransactions: ExportTransactionsFunction = async ({ transactionsToCreate, startDate }, eventPublisher) => {
const createTransactions: ExportTransactionsFunction = async (
{ transactionsToCreate, startDate, outputVendorsConfig },
eventPublisher,
) => {
if (!budgetCategoriesMap.size) {
await initCategories();
}
const accountNumbers = _.uniq(transactionsToCreate.map((t) => t.accountNumber));
const promises = accountNumbers.map((accountNumber) =>
createTransactionsForAccount(
{
accountNumber,
transactionsToCreate,
startDate,
outputVendorsConfig,
},
eventPublisher,
),
);
const results = await Promise.all(promises);
const exportedTransactionsNums = results.map((v) => v.exportedTransactionsNum);
return {
exportedTransactionsNum: _.sum(exportedTransactionsNums),
};
};

const createTransactionsForAccount: ExportTransactionsForAccountFunction = async (
{ accountNumber, transactionsToCreate: allTransactions, startDate },
eventPublisher,
) => {
if (!ynabConfig) {
throw new Error('Must call init before using ynab functions');
}
if (!categoriesMap.size) {
await initCategories();
let budgetId: string;
try {
getYnabAccountIdByAccountNumberFromTransaction(accountNumber);
budgetId = getYnabBudgetIdByAccountNumberFromTransaction(accountNumber);
} catch (e) {
await emitProgressEvent(eventPublisher, allTransactions, `Account ${accountNumber} is unmapped. Skipping.`);
return {
exportedTransactionsNum: 0,
};
}
const transactionsFromFinancialAccount = transactionsToCreate.map(convertTransactionToYnabFormat);
const transactionsFromFinancialAccount = allTransactions
.filter((v) => v.accountNumber === accountNumber)
.map(convertTransactionToYnabFormat);
let transactionsThatDontExistInYnab = await filterOnlyTransactionsThatDontExistInYnabAlready(
budgetId,
startDate,
transactionsFromFinancialAccount,
);
Expand All @@ -54,23 +96,19 @@ const createTransactions: ExportTransactionsFunction = async ({ transactionsToCr
moment(transaction.date, YNAB_DATE_FORMAT).isBefore(NOW),
);
if (!transactionsThatDontExistInYnab.length) {
await emitProgressEvent(
eventPublisher,
transactionsToCreate,
'All transactions already exist in ynab. Doing nothing.',
);
await emitProgressEvent(eventPublisher, allTransactions, 'All transactions already exist in ynab. Doing nothing.');
return {
exportedTransactionsNum: 0,
};
}

await emitProgressEvent(
eventPublisher,
transactionsToCreate,
allTransactions,
`Creating ${transactionsThatDontExistInYnab.length} transactions in ynab`,
);
try {
await ynabAPI!.transactions.createTransactions(ynabConfig.options.budgetId, {
await ynabAPI!.transactions.createTransactions(budgetId, {
transactions: transactionsThatDontExistInYnab,
});
return {
Expand All @@ -83,19 +121,16 @@ const createTransactions: ExportTransactionsFunction = async ({ transactionsToCr
message: (e as Error).message,
error: e as Error,
exporterName: ynabOutputVendor.name,
allTransactions: transactionsToCreate,
allTransactions: allTransactions,
}),
);
console.error('Failed to create transactions in ynab', e);
throw e;
}
};

function getTransactions(startDate: Date): Promise<ynab.TransactionsResponse> {
return ynabAPI!.transactions.getTransactions(
ynabConfig!.options.budgetId,
moment(startDate).format(YNAB_DATE_FORMAT),
);
function getTransactions(budgetId: string, startDate: Date): Promise<ynab.TransactionsResponse> {
return ynabAPI!.transactions.getTransactions(budgetId, moment(startDate).format(YNAB_DATE_FORMAT));
}

export function getPayeeName(transaction: EnrichedTransaction, payeeNameMaxLength = 50) {
Expand All @@ -110,13 +145,14 @@ export function getPayeeName(transaction: EnrichedTransaction, payeeNameMaxLengt
function convertTransactionToYnabFormat(originalTransaction: EnrichedTransaction): ynab.SaveTransaction {
const amount = Math.round(originalTransaction.chargedAmount * 1000);
const date = convertTimestampToYnabDateFormat(originalTransaction);
const budgetId = getYnabBudgetIdByAccountNumberFromTransaction(originalTransaction.accountNumber);
return {
account_id: getYnabAccountIdByAccountNumberFromTransaction(originalTransaction.accountNumber),
date, // "2019-01-17",
amount,
// "payee_id": "string",
payee_name: getPayeeName(originalTransaction, ynabConfig!.options.maxPayeeNameLength),
category_id: getYnabCategoryIdFromCategoryName(originalTransaction.category),
category_id: getYnabCategoryIdFromCategoryName(budgetId, originalTransaction.category),
memo: originalTransaction.memo,
cleared: ynab.SaveTransaction.ClearedEnum.Cleared,
import_id: buildImportId(originalTransaction), // [date][amount][description]
Expand All @@ -138,48 +174,62 @@ function getYnabAccountIdByAccountNumberFromTransaction(transactionAccountNumber
if (!ynabAccountId) {
throw new Error(`Unhandled account number ${transactionAccountNumber}`);
}
return ynabAccountId;
return ynabAccountId.ynabAccountId;
}

function getYnabBudgetIdByAccountNumberFromTransaction(transactionAccountNumber: string): string {
const ynabAccount = ynabConfig!.options.accountNumbersToYnabAccountIds[transactionAccountNumber];
if (!ynabAccount) {
throw new Error(`Unhandled account number ${transactionAccountNumber}`);
}
return ynabAccount.ynabBudgetId;
}

function convertTimestampToYnabDateFormat(originalTransaction: EnrichedTransaction): string {
return moment(originalTransaction.date).format(YNAB_DATE_FORMAT); // 2018-12-29T22:00:00.000Z -> 2018-12-29
}

function getYnabCategoryIdFromCategoryName(categoryName?: string) {
function getYnabCategoryIdFromCategoryName(budgetId: string, categoryName?: string) {
if (!categoryName) {
return null;
}
const categoryToReturn = categoriesMap.get(categoryName);
const categoryToReturn = budgetCategoriesMap.get(budgetId)?.get(categoryName);
if (!categoryToReturn) {
return null;
}
return categoryToReturn?.id;
}

export async function initCategories() {
const categories = await ynabAPI!.categories.getCategories(ynabConfig!.options.budgetId);
categories.data.category_groups.forEach((categoryGroup) => {
categoryGroup.categories
.map((category) => ({
id: category.id,
name: category.name,
category_group_id: category.category_group_id,
}))
.forEach((category) => {
categoriesMap.set(category.name, category);
});
const budgetIds = Object.values(ynabConfig!.options.accountNumbersToYnabAccountIds).map((v) => v.ynabBudgetId);
budgetIds.forEach(async (budgetId) => {
const categories = await ynabAPI!.categories.getCategories(budgetId);
const categoriesMap = new Map<string, YnabCategory>();
categories.data.category_groups.forEach((categoryGroup) => {
categoryGroup.categories
.map((category) => ({
id: category.id,
name: category.name,
category_group_id: category.category_group_id,
}))
.forEach((category) => {
categoriesMap.set(category.name, category);
});
});
budgetCategoriesMap.set(budgetId, categoriesMap);
});
}

async function filterOnlyTransactionsThatDontExistInYnabAlready(
budgetId: string,
startDate: Date,
transactionsFromFinancialAccounts: ynab.SaveTransaction[],
) {
let transactionsInYnabBeforeCreatingTheseTransactions: ynab.TransactionDetail[];
if (transactionsFromYnab.has(startDate)) {
transactionsInYnabBeforeCreatingTheseTransactions = transactionsFromYnab.get(startDate)!;
} else {
const transactionsFromYnabResponse = await getTransactions(startDate);
const transactionsFromYnabResponse = await getTransactions(budgetId, startDate);
transactionsInYnabBeforeCreatingTheseTransactions = transactionsFromYnabResponse.data.transactions;
transactionsFromYnab.set(startDate, transactionsInYnabBeforeCreatingTheseTransactions);
}
Expand Down Expand Up @@ -237,7 +287,7 @@ export async function getYnabAccountDetails(
let categories: YnabAccountDetails['categories'];
if (doesBudgetIdExistInYnab(budgetIdToCheck)) {
console.log('Getting ynab categories');
categories = await getYnabCategories();
categories = await getYnabCategories(budgetIdToCheck);
} else {
// eslint-disable-next-line
console.warn(`Budget id ${budgetIdToCheck} doesn't exist in ynab`);
Expand Down Expand Up @@ -304,8 +354,8 @@ async function getBudgetsAndAccountsData() {
};
}

async function getYnabCategories() {
const categoriesResponse = await ynabAPI!.categories.getCategories(ynabConfig!.options.budgetId);
async function getYnabCategories(budgetId: string) {
const categoriesResponse = await ynabAPI!.categories.getCategories(budgetId);
const categories = _.flatMap(categoriesResponse.data.category_groups, (categoryGroup) => categoryGroup.categories);
const categoryNames = categories.map((category) => category.name);
return categoryNames;
Expand Down
2 changes: 1 addition & 1 deletion packages/preload/src/commonTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export interface GoogleSheetsConfig extends OutputVendorConfigBase {
export interface YnabConfig extends OutputVendorConfigBase {
options: {
accessToken: string;
accountNumbersToYnabAccountIds: Record<string, string>;
accountNumbersToYnabAccountIds: Record<string, { ynabBudgetId: string; ynabAccountId: string }>;
budgetId: string;
maxPayeeNameLength?: number;
};
Expand Down
34 changes: 28 additions & 6 deletions packages/renderer/src/components/exporters/YnabAccountMappingTable.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ interface YnabAccountMappingTableProps {
onUpdate: (accountNumberToYnabIdMapping: AccountNumberToYnabAccountIdMappingObject) => void;
}

type AccountMappingArray = {
interface AccountMapping {
accountNumber: string;
ynabBudgetId: string;
ynabAccountId: string;
index?: number;
}[];
}

type AccountMappingArray = AccountMapping[];

const YnabAccountMappingTable = ({
accountNumberToYnabIdMapping,
Expand All @@ -44,12 +47,29 @@ const YnabAccountMappingTable = ({
type: Type.TEXT,
},
},
{
dataField: 'ynabBudgetId',
text: 'Ynab budget id',
editor: {
type: Type.SELECT,
getOptions: () => {
return ynabAccountData?.ynabAccountData?.budgets.map((ynabBudget) => {
return {
label: ynabBudget.name,
value: ynabBudget.id,
};
});
},
},
},
{
dataField: 'ynabAccountId',
text: 'Ynab account id',
editor: {
type: Type.SELECT,
getOptions: () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
getOptions: (_: any, { row }: { row: AccountMapping }) => {
const budgetId = (row as AccountMapping).ynabBudgetId;
Comment on lines +70 to +72
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no idea how to properly type this.
getOptions isn't covered by DefinitelyTyped, the documentation doesn't indicate the type, and the source code is untyped JavaScript. The only hint was the example here.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I passed the same way you went through (if I was reading your comment completely before trying to solve this... 😂), and then I found out we don't need to research this.

You can read about this here:
https://stackoverflow.com/a/63254304/839513

We use any to ignore type checking, which may hide errors, and we try to avoid it.

But if you use unknown, it will prevent you from accessing and manipulating the object, which is exactly the situation here, we don't know the type of _ and we don't want to use it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, I'll change this tomorrow

return ynabAccountData?.ynabAccountData?.accounts
.filter((ynabAccount) => ynabAccount.active && ynabAccount.budgetId === budgetId)
.map((ynabAccount) => {
Expand Down Expand Up @@ -79,6 +99,7 @@ const YnabAccountMappingTable = ({
...accountMappingArray,
{
accountNumber: '12345678',
ynabBudgetId: budgetId,
ynabAccountId: '##########',
index: accountMappingArray.length,
},
Expand All @@ -102,15 +123,16 @@ function accountMappingObjectToArray(
return {
index,
accountNumber,
ynabAccountId: accountNumbersToYnabAccountIds[accountNumber],
ynabBudgetId: accountNumbersToYnabAccountIds[accountNumber].ynabBudgetId,
ynabAccountId: accountNumbersToYnabAccountIds[accountNumber].ynabAccountId,
};
});
}

function accountMappingArrayToObject(accountMappingArray: AccountMappingArray) {
const mappingObject: AccountNumberToYnabAccountIdMappingObject = {};
accountMappingArray.forEach(({ accountNumber, ynabAccountId }) => {
mappingObject[accountNumber] = ynabAccountId;
accountMappingArray.forEach(({ accountNumber, ynabBudgetId, ynabAccountId }) => {
mappingObject[accountNumber] = { ynabBudgetId, ynabAccountId };
});
return mappingObject;
}
Expand Down
8 changes: 4 additions & 4 deletions packages/renderer/src/store/Store.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,10 @@ export const dummyConfig: Config = {
accessToken: '################-######-####-###',
budgetId: 'advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf',
accountNumbersToYnabAccountIds: {
5555: 'asdvasdvsdvs',
1111: 'xcvxcvxcvsd',
3333: 'xvxcdv',
555555555: 'vsdvsdvserverv',
5555: { ynabBudgetId: 'advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf', ynabAccountId: 'asdvasdvsdvs' },
1111: { ynabBudgetId: 'advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf', ynabAccountId: 'xcvxcvxcvsd' },
3333: { ynabBudgetId: 'advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf', ynabAccountId: 'xvxcdv' },
555555555: { ynabBudgetId: 'advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf', ynabAccountId: 'vsdvsdvserverv' },
},
},
},
Expand Down
20 changes: 16 additions & 4 deletions packages/renderer/src/store/__snapshots__/Store.test.tsx.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,22 @@ exports[`Store > Properties and getters > exporters 1`] = `
"options": {
"accessToken": "################-######-####-###",
"accountNumbersToYnabAccountIds": {
"1111": "xcvxcvxcvsd",
"3333": "xvxcdv",
"5555": "asdvasdvsdvs",
"555555555": "vsdvsdvserverv",
"1111": {
"ynabAccountId": "xcvxcvxcvsd",
"ynabBudgetId": "advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf",
},
"3333": {
"ynabAccountId": "xvxcdv",
"ynabBudgetId": "advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf",
},
"5555": {
"ynabAccountId": "asdvasdvsdvs",
"ynabBudgetId": "advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf",
},
"555555555": {
"ynabAccountId": "vsdvsdvserverv",
"ynabBudgetId": "advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf",
},
},
"budgetId": "advasdvasd-asdvasdva-sdvasdvasdv-asdvasdvf",
},
Expand Down
Loading
Loading