From 9ac77af077fa3fed8b7e1a097dda955745f474f4 Mon Sep 17 00:00:00 2001
From: Stefan Wilkes <Horizon0156@users.noreply.github.com>
Date: Wed, 14 Aug 2024 23:16:33 +0200
Subject: [PATCH] Added configuration to CSV importer that allows to skip lines
(#3234)
* Added configuration to CSV importer that allows to skip lines
* Fixed several type / parser check when import is executed with a different CSV structure on accounts with transactions
* Fixed linter warning
* Reverted changes on sync.ts as initial error is not occuring anymore.
This will also fix the tests again
---------
Co-authored-by: youngcw <calebyoung94@gmail.com>
---
.../components/modals/ImportTransactions.jsx | 54 +++++++++++++++----
.../src/server/accounts/parse-file.ts | 8 ++-
packages/loot-core/src/types/prefs.d.ts | 1 +
upcoming-release-notes/3234.md | 6 +++
4 files changed, 58 insertions(+), 11 deletions(-)
create mode 100644 upcoming-release-notes/3234.md
diff --git a/packages/desktop-client/src/components/modals/ImportTransactions.jsx b/packages/desktop-client/src/components/modals/ImportTransactions.jsx
index 430779264..88ffb4788 100644
--- a/packages/desktop-client/src/components/modals/ImportTransactions.jsx
+++ b/packages/desktop-client/src/components/modals/ImportTransactions.jsx
@@ -186,20 +186,14 @@ function getInitialMappings(transactions) {
return entry ? entry[0] : null;
}
- function isString(value) {
- return typeof value === 'string' || value instanceof String;
- }
-
const dateField = key(
fields.find(([name]) => name.toLowerCase().includes('date')) ||
- fields.find(
- ([, value]) => isString(value) && value.match(/^\d+[-/]\d+[-/]\d+$/),
- ),
+ fields.find(([, value]) => String(value)?.match(/^\d+[-/]\d+[-/]\d+$/)),
);
const amountField = key(
fields.find(([name]) => name.toLowerCase().includes('amount')) ||
- fields.find(([, value]) => isString(value) && value.match(/^-?[.,\d]+$/)),
+ fields.find(([, value]) => String(value)?.match(/^-?[.,\d]+$/)),
);
const categoryField = key(
@@ -880,6 +874,9 @@ export function ImportTransactions({ options }) {
prefs[`csv-delimiter-${accountId}`] ||
(filename.endsWith('.tsv') ? '\t' : ','),
);
+ const [skipLines, setSkipLines] = useState(
+ prefs[`csv-skip-lines-${accountId}`] ?? 0,
+ );
const [hasHeaderRow, setHasHeaderRow] = useState(
prefs[`csv-has-header-${accountId}`] ?? true,
);
@@ -988,6 +985,7 @@ export function ImportTransactions({ options }) {
const parseOptions = getParseOptions(fileType, {
delimiter,
hasHeaderRow,
+ skipLines,
fallbackMissingPayeeToMemo,
});
@@ -1040,6 +1038,7 @@ export function ImportTransactions({ options }) {
const parseOptions = getParseOptions(fileType, {
delimiter,
hasHeaderRow,
+ skipLines,
fallbackMissingPayeeToMemo,
});
@@ -1196,6 +1195,8 @@ export function ImportTransactions({ options }) {
[`csv-mappings-${accountId}`]: JSON.stringify(fieldMappings),
});
savePrefs({ [`csv-delimiter-${accountId}`]: delimiter });
+ savePrefs({ [`csv-has-header-${accountId}`]: hasHeaderRow });
+ savePrefs({ [`csv-skip-lines-${accountId}`]: skipLines });
}
if (filetype === 'csv' || filetype === 'qif') {
@@ -1282,6 +1283,10 @@ export function ImportTransactions({ options }) {
);
break;
}
+ if (trans.payee == null || !(trans.payee instanceof String)) {
+ console.log(`Unable·to·parse·payee·${trans.payee || '(empty)'}`);
+ break;
+ }
const { amount } = parseAmountFields(
trans,
@@ -1575,6 +1580,34 @@ export function ImportTransactions({ options }) {
getParseOptions('csv', {
delimiter: value,
hasHeaderRow,
+ skipLines,
+ }),
+ );
+ }}
+ style={{ width: 50 }}
+ />
+ </label>
+ <label
+ style={{
+ display: 'flex',
+ flexDirection: 'row',
+ gap: 5,
+ alignItems: 'baseline',
+ }}
+ >
+ Skip lines:
+ <Input
+ type="number"
+ value={skipLines}
+ min="0"
+ onChangeValue={value => {
+ setSkipLines(+value);
+ parse(
+ filename,
+ getParseOptions('csv', {
+ delimiter,
+ hasHeaderRow,
+ skipLines: +value,
}),
);
}}
@@ -1591,6 +1624,7 @@ export function ImportTransactions({ options }) {
getParseOptions('csv', {
delimiter,
hasHeaderRow: !hasHeaderRow,
+ skipLines,
}),
);
}}
@@ -1711,8 +1745,8 @@ export function ImportTransactions({ options }) {
function getParseOptions(fileType, options = {}) {
if (fileType === 'csv') {
- const { delimiter, hasHeaderRow } = options;
- return { delimiter, hasHeaderRow };
+ const { delimiter, hasHeaderRow, skipLines } = options;
+ return { delimiter, hasHeaderRow, skipLines };
} else if (isOfxFile(fileType)) {
const { fallbackMissingPayeeToMemo } = options;
return { fallbackMissingPayeeToMemo };
diff --git a/packages/loot-core/src/server/accounts/parse-file.ts b/packages/loot-core/src/server/accounts/parse-file.ts
index 0dd086086..fd6178411 100644
--- a/packages/loot-core/src/server/accounts/parse-file.ts
+++ b/packages/loot-core/src/server/accounts/parse-file.ts
@@ -18,6 +18,7 @@ type ParseFileOptions = {
hasHeaderRow?: boolean;
delimiter?: string;
fallbackMissingPayeeToMemo?: boolean;
+ skipLines?: number;
};
export async function parseFile(
@@ -57,7 +58,12 @@ async function parseCSV(
options: ParseFileOptions,
): Promise<ParseFileResult> {
const errors = Array<ParseError>();
- const contents = await fs.readFile(filepath);
+ let contents = await fs.readFile(filepath);
+
+ if (options.skipLines > 0) {
+ const lines = contents.split(/\r?\n/);
+ contents = lines.slice(options.skipLines).join('\r\n');
+ }
let data;
try {
diff --git a/packages/loot-core/src/types/prefs.d.ts b/packages/loot-core/src/types/prefs.d.ts
index 3d3664a87..a348eca04 100644
--- a/packages/loot-core/src/types/prefs.d.ts
+++ b/packages/loot-core/src/types/prefs.d.ts
@@ -30,6 +30,7 @@ export type SyncedPrefs = Partial<
[key: `parse-date-${string}-${'csv' | 'qif'}`]: string;
[key: `csv-mappings-${string}`]: string;
[key: `csv-delimiter-${string}`]: ',' | ';' | '\t';
+ [key: `csv-skip-lines-${string}`]: number;
[key: `csv-has-header-${string}`]: boolean;
[key: `ofx-fallback-missing-payee-${string}`]: boolean;
[key: `flip-amount-${string}-${'csv' | 'qif'}`]: boolean;
diff --git a/upcoming-release-notes/3234.md b/upcoming-release-notes/3234.md
new file mode 100644
index 000000000..12dafb981
--- /dev/null
+++ b/upcoming-release-notes/3234.md
@@ -0,0 +1,6 @@
+---
+category: Features
+authors: [Horizon0156]
+---
+
+Added an optional configuration value to skip one or more heading lines (added by some banks, like ING) during the CSV transactions import.
\ No newline at end of file
--
GitLab