From 835c1a54f70a8015a87e44403e289d0575a4d9cf Mon Sep 17 00:00:00 2001
From: Jakub Kuczys <me@jacken.men>
Date: Fri, 7 Apr 2023 22:33:19 +0200
Subject: [PATCH] Use Unicode-aware implementations of LOWER() and UPPER() in
SQL queries (#865)
Fixes #840 by creating application-defined SQL functions
(https://www.sqlite.org/appfunc.html) for Unicode-aware implementations
of `LOWER()` and `UPPER()`. This uses
`String.prototype.toLower/UpperCase()` JS method.
I initially wanted to just redefine `LOWER()` and `UPPER()` but due to
[sql.js not supporting the definition of deterministic
functions](https://github.com/sql-js/sql.js/issues/551), I had to just
define them as separate functions and use that in the appropriate
places. It's probably better like that anyway...
---
.../platform/server/sqlite/index.electron.ts | 11 +++-
.../src/platform/server/sqlite/index.web.ts | 60 +++++++++++--------
.../loot-core/src/server/accounts/payees.js | 2 +-
packages/loot-core/src/server/aql/compiler.js | 2 +-
packages/loot-core/src/server/db/index.js | 7 ++-
upcoming-release-notes/865.md | 6 ++
6 files changed, 58 insertions(+), 30 deletions(-)
create mode 100644 upcoming-release-notes/865.md
diff --git a/packages/loot-core/src/platform/server/sqlite/index.electron.ts b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
index 67bcacaa5..ac4224640 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.electron.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.electron.ts
@@ -84,7 +84,16 @@ export async function asyncTransaction(db, fn) {
}
export function openDatabase(pathOrBuffer) {
- return new Database(pathOrBuffer);
+ let db = new Database(pathOrBuffer);
+ // Define Unicode-aware LOWER and UPPER implementation.
+ // This is necessary because better-sqlite3 uses SQLite build without ICU support.
+ db.function('UNICODE_LOWER', { deterministic: true }, arg =>
+ arg?.toLowerCase(),
+ );
+ db.function('UNICODE_UPPER', { deterministic: true }, arg =>
+ arg?.toUpperCase(),
+ );
+ return db;
}
export function closeDatabase(db) {
diff --git a/packages/loot-core/src/platform/server/sqlite/index.web.ts b/packages/loot-core/src/platform/server/sqlite/index.web.ts
index 0ad16c16d..8a2b3c4a7 100644
--- a/packages/loot-core/src/platform/server/sqlite/index.web.ts
+++ b/packages/loot-core/src/platform/server/sqlite/index.web.ts
@@ -151,36 +151,48 @@ export async function asyncTransaction(db, fn) {
}
export async function openDatabase(pathOrBuffer?: string | Buffer) {
+ let db = null;
if (pathOrBuffer) {
if (typeof pathOrBuffer !== 'string') {
- return new SQL.Database(pathOrBuffer);
- }
-
- let path = pathOrBuffer;
- if (path !== ':memory:') {
- if (typeof SharedArrayBuffer === 'undefined') {
- // @ts-expect-error FS missing in sql.js types
- let stream = SQL.FS.open(SQL.FS.readlink(path), 'a+');
- await stream.node.contents.readIfFallback();
- // @ts-expect-error FS missing in sql.js types
- SQL.FS.close(stream);
+ db = new SQL.Database(pathOrBuffer);
+ } else {
+ let path = pathOrBuffer;
+ if (path !== ':memory:') {
+ if (typeof SharedArrayBuffer === 'undefined') {
+ // @ts-expect-error FS missing in sql.js types
+ let stream = SQL.FS.open(SQL.FS.readlink(path), 'a+');
+ await stream.node.contents.readIfFallback();
+ // @ts-expect-error FS missing in sql.js types
+ SQL.FS.close(stream);
+ }
+
+ db = new SQL.Database(
+ // @ts-expect-error FS missing in sql.js types
+ path.includes('/blocked') ? path : SQL.FS.readlink(path),
+ // @ts-expect-error 2nd argument missed in sql.js types
+ { filename: true },
+ );
+ db.exec(`
+ PRAGMA journal_mode=MEMORY;
+ PRAGMA cache_size=-10000;
+ `);
}
-
- let db = new SQL.Database(
- // @ts-expect-error FS missing in sql.js types
- path.includes('/blocked') ? path : SQL.FS.readlink(path),
- // @ts-expect-error 2nd argument missed in sql.js types
- { filename: true },
- );
- db.exec(`
- PRAGMA journal_mode=MEMORY;
- PRAGMA cache_size=-10000;
- `);
- return db;
}
}
- return new SQL.Database();
+ if (db === null) {
+ db = new SQL.Database();
+ }
+
+ // Define Unicode-aware LOWER and UPPER implementation.
+ // This is necessary because sql.js uses SQLite build without ICU support.
+ //
+ // Note that this function should ideally be created with a deterministic flag
+ // to allow SQLite to better optimize calls to it by factoring them out of inner loops
+ // but SQL.js does not support this: https://github.com/sql-js/sql.js/issues/551
+ db.create_function('UNICODE_LOWER', arg => arg?.toLowerCase());
+ db.create_function('UNICODE_UPPER', arg => arg?.toUpperCase());
+ return db;
}
export function closeDatabase(db) {
diff --git a/packages/loot-core/src/server/accounts/payees.js b/packages/loot-core/src/server/accounts/payees.js
index 23f726f3e..9c4c55d0c 100644
--- a/packages/loot-core/src/server/accounts/payees.js
+++ b/packages/loot-core/src/server/accounts/payees.js
@@ -4,7 +4,7 @@ export async function createPayee(description) {
// Check to make sure no payee already exists with exactly the same
// name
let row = await db.first(
- `SELECT id FROM payees WHERE LOWER(name) = ? AND tombstone = 0`,
+ `SELECT id FROM payees WHERE UNICODE_LOWER(name) = ? AND tombstone = 0`,
[description.toLowerCase()],
);
diff --git a/packages/loot-core/src/server/aql/compiler.js b/packages/loot-core/src/server/aql/compiler.js
index 47564fa3e..fce92d9eb 100644
--- a/packages/loot-core/src/server/aql/compiler.js
+++ b/packages/loot-core/src/server/aql/compiler.js
@@ -564,7 +564,7 @@ const compileFunction = saveStack('function', (state, func) => {
case '$lower': {
validateArgLength(args, 1);
let [arg1] = valArray(state, args, ['string']);
- return typed(`LOWER(${arg1})`, 'string');
+ return typed(`UNICODE_LOWER(${arg1})`, 'string');
}
// integer/float functions
diff --git a/packages/loot-core/src/server/db/index.js b/packages/loot-core/src/server/db/index.js
index fc406c145..c3901d39b 100644
--- a/packages/loot-core/src/server/db/index.js
+++ b/packages/loot-core/src/server/db/index.js
@@ -516,9 +516,10 @@ export async function getOrphanedPayees() {
}
export async function getPayeeByName(name) {
- return first(`SELECT * FROM payees WHERE LOWER(name) = ? AND tombstone = 0`, [
- name.toLowerCase(),
- ]);
+ return first(
+ `SELECT * FROM payees WHERE UNICODE_LOWER(name) = ? AND tombstone = 0`,
+ [name.toLowerCase()],
+ );
}
export function insertPayeeRule(rule) {
diff --git a/upcoming-release-notes/865.md b/upcoming-release-notes/865.md
new file mode 100644
index 000000000..e670684dc
--- /dev/null
+++ b/upcoming-release-notes/865.md
@@ -0,0 +1,6 @@
+---
+category: Bugfix
+authors: [Jackenmen]
+---
+
+Fix case-insensitive matching of strings for uppercase letters from non-English alphabets
--
GitLab