feat(db): keep SQLite's write and search semantics on Postgres

- UnitOfWork transactions take one advisory lock for the database on
  Postgres (DatabaseEngine.serializeWrites, pg_advisory_xact_lock), so a
  MAX(...) + 1 read inside one cannot race another, as with SQLite's single
  writer. SQLite needs nothing.
- ilike(platform, refs) builds the case-insensitive search condition: LIKE
  on SQLite, ILIKE on Postgres, where LIKE is exact. The place searches use it.
- Plugin data erasure runs each statement in its own savepoint: on Postgres a
  failed statement would abort the user's whole erasure transaction.
This commit is contained in:
Isaias Tavares
2026-11-02 14:58:29 -03:00
parent a77705984e
commit 515caffa79
13 changed files with 98 additions and 15 deletions
+11
View File
@@ -620,6 +620,17 @@ export function collateNoCase(platform: Platform, ref: string): RawQueryFragment
return unsupported(platform);
}
/**
* A search that ignores case over `refs`: `(a LIKE ? ESCAPE '\' OR b LIKE ?
* ...)`, one bound pattern per column, the caller escaping `%`, `_` and `\`.
* SQLite's `LIKE` already ignores case (ASCII letters only); Postgres's is
* exact, so it gets `ILIKE`, which folds every letter.
*/
export function ilike(platform: Platform, refs: readonly string[]): string {
const operator = isSqlite(platform) ? 'LIKE' : isPostgres(platform) ? 'ILIKE' : unsupported(platform);
return `(${refs.map((ref) => `${column(ref)} ${operator} ? ESCAPE '\\'`).join(' OR ')})`;
}
/**
* `datetime('now', '-N hours')` — the hour-granularity sibling of
* `nowMinusDays()` above, for `reminder-jobs.service.ts#todoTick`'s
+7 -1
View File
@@ -1,5 +1,5 @@
import type { AnyOrm } from '../orm';
import type { Options } from '@mikro-orm/core';
import type { EntityManager, Options } from '@mikro-orm/core';
/** The databases TREK can be pointed at. MySQL/MariaDB are out of scope (server/CLAUDE.md). */
type EngineId = 'sqlite' | 'postgres';
@@ -37,4 +37,10 @@ export interface DatabaseEngine {
* archive is the copy).
*/
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void>;
/**
* Runs first in every `UnitOfWork` transaction, on its EntityManager: makes
* the transactions that write run one at a time, as SQLite's single writer
* does, so a `MAX(...) + 1` read inside one cannot race another.
*/
serializeWrites(em: EntityManager): Promise<void>;
}
@@ -3,6 +3,7 @@ import type { DatabaseEngine } from '../engine';
import { redactUrl } from '../redact-url';
import { TrekPostgreSqlDriver } from './postgres-driver';
import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap';
import type { EntityManager } from '@mikro-orm/core';
import { readFileSync } from 'node:fs';
@@ -53,6 +54,9 @@ export function sslOptionFor(url: string): PgSslOption | undefined {
}
}
/** The advisory lock key every write transaction takes: "TREK" in ASCII. */
const TREK_WRITE_LOCK = 0x5452454b;
/**
* The Postgres engine: MikroORM's own `pg` driver on `TREK_DB_URL`, with a
* pool of at most `TREK_DB_POOL_MAX` connections that keeps none open while
@@ -80,6 +84,17 @@ export class PostgresEngine implements DatabaseEngine {
return Promise.reject(new Error('Restoring a backup into a Postgres database is not available in this release.'));
}
/**
* `pg_advisory_xact_lock` on one key for the whole database: held until the
* transaction commits or rolls back, taken again by a savepoint in the same
* session without waiting, and shared by every process on the database.
*/
async serializeWrites(em: EntityManager): Promise<void> {
await em
.getConnection()
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
}
async schemaBootstrap(orm: AnyOrm): Promise<void> {
await bootstrapPostgresSchema(orm, this.bootstrapDeps());
await finishSchemaBootstrap(orm);
@@ -3,6 +3,7 @@ import { resolveDbPath } from '../../db-path';
import { runSchemaBootstrap, type AnyOrm } from '../../orm';
import { SharedSqliteDriver } from '../../orm-driver';
import type { DatabaseEngine } from '../engine';
import type { EntityManager } from '@mikro-orm/core';
/**
* The SQLite file engine, TREK's default: one better-sqlite3 handle that
@@ -40,4 +41,9 @@ export class SqliteEngine implements DatabaseEngine {
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
return runSchemaBootstrap(orm, options);
}
serializeWrites(_em: EntityManager): Promise<void> {
// One connection, one writer: SQLite serializes the transactions already.
return Promise.resolve();
}
}
@@ -69,14 +69,20 @@ export class MaintenanceRepository {
* otherwise, so passing it through is correct both inside and outside a
* transaction. Each statement is independently best-effort: a
* slimmed-down test schema may not carry one or more of these tables at
* all, matching the legacy per-table try/catch exactly.
* all, matching the legacy per-table try/catch exactly. Each one runs in
* its own savepoint (a nested `em.transactional`), because on Postgres a
* failed statement aborts the whole transaction it is in: the savepoint
* rolls back that one statement and the user's erasure goes on.
*/
async deletePluginUserData(userId: number): Promise<void> {
this.validateRequestContext();
const ctx = this.em.getTransactionContext();
for (const table of ['plugin_user_config', 'plugin_oauth_tokens', 'plugin_oauth_state']) {
try {
await this.em.getConnection().execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', ctx);
await this.em.transactional((em) =>
em
.getConnection()
.execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', em.getTransactionContext()),
);
} catch {
/* table absent (slim schema) */
}
@@ -1,6 +1,6 @@
import type { Places } from '../entities/Places.entity';
import { wrapOptionalLongitude } from '@trek/shared';
import { absDifference, coalesceParam, columnRef, currentTimestamp, lowerTrim } from '../dialect/sql-functions';
import { absDifference, coalesceParam, columnRef, currentTimestamp, ilike, lowerTrim } from '../dialect/sql-functions';
import { type AssertRowKeys } from './_shared/rows';
import { TrekRepository } from './_shared/trek-repository';
import { tourPlaceTextPatch, type TourPlaceWrite } from './tour-place-write';
@@ -680,10 +680,8 @@ export class PlacesRepository extends TrekRepository<Places> {
.where('p.trip_id = ?', [trip_id]);
if (filters.searchPattern) {
qb.andWhere(
"(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\' OR p.description LIKE ? ESCAPE '\\')",
[filters.searchPattern, filters.searchPattern, filters.searchPattern],
);
const fields = ['p.name', 'p.address', 'p.description'];
qb.andWhere(ilike(platform, fields), fields.map(() => filters.searchPattern));
}
if (filters.category !== undefined) {
qb.andWhere('p.category_id = ?', [filters.category]);
@@ -1,6 +1,6 @@
import { QueryFlag } from '@mikro-orm/core';
import type { Trips } from '../entities/Trips.entity';
import { coalesceParam, currentTimestamp, nowDateOffset } from '../dialect/sql-functions';
import { coalesceParam, currentTimestamp, ilike, nowDateOffset } from '../dialect/sql-functions';
import type { AssertRowKeys } from './_shared/rows';
import { TrekRepository } from './_shared/trek-repository';
import { tripAccessExpr } from './_shared/trip-access';
@@ -174,7 +174,7 @@ export class TripsRepository extends TrekRepository<Trips> {
return await this.accessibleTripsQuery(user_id)
.join('t.places_collection', 'p')
.select(['p.trip', 'p.name'])
.andWhere("(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\')", [likePattern, likePattern])
.andWhere(ilike(this.getEntityManager().getPlatform(), ['p.name', 'p.address']), [likePattern, likePattern])
.orderBy([{ 'p.trip': 'asc' }, { 'p.name': 'asc' }])
.limit(500)
// The limit is on the joined place rows, like the legacy statement's.
+12 -1
View File
@@ -1,3 +1,4 @@
import { resolveEngine } from '../../db/engine';
import { EntityManager } from '@mikro-orm/core';
import { Injectable } from '@nestjs/common';
@@ -17,12 +18,22 @@ import { Injectable } from '@nestjs/common';
*
* Never awaits network I/O inside: the SQLite connection is one Kysely client
* with a mutex, and an open transaction holds it for every other request.
*
* The services read `MAX(...) + 1` inside these transactions on the strength
* of SQLite's single writer. On Postgres a pool of connections would let two
* of them read the same MAX, so every transaction first takes the engine's
* write lock (`DatabaseEngine.serializeWrites`), which keeps that guarantee.
*/
@Injectable()
export class UnitOfWork {
private readonly engine = resolveEngine();
constructor(private readonly em: EntityManager) {}
transactional<T>(fn: () => Promise<T>): Promise<T> {
return this.em.transactional(() => fn());
return this.em.transactional(async (em) => {
await this.engine.serializeWrites(em);
return fn();
});
}
}