feat(db): run TREK on PostgreSQL

TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before
otherwise: an engine port under src/db/engine, the dialect helpers under
src/db/dialect, a Postgres baseline for the migrations up to the cut,
pg_dump/pg_restore backups, a probe of every repository method against a
real server and a Postgres test lane in CI.
This commit is contained in:
Isaias Tavares
2026-11-05 01:15:21 -03:00
committed by Maurice
parent 17b47b38b9
commit 2e3762afe1
176 changed files with 6728 additions and 777 deletions
+37
View File
@@ -0,0 +1,37 @@
import { isPostgres, isSqlite, unsupported } from './platform';
import type { Platform } from '@mikro-orm/core';
/**
* The schema catalog, for the migrations' helpers (`db/migration-utils.ts`):
* which tables and columns exist, each spelled the way its engine keeps them.
* Both statements take the table name as their one bound parameter and answer
* one row per hit with the name in `name`. Postgres reads the current schema
* only, the one TREK's own tables live in.
*/
/** Lists the columns of the table bound as `?`, one `name` per row; none for a missing table. */
export function columnNamesSql(platform: Platform): string {
if (isSqlite(platform)) return 'select name from pragma_table_info(?)';
if (isPostgres(platform)) {
return 'select column_name as name from information_schema.columns where table_schema = current_schema() and table_name = ?';
}
return unsupported(platform);
}
/** One row when the table bound as `?` exists, none otherwise. */
export function tableExistsSql(platform: Platform): string {
if (isSqlite(platform)) return "select name from sqlite_master where type = 'table' and name = ?";
if (isPostgres(platform)) {
return 'select table_name as name from information_schema.tables where table_schema = current_schema() and table_name = ?';
}
return unsupported(platform);
}
/**
* The tables in the current schema, one `table_name` per row in name order:
* the schema TREK's own tables live in. Postgres only, and the one definition
* of that question: the bootstrap reads it through the connection, the client
* tools' table listing hands it to psql.
*/
export const CURRENT_SCHEMA_TABLES_SQL =
'select table_name from information_schema.tables where table_schema = current_schema() order by table_name';
+11
View File
@@ -620,6 +620,17 @@ export function collateNoCase(platform: Platform, ref: string): RawQueryFragment
return unsupported(platform);
}
/**
* A search that ignores case over `refs`: `(a LIKE ? ESCAPE '\' OR b LIKE ?
* ...)`, one bound pattern per column, the caller escaping `%`, `_` and `\`.
* SQLite's `LIKE` already ignores case (ASCII letters only); Postgres's is
* exact, so it gets `ILIKE`, which folds every letter.
*/
export function ilike(platform: Platform, refs: readonly string[]): string {
const operator = isSqlite(platform) ? 'LIKE' : isPostgres(platform) ? 'ILIKE' : unsupported(platform);
return `(${refs.map((ref) => `${column(ref)} ${operator} ? ESCAPE '\\'`).join(' OR ')})`;
}
/**
* `datetime('now', '-N hours')` — the hour-granularity sibling of
* `nowMinusDays()` above, for `reminder-jobs.service.ts#todoTick`'s
+14 -5
View File
@@ -18,13 +18,22 @@ export async function ensureEmailCaseIndex(connection: Connection): Promise<void
const pending: unknown[] = await connection.execute(`SELECT value FROM app_settings WHERE key = ?`, [PENDING_KEY]);
if (pending.length === 0) return;
const duplicates: Array<{ ids: string }> = await connection.execute(
`SELECT group_concat(id, ', ') AS ids FROM (SELECT id, email FROM users ORDER BY id) GROUP BY lower(email) HAVING COUNT(*) > 1`,
// Grouped here rather than with group_concat, which Postgres spells string_agg.
const rows: Array<{ id: number; email_key: string }> = await connection.execute(
`SELECT id, lower(email) AS email_key FROM users WHERE lower(email) IN ` +
`(SELECT lower(email) FROM users GROUP BY lower(email) HAVING COUNT(*) > 1) ORDER BY id`,
);
if (duplicates.length > 0) {
const duplicates = new Map<string, number[]>();
for (const row of rows) {
const group = duplicates.get(row.email_key);
if (group) group.push(row.id);
else duplicates.set(row.email_key, [row.id]);
}
if (duplicates.size > 0) {
const ids = [...duplicates.values()].map((group) => group.join(', '));
console.warn(
`[DB] ${duplicates.length} email address(es) still belong to more than one account when case is ignored ` +
`(user ids ${duplicates.map((row) => row.ids).join('; ')}), so users.email has no case-insensitive unique ` +
`[DB] ${duplicates.size} email address(es) still belong to more than one account when case is ignored ` +
`(user ids ${ids.join('; ')}), so users.email has no case-insensitive unique ` +
'index yet. Give each of those accounts its own address under Admin > Users; the next start creates it.',
);
return;
+49
View File
@@ -0,0 +1,49 @@
import type { EngineId } from '../../app-config/db-settings';
import type { AnyOrm } from '../orm';
import type { EntityManager, Options } from '@mikro-orm/core';
/**
* One database engine: what the ORM needs to reach it, how the connection
* opens, closes and comes back after a restore, and how a database is brought
* to the current schema. Code that differs by engine dispatches through this,
* or through the helpers in `db/dialect/`, never through its own platform
* check (`lint:engine-dispatch`).
*
* Free of Nest like every other file under `db/`: `src/db` does not import
* `src/nest` (`lint:boundaries`). `DatabaseLifecycle` is the provider that
* calls it.
*/
export interface DatabaseEngine {
readonly id: EngineId;
/** For log lines and errors: the file path, or the URL without its password. */
describe(): string;
/** What MikroORM needs to reach the database: the driver and its target, the pool, driver options. */
ormOptions(): Partial<Options>;
/**
* Called once at boot, before the schema bootstrap. `onReopened` runs after
* each later reopen, once the ORM is pointed at the reopened database: the
* engine does what its own reopen needs first (SQLite hands the ORM a new
* handle, so the ORM's connection is closed and connected again), then calls
* it, and the caller migrates the database and runs whatever follows.
*/
open(orm: AnyOrm, onReopened: () => Promise<void>): void;
/** Closes what the engine itself holds; the ORM's own pool is closed with the Nest container. */
close(): void;
/**
* Comes back after the database was replaced by a restore, then runs the
* `onReopened` that `open` registered. Throws when `open` has not run.
*/
reopen(): Promise<void>;
/**
* Brings the database to the current schema, then seeds it. `snapshot:
* false` skips the copy taken before pending migrations (a restore, whose
* archive is the copy).
*/
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void>;
/**
* Runs first in every `UnitOfWork` transaction, on its EntityManager: makes
* the transactions that write run one at a time, as SQLite's single writer
* does, so a `MAX(...) + 1` read inside one cannot race another.
*/
serializeWrites(em: EntityManager): Promise<void>;
}
+58
View File
@@ -0,0 +1,58 @@
import { readEnv } from '../../app-config';
import type { DbEngineSettings } from '../../app-config/db-settings';
import type { DbSettings } from '../../app-config/derive';
import { ALL_ENTITIES } from '../entities';
import type { DatabaseEngine } from './engine';
import { PostgresEngine } from './postgres/postgres-engine';
import { SqliteEngine } from './sqlite/sqlite-engine';
import { defineConfig, type Options } from '@mikro-orm/core';
import { Migrator } from '@mikro-orm/migrations';
import { SeedManager } from '@mikro-orm/seeder';
export type { DatabaseEngine } from './engine';
/**
* The engine this process runs on, from `TREK_DB_URL` (`readEnv().db`): the
* SQLite file unless the URL names a Postgres database. Read afresh on every
* call, like everything `readEnv()` answers, so nothing caches a choice a test
* or a restart changes.
*/
export function resolveEngine(db: DbEngineSettings = readEnv().db): DatabaseEngine {
if (db.engine === 'postgres') return new PostgresEngine(db);
return new SqliteEngine();
}
/**
* The Postgres URL and the directory of its client tools, when the settings
* name a Postgres database; null on SQLite. For the code that drives pg_dump
* and pg_restore, which needs both and must not read them from the environment
* again.
*/
export function postgresTarget(db: DbSettings): { url: string; binDir: string | undefined } | null {
if (db.engine !== 'postgres') return null;
return { url: db.url, binDir: db.pgBinDir };
}
/**
* The MikroORM config for `engine`: what every engine shares (entities,
* migrations, seeders) and what the engine adds. `src/mikro-orm.config.ts`
* exports it for the CLI and the Nest container, so both reach the same
* database.
*/
export function buildOrmConfig(engine: DatabaseEngine): Options {
return defineConfig({
entities: ALL_ENTITIES,
extensions: [Migrator, SeedManager],
migrations: {
path: 'dist/db/migrations',
pathTs: 'src/db/migrations',
snapshot: false,
},
seeder: {
path: 'dist/db/seeders',
pathTs: 'src/db/seeders',
defaultSeeder: 'DatabaseSeeder',
},
...engine.ormOptions(),
}) as Options;
}
+93
View File
@@ -0,0 +1,93 @@
import { splitStatements } from './sql-split';
import fs from 'node:fs';
import path from 'node:path';
/**
* Where the Postgres baseline lives: `server/assets/postgres/`, which the
* Docker image copies beside `dist/` (the same place the airports and atlas
* data are read from), reached the same way from `src/` and from `dist/`.
*/
const BASELINE_DIR = path.join(__dirname, '..', '..', '..', '..', 'assets', 'postgres');
const SQL_FILE = 'baseline.sql';
const MANIFEST_FILE = 'baseline.json';
/** The manifest beside the SQL: the last migration the baseline stands for, and every migration it replaces. */
interface BaselineManifest {
through: string;
migrations: string[];
}
/**
* The schema a fresh Postgres database starts from.
*
* The migrations up to `through` are written in SQLite and frozen, so a
* Postgres database does not replay them: it runs these statements, generated
* from the entities (`npm run gen:pg-baseline`), and records every migration
* in `migrations` as executed in the same transaction. Migrations after
* `through` run on both engines through the migrator.
*/
export interface PostgresBaseline extends BaselineManifest {
statements: string[];
}
/** The two files a baseline is written as, ready to compare or write. */
export function renderBaseline(baseline: PostgresBaseline): { sql: string; manifest: string } {
return {
sql:
'-- The schema a fresh Postgres database starts from. Generated by `npm run gen:pg-baseline`; do not edit.\n' +
baseline.statements.map((statement) => `${statement};\n`).join(''),
manifest: `${JSON.stringify({ through: baseline.through, migrations: baseline.migrations }, null, 2)}\n`,
};
}
/**
* Whether `data` is a manifest a baseline can be read from: a `through` naming
* one of a non-empty list of `migrations`, every entry a string.
*/
function isBaselineManifest(data: unknown): data is BaselineManifest {
if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false;
const { through, migrations } = data;
return (
typeof through === 'string' &&
Array.isArray(migrations) &&
migrations.length > 0 &&
migrations.every((name): name is string => typeof name === 'string') &&
migrations.includes(through)
);
}
function parseManifest(text: string, file: string): BaselineManifest {
const data: unknown = JSON.parse(text);
if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`);
return { through: data.through, migrations: data.migrations };
}
/** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */
export function readPostgresBaseline(dir = BASELINE_DIR): PostgresBaseline {
const manifestFile = path.join(dir, MANIFEST_FILE);
const manifest = parseManifest(fs.readFileSync(manifestFile, 'utf8'), manifestFile);
const statements = splitStatements(fs.readFileSync(path.join(dir, SQL_FILE), 'utf8').replace(/^--[^\n]*\n/gm, ''));
if (statements.length === 0) throw new Error(`${path.join(dir, SQL_FILE)} holds no statements`);
// The boot sends the baseline as one statement without parameters, where a `?` would be read as a binding.
if (statements.some((statement) => statement.includes('?'))) {
throw new Error(`${path.join(dir, SQL_FILE)} holds a "?", which the boot would read as a parameter`);
}
return { ...manifest, statements };
}
/** Writes the two files (the generator's `--write`). */
export function writePostgresBaseline(baseline: PostgresBaseline, dir = BASELINE_DIR): void {
const { sql, manifest } = renderBaseline(baseline);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, SQL_FILE), sql);
fs.writeFileSync(path.join(dir, MANIFEST_FILE), manifest);
}
/** Whether the committed files are what `baseline` renders to (the generator's `--check`). */
export function baselineMatches(baseline: PostgresBaseline, dir = BASELINE_DIR): boolean {
const { sql, manifest } = renderBaseline(baseline);
const read = (file: string) =>
fs.existsSync(path.join(dir, file)) ? fs.readFileSync(path.join(dir, file), 'utf8') : null;
return read(SQL_FILE) === sql && read(MANIFEST_FILE) === manifest;
}
@@ -0,0 +1,41 @@
/**
* Drops every table, view, sequence, routine, collation and standalone type in
* the current schema, except an extension's, before a dump is loaded: a dump
* only recreates what it holds, so whatever a later migration added would
* otherwise survive the restore and break the migrations the reopen runs. It
* runs in the same transaction as the load (`pgRestore`), so a restore that
* fails leaves the database as it was.
*/
export const CLEAR_SCHEMA_SQL = `DO $clear$
DECLARE r record;
BEGIN
FOR r IN SELECT c.oid::regclass AS name, c.relkind FROM pg_class c
WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype IN ('e', 'a', 'i'))
LOOP
EXECUTE format('DROP %s IF EXISTS %s CASCADE',
CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' WHEN 'S' THEN 'SEQUENCE'
WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END, r.name);
END LOOP;
FOR r IN SELECT p.oid::regprocedure AS sig FROM pg_proc p
WHERE p.pronamespace = current_schema()::regnamespace
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
LOOP
EXECUTE format('DROP ROUTINE IF EXISTS %s CASCADE', r.sig);
END LOOP;
FOR r IN SELECT format('%I.%I', n.nspname, c.collname) AS name FROM pg_collation c
JOIN pg_namespace n ON n.oid = c.collnamespace
WHERE n.nspname = current_schema()
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype = 'e')
LOOP
EXECUTE format('DROP COLLATION IF EXISTS %s CASCADE', r.name);
END LOOP;
FOR r IN SELECT t.oid::regtype AS name FROM pg_type t
WHERE t.typnamespace = current_schema()::regnamespace AND t.typtype IN ('e', 'd', 'r')
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = t.oid AND d.deptype IN ('e', 'i'))
LOOP
EXECUTE format('DROP TYPE IF EXISTS %s CASCADE', r.name);
END LOOP;
END
$clear$;
`;
+119
View File
@@ -0,0 +1,119 @@
import { execFile } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { promisify } from 'node:util';
const execFileAsync = promisify(execFile);
/** The Postgres client tools a backup runs. */
export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
/**
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
* Debian's versioned directories newest first (the Docker image installs 17),
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
*/
const DEFAULT_BIN_DIRS = [
'/usr/lib/postgresql/17/bin',
'/usr/lib/postgresql/16/bin',
'/usr/lib/postgresql/15/bin',
'/usr/lib/postgresql/14/bin',
'/usr/local/bin',
'/usr/bin',
];
/** A dump or a restore of a large database takes minutes, not hours. */
const TOOL_TIMEOUT_MS = 30 * 60 * 1000;
const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
/** An unreachable host fails the tool in seconds instead of holding a boot or a restore. */
const CONNECT_TIMEOUT_SECONDS = '10';
/** How much of a tool's stderr an error keeps. */
const STDERR_LIMIT = 2000;
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
const dirs = binDir ? [binDir] : DEFAULT_BIN_DIRS;
const executable = (candidate: string): boolean => {
try {
fs.accessSync(candidate, fs.constants.X_OK);
return true;
} catch {
return false; // missing or not executable: the next directory may have it
}
};
return dirs.map((dir) => path.join(dir, tool)).find(executable) ?? null;
}
/**
* The libpq environment for `url`: the tools read the password from
* PGPASSWORD, so it never appears in an argument list another process can
* read. An sslmode that asks for SSL verifies the server's certificate, as the
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
* else the system's trusted CAs.
*/
export function pgConnectionEnv(url: string): Record<string, string> {
const parsed = new URL(url);
const env: Record<string, string> = {
PGHOST: parsed.hostname.replace(/^\[(.*)\]$/, '$1'),
PGPORT: parsed.port || '5432',
PGDATABASE: decodeURIComponent(parsed.pathname.replace(/^\//, '')),
PGCONNECT_TIMEOUT: CONNECT_TIMEOUT_SECONDS,
};
if (parsed.username) env.PGUSER = decodeURIComponent(parsed.username);
if (parsed.password) env.PGPASSWORD = decodeURIComponent(parsed.password);
const sslmode = parsed.searchParams.get('sslmode');
if (sslmode === 'disable') env.PGSSLMODE = 'disable';
else if (sslmode) {
env.PGSSLMODE = 'verify-full';
env.PGSSLROOTCERT = parsed.searchParams.get('sslrootcert') ?? 'system';
}
return env;
}
/** A client tool, refused with a clear message when it is not installed. */
export class PgToolMissingError extends Error {
constructor(tool: PgTool) {
super(
`${tool} was not found, so a Postgres database cannot be backed up or restored. ` +
'Install the PostgreSQL client tools (the Docker image has them) or point TREK_DB_PG_BIN_DIR at them.',
);
this.name = 'PgToolMissingError';
}
}
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
function summarizeStderr(stderr: string | undefined): string {
const kept = (stderr ?? '')
.split('\n')
.filter((line) => line.trim() !== '' && !DATA_LINE.test(line))
.join('\n');
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
}
/**
* Runs `tool` by its absolute path and resolves with its stdout. A failure
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
* the cause.
*/
export async function run(
tool: PgTool,
binDir: string | undefined,
args: string[],
env: Record<string, string>,
): Promise<string> {
const binary = resolvePgTool(tool, binDir);
if (!binary) throw new PgToolMissingError(tool);
try {
const { stdout } = await execFileAsync(binary, args, {
env,
timeout: TOOL_TIMEOUT_MS,
maxBuffer: TOOL_MAX_OUTPUT_BYTES,
});
return stdout;
} catch (err) {
const message = summarizeStderr((err as { stderr?: string }).stderr);
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
}
}
+206
View File
@@ -0,0 +1,206 @@
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
import { CLEAR_SCHEMA_SQL } from './clear-schema';
import { pgConnectionEnv, run } from './pg-process';
import { refuseToolsWithoutRestrict } from './restore-safety';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import readline from 'node:readline';
/**
* Restore's session settings: a restore waiting on a lock another session
* holds gives up, and rolls back, instead of hanging; the NOTICEs every
* CASCADE prints stay out of its stderr.
*/
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
function createPrivateFile(file: string): void {
fs.writeFileSync(file, '', { mode: 0o600 });
}
/** The schema TREK's tables live in: its name, and the name as SQL quotes it (`quote_ident`). */
interface PgSchema {
name: string;
quoted: string;
}
/**
* The current schema of the database at `url`, the one TREK creates its
* tables in. A backup and a restore stay inside it: a hosted database
* (Supabase) keeps the platform's own schemas, `auth` among them, beside it,
* and they are neither TREK's to copy nor to replace.
*/
async function pgCurrentSchema(url: string, binDir: string | undefined): Promise<PgSchema> {
const out = await run(
'psql',
binDir,
[
'--no-psqlrc',
'--no-align',
'--tuples-only',
"--command=select concat_ws(E'\\t', current_schema(), quote_ident(current_schema()))",
],
pgConnectionEnv(url),
);
const [name, quoted] = out.replace(/\n$/, '').split('\t');
if (!name || !quoted) {
throw new Error('The database has no current schema: set the search_path of the role in TREK_DB_URL.');
}
return { name, quoted };
}
/**
* Writes a custom-format dump of the current schema of the database at `url`
* to `target`, readable by its owner only. It is written beside the target and
* renamed into place, so a dump that fails or is killed never leaves a file
* that passes for a copy.
*/
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
const partial = `${target}.partial`;
try {
createPrivateFile(partial);
const schema = await pgCurrentSchema(url, binDir);
await run(
'pg_dump',
binDir,
[
'--format=custom',
'--no-owner',
'--no-privileges',
`--file=${partial}`,
`--schema="${schema.name.replaceAll('"', '""')}"`,
],
pgConnectionEnv(url),
);
fs.renameSync(partial, target);
} catch (err) {
fs.rmSync(partial, { force: true });
throw err;
}
}
/** Escapes every character a RegExp would read as syntax. */
function escapeRegExp(text: string): string {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
* Whether the restore script creates TREK's migration table in `schema`: a
* dump of TREK's own schema does. The script is read line by line and the
* read stops at the first match: the COPY data that fills a large script is
* never loaded, and a script past V8's maximum string length would not load
* at all.
*/
async function scriptHoldsTrekSchema(script: string, schema: PgSchema): Promise<boolean> {
const createsHistory = new RegExp(`^CREATE TABLE ${escapeRegExp(`${schema.quoted}.mikro_orm_migrations`)} \\(`);
const input = fs.createReadStream(script, { encoding: 'utf8' });
const lines = readline.createInterface({ input, crlfDelay: Infinity });
try {
for await (const line of lines) {
if (createsHistory.test(line)) return true;
}
return false;
} finally {
lines.close();
input.destroy();
}
}
/**
* Replaces the current schema of the database at `url` with the dump at
* `source`: pg_restore turns the dump's objects in that schema into a script
* beside it (anything else the dump holds is left out), then psql clears the
* schema and runs the script in one transaction, so a restore that fails
* leaves the database as it was. A dump without TREK's tables in that schema
* is refused before anything is cleared.
*/
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
await refuseToolsWithoutRestrict(binDir);
const schema = await pgCurrentSchema(url, binDir);
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
try {
const clear = path.join(workDir, 'clear-schema.sql');
const script = path.join(workDir, 'restore.sql');
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
createPrivateFile(script);
await run(
'pg_restore',
binDir,
['--no-owner', '--no-privileges', `--schema=${schema.name}`, `--file=${script}`, '--', source],
{},
);
if (!(await scriptHoldsTrekSchema(script, schema))) {
throw new Error(`The backup holds no TREK tables in the schema "${schema.name}"; nothing was replaced.`);
}
await run(
'psql',
binDir,
[
'--no-psqlrc',
'--quiet',
'--single-transaction',
'--set=ON_ERROR_STOP=1',
`--output=${os.devNull}`,
`--file=${clear}`,
`--file=${script}`,
],
{ ...pgConnectionEnv(url), PGOPTIONS: RESTORE_SESSION_OPTIONS },
);
} finally {
fs.rmSync(workDir, { recursive: true, force: true });
}
}
/** The tables in the current schema of the database at `url`, read before TREK opens it. */
export async function pgTableNames(url: string, binDir: string | undefined): Promise<string[]> {
const out = await run(
'psql',
binDir,
['--no-psqlrc', '--no-align', '--tuples-only', `--command=${CURRENT_SCHEMA_TABLES_SQL}`],
pgConnectionEnv(url),
);
return out.split('\n').filter((line) => line !== '');
}
/**
* The migration names in a restore script's COPY block for
* `mikro_orm_migrations`, read from its `name` column; none when the script
* has no such block.
*/
function migrationNamesFromCopy(script: string): string[] {
const lines = script.split('\n');
const copy = lines.findIndex((line) => /^COPY \S*mikro_orm_migrations \(/.test(line));
if (copy < 0) return [];
const columns =
/\(([^)]*)\)/
.exec(lines[copy])?.[1]
.split(',')
.map((column) => column.trim()) ?? [];
const nameAt = columns.indexOf('name');
if (nameAt < 0) return [];
const names: string[] = [];
for (const line of lines.slice(copy + 1)) {
if (line === '\\.' || line === '') break;
const name = line.split('\t')[nameAt];
if (name) names.push(name);
}
return names;
}
/**
* The migrations a dump records as executed, read from the dump itself (its
* `mikro_orm_migrations` rows as COPY text), so a restore can refuse a newer
* TREK's backup before anything is replaced. Throws on a file that is not a
* dump; an empty list means the dump holds no TREK history.
*/
export async function pgDumpMigrations(source: string, binDir: string | undefined): Promise<string[]> {
const script = await run(
'pg_restore',
binDir,
['--data-only', '--table=mikro_orm_migrations', '--file=-', '--', source],
{},
);
return migrationNamesFromCopy(script);
}
@@ -0,0 +1,67 @@
import { type Configuration } from '@mikro-orm/core';
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
/** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */
const INT8_OID = 20;
/** `numeric`: what `sum()` of a bigint and `avg()` answer. */
const NUMERIC_OID = 1700;
/**
* Reads a `bigint` as a JavaScript number, the way better-sqlite3 hands
* SQLite's 64-bit INTEGER to the services. `pg` returns it as a string to
* keep precision past 2^53; no TREK value comes near that (ids, counts,
* epoch milliseconds, cents), so one that does is refused rather than
* rounded.
*/
export function parseInt8(text: string): number {
const value = Number(text);
if (!Number.isSafeInteger(value)) throw new Error(`A bigint of ${text} is beyond what TREK reads exactly (2^53).`);
return value;
}
/**
* Reads a `numeric` as a number, as SQLite answers `sum()` and `avg()`. `NaN`
* and the infinities are refused, as is an integral result past 2^53, which is
* refused like a bigint; a fraction is a double, the precision SQLite's REAL has.
*/
export function parseNumeric(text: string): number {
const value = Number(text);
if (!Number.isFinite(value)) {
throw new Error(`A numeric of ${text} is not a finite number, which TREK does not read.`);
}
if (/^-?\d+$/.test(text) && !Number.isSafeInteger(value)) {
throw new Error(`A numeric of ${text} is beyond what TREK reads exactly (2^53).`);
}
return value;
}
/**
* MikroORM's `PostgreSqlConnection`, with the pool's own type parsers replaced
* for `int8` and `numeric`. MikroORM keeps dates as text, so only these two
* change. `mapOptions` builds the `pg` pool options, whose `types` is the
* `pg` TypeOverrides, the object that holds the parsers.
*/
class NumberTypesConnection extends PostgreSqlConnection {
override mapOptions(overrides: Parameters<PostgreSqlConnection['mapOptions']>[0]) {
const options = super.mapOptions(overrides);
const types = options.types as { setTypeParser(oid: number, parse: (text: string) => unknown): void };
types.setTypeParser(INT8_OID, parseInt8);
types.setTypeParser(NUMERIC_OID, parseNumeric);
return options;
}
}
/**
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
* as numbers. The parsers belong to this pool only: nothing global in `pg`
* changes.
*/
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
constructor(config: Configuration) {
super(config);
// PostgreSqlDriver's constructor hardcodes its connection class, as
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
// field, which is not public, so it is replaced here, before anything connects.
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
}
}
@@ -0,0 +1,90 @@
import type { DbEngineSettings } from '../../../app-config/db-settings';
import { NO_SAFETY_NET } from '../../legacy-baseline';
import { finishSchemaBootstrap, type AnyOrm } from '../../orm';
import type { DatabaseEngine } from '../engine';
import { redactUrl } from '../redact-url';
import { TrekPostgreSqlDriver } from './postgres-driver';
import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap';
import { sslOptionFor } from './ssl';
import type { EntityManager } from '@mikro-orm/core';
/** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */
type PostgresSettings = Extract<DbEngineSettings, { engine: 'postgres' }>;
/** The advisory lock key every write transaction takes: "TREK" in ASCII. */
const TREK_WRITE_LOCK = 0x5452454b;
/**
* The Postgres engine: MikroORM's own `pg` driver on `TREK_DB_URL`, with a
* pool of at most `TREK_DB_POOL_MAX` connections that keeps none open while
* idle. The pool belongs to the ORM, which the Nest container closes; the
* schema comes from the baseline plus the migrations after it
* (`schema-bootstrap.ts`).
*/
export class PostgresEngine implements DatabaseEngine {
readonly id = 'postgres';
constructor(
private readonly settings: PostgresSettings,
private readonly bootstrapDeps: (url: string) => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
) {}
private onReopened: (() => Promise<void>) | null = null;
/**
* Nothing of its own to open: the ORM connects the pool. Keeps what a reopen
* runs. A reopen replaces the database's contents, which the pool's own
* connections keep reaching, so the ORM is not pointed at anything new and
* the pool is never closed (ending a pool waits for every connection it has
* lent out, and one lent to a job during the restore would never come back).
*/
open(_orm: AnyOrm, onReopened: () => Promise<void>): void {
this.onReopened = onReopened;
}
close(): void {
// Nothing of its own to close: the pool goes with the ORM.
}
/**
* After `pg_restore` replaced the database's contents: the restored
* database is migrated forward on the same pool (`onReopened`, which
* `DatabaseLifecycle` hands to `open`).
*/
async reopen(): Promise<void> {
if (!this.onReopened) throw new Error('The Postgres engine was reopened before it was opened.');
await this.onReopened();
}
/**
* `pg_advisory_xact_lock` on one key for the whole database: held until the
* transaction commits or rolls back, taken again by a savepoint in the same
* session without waiting, and shared by every process on the database.
*/
async serializeWrites(em: EntityManager): Promise<void> {
await em
.getConnection()
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
}
/** `snapshot: false` (a restore, whose archive is the copy) migrates without a pg_dump first. */
async schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
const deps = this.bootstrapDeps(this.settings.url);
await bootstrapPostgresSchema(orm, options.snapshot ? deps : { ...deps, safetyNet: NO_SAFETY_NET });
await finishSchemaBootstrap(orm);
}
describe(): string {
return `Postgres ${redactUrl(this.settings.url)}`;
}
ormOptions() {
const ssl = sslOptionFor(this.settings.url);
return {
driver: TrekPostgreSqlDriver,
clientUrl: this.settings.url,
pool: { min: 0, max: this.settings.poolMax },
driverOptions: ssl === undefined ? {} : { ssl },
};
}
}
@@ -0,0 +1,32 @@
/**
* The SQLite schema's triggers as Postgres writes them. The entities describe
* tables only, so the baseline (`scripts/generate-pg-baseline.ts`) adds these
* after the tables, and `check:pg-baseline` fails while a trigger in the
* migrated SQLite schema has no entry here. Each entry is the trigger's
* function and the trigger that runs it, under the SQLite trigger's name.
*/
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
trg_release_stop_on_stay_delete: [
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
RETURN OLD;
END
$$`,
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
],
// A place that moved or was renamed drops the region it was resolved to (#2527).
trg_place_regions_follow_place: [
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM place_regions WHERE place_id = NEW.id;
RETURN NEW;
END
$$`,
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
'EXECUTE FUNCTION trg_place_regions_follow_place()',
],
};
@@ -0,0 +1,50 @@
import { run } from './pg-process';
/**
* The client-tool versions a restore accepts (CVE-2025-8714). A script from a
* dump an admin uploaded is run by psql, which would also run a psql
* meta-command (`\!` runs a shell command) smuggled into an object name. A
* fixed pg_restore opens the script with `\restrict <random key>`, and a fixed
* psql then refuses every meta-command until the matching `\unrestrict`.
*
* RESTRICT_SINCE holds, per major, the first minor that carries the fix. Every
* major after the last one listed has it from its first release.
*/
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
const LAST_MAJOR_WITHOUT_IT = Math.max(...Object.keys(RESTRICT_SINCE).map(Number));
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
export function hasRestrictFix(version: string): boolean {
const match = /\(PostgreSQL\) (\d+)\.(\d+)/.exec(version);
if (!match) return false;
const major = Number(match[1]);
const minor = Number(match[2]);
if (major > LAST_MAJOR_WITHOUT_IT) return true;
const firstFixedMinor = RESTRICT_SINCE[major];
return firstFixedMinor !== undefined && minor >= firstFixedMinor;
}
/** The minimum versions as the error names them, newest major first: "17.6, 16.10, 15.14 or 14.19". */
function minimumVersionsText(): string {
const floors = Object.keys(RESTRICT_SINCE)
.map(Number)
.sort((a, b) => b - a)
.map((major) => `${major}.${RESTRICT_SINCE[major]}`);
const last = floors.pop() ?? '';
return floors.length > 0 ? `${floors.join(', ')} or ${last}` : last;
}
/**
* Refuses the restore when pg_restore or psql predates the fix. The tools'
* own versions are checked, not the script, whose text the dump controls.
*/
export async function refuseToolsWithoutRestrict(binDir: string | undefined): Promise<void> {
const tools = ['pg_restore', 'psql'] as const;
const versions = await Promise.all(tools.map((tool) => run(tool, binDir, ['--version'], {})));
const stale = tools.findIndex((_, i) => !hasRestrictFix(versions[i]));
if (stale < 0) return;
throw new Error(
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least ${minimumVersionsText()} ` +
'(the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
);
}
@@ -0,0 +1,109 @@
/**
* How the entities' schema is written in Postgres: the one definition the
* Postgres baseline (`scripts/generate-pg-baseline.ts`), the schema parity
* check and the probe (`scripts/pg-probe.ts`) share.
*
* The entities are generated from the migrated SQLite schema and a parity
* test pins them to it (`npm run check:entities`), so they are the one
* description of the tables both engines can read. MikroORM's schema
* generator turns them into Postgres DDL after the adjustments below, which
* keep the storage model the services and the dialect helpers are written
* for (server/CLAUDE.md, "Storage model"):
*
* - timestamps stay TEXT;
* - an INTEGER column is `bigint`: SQLite's INTEGER holds 64 bits, so a
* column written with epoch milliseconds or a byte count past 2 GB would
* overflow Postgres's 32-bit `integer` (the engine's driver reads
* `bigint` back as a number, `postgres-driver.ts`);
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`,
* `CURRENT_TIMESTAMP` in a TEXT column) gets its Postgres equivalent, and
* a column declared `COLLATE NOCASE` uses the `nocase` collation
* `collateNoCase()` names on Postgres.
*
* The baseline keeps the foreign keys; the probe leaves them out, so a
* statement it sends is never refused only because the sample row it writes
* points at nothing. The triggers the SQLite schema has are in
* `postgres-triggers.ts`.
*/
import { pgTimestampText, PG_UTC_NOW } from '../../dialect/platform';
import { DbTimestampType } from '../../types';
/** SQLite's timestamp text on Postgres, for a column default. */
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
export const NOCASE_COLLATION_SQL =
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
/** The Postgres spelling of a column default the entities carry in SQLite's. */
export function portableDefault(raw: string): string {
const text = raw
.trim()
.replace(/^\((.*)\)$/s, '$1')
.trim();
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
return raw;
}
/** A column type SQLite stores as its 64-bit INTEGER. */
const INTEGER_TYPE = /^(integer|int)$/i;
/**
* The subset of MikroORM's property metadata the adjustments read and write.
* An entity's `props` satisfy it as they are, so no cast is needed.
*/
export interface ColumnMeta {
name: string;
defaultRaw?: string;
collation?: string;
columnTypes?: string[];
customType?: unknown;
}
/** A timestamp column becomes text. Returns whether it changed. */
function timestampAsText(column: ColumnMeta): boolean {
if (!(column.customType instanceof DbTimestampType) || !column.columnTypes) return false;
column.columnTypes = column.columnTypes.map(() => 'text');
return true;
}
/** An integer column becomes bigint. Returns whether it changed. */
function integerAsBigint(column: ColumnMeta): boolean {
if (!column.columnTypes?.some((type) => INTEGER_TYPE.test(type))) return false;
column.columnTypes = column.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
return true;
}
/** A SQLite-only default gets its Postgres spelling. Returns whether it changed. */
function defaultAsPostgres(column: ColumnMeta): boolean {
if (typeof column.defaultRaw !== 'string') return false;
const next = portableDefault(column.defaultRaw);
if (next === column.defaultRaw) return false;
column.defaultRaw = next;
return true;
}
/** A NOCASE column uses the `nocase` collation. Returns whether it changed. */
function nocaseAsCollation(column: ColumnMeta): boolean {
if (column.collation?.toUpperCase() !== 'NOCASE') return false;
column.collation = 'nocase';
return true;
}
/**
* Applies the adjustments above to an entity's properties, in place, and
* returns the names of the properties it changed, for the log. A timestamp
* column is text, so the integer adjustment never sees it.
*/
export function applyPostgresColumnAdjustments(properties: readonly ColumnMeta[]): string[] {
const changed: string[] = [];
for (const property of properties) {
const typeChanged = timestampAsText(property) || integerAsBigint(property);
const defaultChanged = defaultAsPostgres(property);
const collationChanged = nocaseAsCollation(property);
if (typeChanged || defaultChanged || collationChanged) changed.push(property.name);
}
return changed;
}
@@ -0,0 +1,185 @@
import { readEnv, type AppEnv } from '../../../app-config';
import { resolveDataPaths } from '../../../app-config/data-paths';
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
import { knownMigrationNames } from '../../known-migrations';
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
import type { AnyOrm } from '../../orm';
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
import { pgDump } from './pg-tools';
import type { Connection } from '@mikro-orm/core';
import type { Migrator } from '@mikro-orm/migrations';
import { Logger } from '@nestjs/common';
import fs from 'node:fs';
import path from 'node:path';
const logger = new Logger('DB');
/** A Postgres copy is a pg_dump custom-format archive. */
const DUMP_EXTENSION = '.pgdump';
/** What a Postgres bootstrap reads, passed in so a test can supply its own. */
export interface PostgresBootstrapDeps {
baseline: () => PostgresBaseline;
known: Set<string>;
safetyNet: MigrationSafetyNet;
}
/**
* The tables in the current schema. A database with no TREK migration history
* must hold none: TREK creates its schema in an empty database and never
* adopts tables something else made. Read before anything is written, so a
* refused database is left exactly as it was.
*/
async function currentTables(connection: Connection): Promise<string[]> {
const rows: Array<{ table_name: string }> = await connection.execute(CURRENT_SCHEMA_TABLES_SQL);
return rows.map((row) => row.table_name);
}
/**
* Creates the schema in an empty Postgres database: the baseline's statements
* and the record of every migration it stands for, in one transaction (DDL is
* transactional on Postgres, so the database ends with all of it or none). A
* database holding tables beside no TREK history is refused before anything is
* written.
*/
async function createFromBaseline(
connection: Connection,
storage: ReturnType<Migrator['getStorage']>,
tables: readonly string[],
historyTable: string,
baseline: () => PostgresBaseline,
): Promise<void> {
const foreign = tables.filter((table) => table !== historyTable);
if (foreign.length > 0) {
throw new Error(
`${REFUSAL} the database holds ${foreign.length} table(s) (${foreign.slice(0, 5).join(', ')}) but no TREK ` +
'migration history. Point TREK_DB_URL at an empty database.',
);
}
await storage.ensureTable();
const schema = baseline();
logger.log(`Creating the schema in an empty Postgres database (through ${migrationLabel(schema.through)})`);
await connection.transactional(async (trx) => {
await connection.execute(schema.statements.join(';\n'), [], 'run', trx);
await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx)));
});
}
/**
* Refuses a database whose recorded history names a migration this build does
* not know: a newer TREK migrated it. The refusal names the copy taken before
* that migration, when there is one.
*/
async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDeps): Promise<void> {
const thisRelease = migrationLabel([...deps.known].sort().at(-1) ?? '');
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
}
/**
* Brings a Postgres database to the current schema: inspect what it holds,
* then create the schema or verify the history it has, then migrate.
*
* - No migration recorded and no table: the baseline is created
* (`createFromBaseline`).
* - No migration recorded but tables present: refused, naming a few of them.
* - Migrations recorded: refused when one is unknown to this build, else the
* pending migrations run, after the safety net. A database the baseline just
* created runs the migrations after the baseline's cut without it: there is
* nothing yet to lose.
*/
export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstrapDeps): Promise<void> {
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
const connection = orm.em.getConnection();
const historyTable = orm.config.get('migrations').tableName ?? 'mikro_orm_migrations';
const tables = await currentTables(connection);
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
if (executed.length === 0) {
await createFromBaseline(connection, migrator.getStorage(), tables, historyTable, deps.baseline);
} else {
await refuseNewerHistory(executed, deps);
}
const pending = await migrator.getPending();
if (pending.length === 0) return;
// Only a database with history has a last migration to label its copy by; a
// database the baseline just created has nothing yet to lose.
const latest = [...executed].sort().at(-1);
if (latest !== undefined) await deps.safetyNet.beforeMigrate(migrationLabel(latest));
logger.log(`Applying ${pending.length} pending migration(s)`);
await migrator.up();
}
/** Dumps a killed boot left half-written: never a copy, only disk taken. */
function removeOrphanedPartials(dir: string): void {
for (const file of fs.readdirSync(dir)) {
if (file.startsWith(PREFIX) && file.endsWith(`${DUMP_EXTENSION}.partial`)) {
fs.rmSync(path.join(dir, file), { force: true });
}
}
}
/**
* The Postgres safety net before pending migrations, as the SQLite file's
* (`db/pre-migrate-snapshot.ts`): a `pg_dump` of the database to
* `<data>/pre-migrate-<schema>-<time>.pgdump`, the newest
* `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` kept. A copy that cannot be written
* (pg_dump missing, the data dir full) refuses the boot before anything is
* migrated, unless `TREK_DB_PRE_MIGRATE_SNAPSHOT=false`. Off under
* NODE_ENV=test unless a suite asks for it.
*/
export function postgresSafetyNet(
url: string,
env: () => AppEnv = readEnv,
now: () => Date = () => new Date(),
dataDir: () => string = () => resolveDataPaths(env()).dataDir,
): MigrationSafetyNet {
return {
async beforeMigrate(from) {
const { db, app } = env();
if (db.preMigrateSnapshot === undefined && app.isTest) return;
if (db.preMigrateSnapshot === false) {
logger.warn('TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating the Postgres database without a copy');
return;
}
const target = path.join(dataDir(), `${PREFIX}${from ?? 'unversioned'}-${stamp(now())}${DUMP_EXTENSION}`);
try {
fs.mkdirSync(dataDir(), { recursive: true });
removeOrphanedPartials(dataDir());
await pgDump(url, target, db.pgBinDir);
} catch (err) {
throw new Error(
`${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
`(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Fix that, ` +
'or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
{ cause: err },
);
}
logger.log(`Copy of the database before migrating: ${target}`);
pruneSnapshots(dataDir(), db.preMigrateSnapshotKeep, DUMP_EXTENSION);
},
async restoreHint(thisRelease) {
const snapshots = listSnapshots(dataDir(), DUMP_EXTENSION);
const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
if (match) {
return (
`The copy taken right before the newer TREK migrated it is ${match}: stop TREK, empty the database ` +
'(drop and recreate it), and restore it with pg_restore --no-owner --no-privileges --single-transaction.'
);
}
if (snapshots.length > 0) {
return `Copies taken before earlier migrations are in ${dataDir()} (${PREFIX}*${DUMP_EXTENSION}), none of them from this release.`;
}
return null;
},
};
}
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
}
@@ -0,0 +1,59 @@
/**
* Splits generated DDL into single statements, so one statement Postgres
* refuses does not take the rest of the schema with it.
*
* Understood: `;` outside quotes, `'` and `"` quoted text, and the `$$` dollar
* quote a function body is written in, which is the only dollar quote TREK's
* generated DDL and baseline contain. A tagged dollar quote (`$tag$`) is not
* understood, so it is refused rather than split inside.
*/
/** A tagged dollar quote (`$tag$`), matched where the scan stands. */
const TAGGED_DOLLAR_QUOTE = /\$[A-Za-z_]\w*\$/y;
function refuseTaggedDollarQuote(sql: string, at: number): void {
TAGGED_DOLLAR_QUOTE.lastIndex = at;
const tag = TAGGED_DOLLAR_QUOTE.exec(sql)?.[0];
if (tag) throw new Error(`splitStatements understands only the $$ dollar quote, not ${tag} (at ${at})`);
}
/**
* The statements in `sql`, trimmed and in order. A statement that sets a
* session parameter (`SET ...`) is dropped: a dump's preamble, which a schema
* the entities generate never needs.
*/
export function splitStatements(sql: string): string[] {
const statements: string[] = [];
let current = '';
let quote: '"' | "'" | '$$' | null = null;
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const dollar = sql.startsWith('$$', i);
if (quote === '$$') {
if (dollar) {
quote = null;
current += '$';
i++;
}
current += char;
} else if (quote) {
if (char === quote) quote = null;
current += char;
} else if (dollar) {
quote = '$$';
current += '$$';
i++;
} else if (char === "'" || char === '"') {
quote = char;
current += char;
} else if (char === ';') {
if (current.trim()) statements.push(current.trim());
current = '';
} else {
if (char === '$') refuseTaggedDollarQuote(sql, i);
current += char;
}
}
if (current.trim()) statements.push(current.trim());
return statements.filter((statement) => !/^set\s/i.test(statement));
}
+42
View File
@@ -0,0 +1,42 @@
import { readFileSync } from 'node:fs';
/** The sslmode values that verify the server, read the way the `pg` driver reads them. */
const VERIFYING_MODES = new Set(['require', 'verify-ca', 'verify-full']);
/** The `ssl` option of a `pg` pool, as `sslOptionFor` builds it. */
type PgSslOption = false | { rejectUnauthorized: true; ca?: string };
/**
* The `ssl` option for the pool, from the URL's `sslmode` and `sslrootcert`.
*
* MikroORM builds the pool from the URL's host, port, user, password and
* database alone and drops the rest of the query, so an `sslmode` in
* `TREK_DB_URL` would be ignored without this. `require`, `verify-ca` and
* `verify-full` all verify the server's certificate, as the `pg` driver itself
* treats them; `disable` turns SSL off, and no `sslmode` leaves the driver's
* default (no SSL). A mode that would connect without verifying (`prefer`,
* `allow`, `no-verify`) or that is misspelt refuses, rather than quietly
* connecting with less than was asked. `sslrootcert` names the CA file a host
* that signs its own certificates needs (Supabase's, say).
*/
export function sslOptionFor(url: string): PgSslOption | undefined {
const params = new URL(url).searchParams;
const mode = params.get('sslmode');
const rootCert = params.get('sslrootcert');
if (mode === null) {
if (rootCert !== null) throw new Error('TREK_DB_URL names an sslrootcert without sslmode=verify-full.');
return undefined;
}
if (mode === 'disable') return false;
if (!VERIFYING_MODES.has(mode)) {
throw new Error(
`TREK_DB_URL has sslmode "${mode}"; use disable, require, verify-ca or verify-full (all but disable verify the server).`,
);
}
if (rootCert === null) return { rejectUnauthorized: true };
try {
return { rejectUnauthorized: true, ca: readFileSync(rootCert, 'utf8') };
} catch (error) {
throw new Error(`TREK_DB_URL names sslrootcert ${rootCert}, which cannot be read.`, { cause: error });
}
}
+17
View File
@@ -0,0 +1,17 @@
/**
* A connection URL fit for a log line: the password masked and the query
* dropped, since libpq reads a password from the query too
* (`?password=...`). A value that does not parse is not echoed at all.
*/
export function redactUrl(url: string): string {
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return '<unparseable URL>';
}
if (parsed.password) parsed.password = '***';
parsed.search = '';
parsed.hash = '';
return parsed.toString();
}
@@ -0,0 +1,61 @@
import { closeDb, getRawConnection, registerReinitializeHook, reinitialize } from '../../database';
import { resolveDbPath } from '../../db-path';
import { runSchemaBootstrap, type AnyOrm } from '../../orm';
import { SharedSqliteDriver } from '../../orm-driver';
import type { DatabaseEngine } from '../engine';
import type { EntityManager } from '@mikro-orm/core';
/**
* The SQLite file engine, TREK's default: one better-sqlite3 handle that
* `db/database.ts` owns and the ORM's bound driver shares (`db/orm-driver.ts`),
* on `TREK_DB_FILE` or `data/travel.db` (`:memory:` under test). A restore
* swaps the file and reopens the handle; the schema comes from the migration
* chain (`db/orm.ts`).
*/
export class SqliteEngine implements DatabaseEngine {
readonly id = 'sqlite';
describe(): string {
return `SQLite file ${resolveDbPath()}`;
}
ormOptions() {
return { driver: SharedSqliteDriver, dbName: resolveDbPath() };
}
/**
* Opens the file (normally a no-op: the ORM's first connect inside
* NestFactory.create() already did, said here so the boot does not depend on
* that), and registers the reopen. A reopen swaps the file under the same
* handle, which Kysely caches, so the ORM's connection is closed and connected
* again first: the driver then reaches the new handle through
* `createKyselyDialect()`. The bound driver's `destroy()` is a no-op
* (`orm-driver.ts`), so the close never reaches the handle `db/database.ts` owns.
*/
open(orm: AnyOrm, onReopened: () => Promise<void>): void {
getRawConnection();
registerReinitializeHook(async () => {
const connection = orm.em.getConnection();
await connection.close(true);
await connection.connect();
await onReopened();
});
}
close(): void {
closeDb();
}
reopen(): Promise<void> {
return reinitialize();
}
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
return runSchemaBootstrap(orm, options);
}
/** A no-op: SQLite has one connection and one writer, so its transactions already run one at a time. */
serializeWrites(_em: EntityManager): Promise<void> {
return Promise.resolve();
}
}
@@ -0,0 +1,49 @@
import type { EntityManager } from '@mikro-orm/core';
/**
* The SQLite engine's whole-file maintenance: the two statements a backup,
* a restore and the demo reset need that are about the database file rather
* than any table (`PRAGMA wal_checkpoint`, `VACUUM INTO`). Neither has a
* Postgres spelling, so they live under the engine that owns the file instead
* of in `MaintenanceRepository`, whose remaining statements run on every
* engine (and which the Postgres probe calls method by method).
*
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
* factory over the context-resolving global `EntityManager`, and injected by
* the SQLite backup port (`SqliteDatabaseBackup`). Free of Nest decorators
* like every other file under `db/`, so a test can build one over its own
* `EntityManager`.
*/
export class SqliteMaintenance {
constructor(private readonly em: EntityManager) {}
/**
* MikroORM's own validating `getContext()`: throws outside a request
* context, the fail-closed check `TrekRepository` applies to every
* repository, since a raw `connection.execute()` has no such guard.
*/
private validateRequestContext(): void {
this.em.getContext();
}
/**
* `PRAGMA wal_checkpoint(TRUNCATE)`: flushes the WAL into the main file
* before a snapshot or a swap. Reached through the backup port's
* `checkpoint()`; every caller treats it as best effort.
*/
async walCheckpoint(): Promise<void> {
this.validateRequestContext();
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
}
/**
* `` VACUUM INTO '<path>' ``: a consistent point-in-time copy of the file
* under concurrent writers, reached through the backup port's
* `snapshot()`. Single quotes in `path` are escaped by doubling.
*/
async vacuumInto(path: string): Promise<void> {
this.validateRequestContext();
const escaped = path.replaceAll("'", "''");
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
}
}
+21 -11
View File
@@ -1,3 +1,5 @@
import { columnNamesSql, tableExistsSql } from './dialect/catalog';
import { TableNotFoundException } from '@mikro-orm/core';
import type { Migration } from '@mikro-orm/migrations';
/**
@@ -8,7 +10,7 @@ import type { Migration } from '@mikro-orm/migrations';
* a migration class.
*/
/** A single `pragma_table_info` row — only the column name is ever read. */
/** A catalog row (`dialect/catalog.ts`): only the name is ever read. */
interface ColumnRow {
name: string;
}
@@ -30,21 +32,18 @@ export async function addColumnIfMissing(
column: string,
definition: string,
): Promise<void> {
const existing = (await migration.execute('select name from pragma_table_info(?) where name = ?', [
table,
column,
])) as ColumnRow[];
if (existing.length > 0) return;
await migration.execute(`alter table \`${table}\` add column ${definition}`);
if ((await columnNames(migration, table)).has(column)) return;
const platform = migration.getEntityManager().getPlatform();
await migration.execute(`alter table ${platform.quoteIdentifier(table)} add column ${definition}`);
}
/** True when `table` exists in the current database. */
export async function tableExists(migration: Migration, table: string): Promise<boolean> {
const rows = await migration.execute(`select name from sqlite_master where type = 'table' and name = ?`, [table]);
const rows = await migration.execute(tableExistsSql(migration.getEntityManager().getPlatform()), [table]);
return rows.length > 0;
}
/** The `CREATE TABLE` DDL SQLite stored for `table`, or null when it does not exist. */
/** The `CREATE TABLE` DDL SQLite stored for `table`, or null when it does not exist. SQLite only: Postgres keeps no such text. */
export async function tableSql(migration: Migration, table: string): Promise<string | null> {
const rows = (await migration.execute(`select sql from sqlite_master where name = ?`, [table])) as {
sql: string | null;
@@ -68,7 +67,8 @@ export async function execBestEffort(migration: Migration, sql: string, params?:
/** The column names of `table`, or an empty set when the table does not exist. */
export async function columnNames(migration: Migration, table: string): Promise<Set<string>> {
const rows = (await migration.execute('select name from pragma_table_info(?)', [table])) as ColumnRow[];
const sql = columnNamesSql(migration.getEntityManager().getPlatform());
const rows = (await migration.execute(sql, [table])) as ColumnRow[];
return new Set(rows.map((r) => r.name));
}
@@ -80,10 +80,20 @@ export async function execUnlessTableMissing(migration: Migration, sql: string,
try {
await migration.execute(sql, params);
} catch (err: unknown) {
if (!(err instanceof Error) || !err.message.includes('no such table')) throw err;
if (!isMissingTableError(err)) throw err;
}
}
/**
* True when a statement failed because its table does not exist. The ORM maps
* SQLite's "no such table" and Postgres's SQLSTATE 42P01 to
* `TableNotFoundException`; the raw SQLite message is checked too, for an error
* that reached the caller unconverted.
*/
export function isMissingTableError(err: unknown): boolean {
return err instanceof TableNotFoundException || (err instanceof Error && err.message.includes('no such table'));
}
/**
* Runs a group of statements the legacy step wrapped in a single swallowing
* try/catch. A failure part-way through leaves the earlier statements applied,
+15 -35
View File
@@ -1,4 +1,3 @@
import mikroOrmConfig from '../mikro-orm.config';
import { withRequestContext } from '../nest/database/request-context';
import { runDemoSeed } from './database';
import { ensureEmailCaseIndex } from './email-case-index';
@@ -6,16 +5,11 @@ import { migrateToHead, NO_SAFETY_NET } from './legacy-baseline';
import { preMigrateSnapshot } from './pre-migrate-snapshot';
import type { AnyEntity, EntityClass, EntityManager, EntitySchema, IDatabaseDriver, MikroORM } from '@mikro-orm/core';
import type { Migrator } from '@mikro-orm/migrations';
import { MikroORM as SqliteMikroORM } from '@mikro-orm/sqlite';
/**
* Schema ownership at startup.
*
* This replaces `createTables()` → `runMigrations()` → `runSeeds()`, which used
* to run synchronously inside `database.ts::initDb()` at module load. The
* migrator is async, so the work moved to `buildApp()`, which is the first place
* in the boot with an await and still runs before anything reads the database
* (`bootstrap.ts` resolves SettingsService right after).
* Schema ownership at startup. The migrator is async, so the schema is brought
* to head from `DatabaseLifecycle.open()`, which `buildApp()` calls before
* anything reads the database. Nothing runs at module load.
*/
/**
@@ -35,10 +29,10 @@ export type AnyOrm = MikroORM<
* Migrate to head, then seed. Safe to call again, both halves are idempotent.
*
* `snapshot: false` skips the pre-migration copy (`pre-migrate-snapshot.ts`).
* Only the restore hook passes it: the file it migrates was just unpacked from
* a backup archive, which is still there to go back to.
* Only the reopen after a restore passes it: the file it migrates was just
* unpacked from a backup archive, which is still there to go back to.
*/
export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { snapshot?: boolean } = {}): Promise<void> {
export async function runSchemaBootstrap(orm: AnyOrm, { snapshot }: { snapshot: boolean }): Promise<void> {
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
const connection = orm.em.getConnection();
// A database the retired positional runner migrated has no migrator rows yet;
@@ -52,8 +46,16 @@ export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { sna
undefined,
snapshot ? preMigrateSnapshot(connection) : NO_SAFETY_NET,
);
await finishSchemaBootstrap(orm);
}
/**
* What every engine does once its schema is at head: the deferred index, the
* seeders, the demo seed. Idempotent, like the migration step before it.
*/
export async function finishSchemaBootstrap(orm: AnyOrm): Promise<void> {
// The one index a migration may have had to defer (see email-case-index.ts).
await ensureEmailCaseIndex(connection);
await ensureEmailCaseIndex(orm.em.getConnection());
const defaultSeeder = orm.config.get('seeder').defaultSeeder;
if (defaultSeeder) await orm.seeder.seedString(defaultSeeder);
@@ -77,25 +79,3 @@ export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { sna
// fully-synchronous version had by construction.
await withRequestContext(orm, () => runDemoSeed());
}
/**
* An ORM instance outside Nest, for the vitest global setup that builds the
* schema snapshot. Production uses the container-owned one from
* `MikroOrmModule.forRoot()` instead of creating a second.
*/
export function createStandaloneOrm(): Promise<AnyOrm> {
return SqliteMikroORM.init({
...mikroOrmConfig,
// Entity discovery is skipped deliberately. Migrations and seeders are raw
// SQL and need no metadata, and the global setup that calls this runs as ESM
// in vitest's main process, where discovering `*.entity.ts` fails on Node's
// extensionless import resolution.
entities: [],
entitiesTs: [],
discovery: { warnWhenNoEntities: false },
// The migrator's per-migration progress log is production boot output; the
// global setup already reports the snapshot in one line of its own, and a
// failing migration still throws.
migrations: { ...mikroOrmConfig.migrations, silent: true },
});
}
+7 -8
View File
@@ -25,11 +25,10 @@ import path from 'node:path';
* migrated; `TREK_DB_PRE_MIGRATE_SNAPSHOT=false` is the way past that.
*/
const PREFIX = 'pre-migrate-';
const SNAPSHOT_FILE = /^pre-migrate-.+\.db$/;
export const PREFIX = 'pre-migrate-';
/** `20261008T091500Z`: sortable, and free of the colons Windows refuses in a file name. */
function stamp(now: Date): string {
export function stamp(now: Date): string {
return now
.toISOString()
.replace(/\.\d+Z$/, 'Z')
@@ -51,11 +50,11 @@ async function holdsAnything(connection: Connection): Promise<boolean> {
return rows.length > 0;
}
/** The snapshots in `dir`, newest first. */
export function listSnapshots(dir: string): string[] {
/** The snapshots in `dir`, newest first: the SQLite file's (`.db`) or a Postgres engine's (`.pgdump`). */
export function listSnapshots(dir: string, extension = '.db'): string[] {
let entries: string[];
try {
entries = fs.readdirSync(dir).filter((entry) => SNAPSHOT_FILE.test(entry));
entries = fs.readdirSync(dir).filter((entry) => entry.startsWith(PREFIX) && entry.endsWith(extension));
} catch {
return [];
}
@@ -66,8 +65,8 @@ export function listSnapshots(dir: string): string[] {
}
/** Deletes all but the newest `keep`. A failure only warns: the new snapshot is already safe. */
export function pruneSnapshots(dir: string, keep: number): void {
for (const file of listSnapshots(dir).slice(keep)) {
export function pruneSnapshots(dir: string, keep: number, extension = '.db'): void {
for (const file of listSnapshots(dir, extension).slice(keep)) {
try {
fs.rmSync(file, { force: true });
} catch (err) {
@@ -1,6 +1,7 @@
import { coalesceOverride, columnRef } from '../dialect/sql-functions';
import type { DayAccommodations } from '../entities/DayAccommodations.entity';
import type { DB } from '../kysely/db';
import { accommodationIdMatches } from './_shared/reservation-accommodation';
import { type AssertRowKeys } from './_shared/rows';
import { TrekRepository } from './_shared/trek-repository';
@@ -249,10 +250,13 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
* parity keeps the duplicates, no `DISTINCT`.
*/
async listForTripWithPlaceAndBooking(trip_id: number | string): Promise<DayAccommodationListRow[]> {
const platform = this.getEntityManager().getPlatform();
const rows = await this.kysely<DayAccommodationListKyselyDB>()
.selectFrom('day_accommodations as a')
.leftJoin('places as p', 'p.id', 'a.place_id')
.leftJoin('reservations as r', 'r.accommodation_id', 'a.id')
.leftJoin('reservations as r', (join) =>
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'a.id')),
)
.selectAll('a')
.select([
'p.name as place_name',
@@ -469,6 +473,7 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
* this table (see {@link DayAccommodationListKyselyDB}).
*/
async listRoadtripStays(trip_id: number): Promise<RoadtripStayRow[]> {
const platform = this.getEntityManager().getPlatform();
const rows = await this.kysely<RoadtripStaysKyselyDB>()
.selectFrom('day_accommodations as a')
.leftJoin('places as p', 'p.id', 'a.place_id')
@@ -485,7 +490,7 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
eb
.selectFrom('reservations as r')
.select((eb2) => eb2.fn.min<number | null>('r.id').as('m'))
.whereRef('r.accommodation_id', '=', 'a.id')
.where((eb2) => accommodationIdMatches(platform, eb2, 'r.accommodation_id', 'a.id'))
.as('reservation_id'),
])
.where('a.trip_id', '=', trip_id)
@@ -43,7 +43,9 @@ export class GoogleApiUsageRepository extends TrekRepository<GoogleApiUsage> {
await this.kysely<GoogleApiUsageKyselyDB>()
.insertInto('google_api_usage')
.values({ day, calls: 1 })
.onConflict((oc) => oc.column('day').doUpdateSet({ calls: (eb) => eb('calls', '+', 1) }))
// The target row's column is qualified: Postgres reads a bare name as
// ambiguous between it and `excluded` (42702).
.onConflict((oc) => oc.column('day').doUpdateSet({ calls: (eb) => eb(eb.ref('google_api_usage.calls'), '+', 1) }))
.execute();
}
@@ -1,27 +1,18 @@
import { isMissingTableError } from '../migration-utils';
import type { EntityManager } from '@mikro-orm/core';
/**
* Plan 3i Task 3 — R1's raw-connection exception, named ahead of this plan by
* the program brief's own rule 4 ("connection.execute() (only in
* MaintenanceRepository/DemoRepository, with a comment)").
*
* Not a `TrekRepository<Entity>` — it backs no table and no entity of its
* own. It exists purely as the one place two whole-database-file statements
* (a `PRAGMA` and a `VACUUM INTO`) run against the ORM's own bound driver
* connection instead of scattering raw `better-sqlite3` calls across
* `backup.impl.ts`/`demo-reset.ts`. `find/findOne → create/assign/flush →
* nativeUpdate/nativeDelete → em.upsert → qb() → em.getKysely()` all fail
* for these two statements — neither is expressible against an entity at
* all — so `connection.execute()` (the last tier in the program's query-API
* order) is the correct tool, not a shortcut around it.
* Raw-connection statements that run on every engine and that no entity can
* express: the readiness ping and the plugin-table erasure. Not a
* `TrekRepository<Entity>`, since it backs no entity. The SQLite file's own
* maintenance (`PRAGMA wal_checkpoint`, `VACUUM INTO`) lives in
* `SqliteMaintenance`, under `db/engine/sqlite/`.
*
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
* factory over the context-resolving global `EntityManager`; its consumers
* inject it: the readiness probe, `UserCleanupService` and the SQLite backup
* port (`SqliteDatabaseBackup`), which the backup, the restore and the demo
* reset all go through. The class stays free of Nest decorators like every
* other file under `db/`, so a test can still build one over its own
* `EntityManager`.
* inject it: the readiness probe and `UserCleanupService`. The class stays
* free of Nest decorators like every other file under `db/`, so a test can
* still build one over its own `EntityManager`.
*/
export class MaintenanceRepository {
constructor(private readonly em: EntityManager) {}
@@ -30,7 +21,7 @@ export class MaintenanceRepository {
* MikroORM's own validating `getContext()` — throws
* `ValidationError.cannotUseGlobalContext()` outside a request context (the
* production default), the same fail-closed ratchet `TrekRepository`
* applies to every entity-bound repository. Called first by both methods
* applies to every entity-bound repository. Called first by every method
* below, since raw `connection.execute()` has no such guard of its own.
*/
private validateRequestContext(): void {
@@ -44,69 +35,27 @@ export class MaintenanceRepository {
}
/**
* `PRAGMA wal_checkpoint(TRUNCATE)` — rendered text pinned, identical to
* the legacy `db.exec('PRAGMA wal_checkpoint(TRUNCATE)')` it replaces.
* Reached through the backup port's `checkpoint()`, which the backup
* (BK1) and the demo reset (DMR3) call. Flushes the WAL into the main db
* file before a snapshot or a swap; every caller treats it as best effort.
*/
async walCheckpoint(): Promise<void> {
this.validateRequestContext();
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
}
/**
* `` VACUUM INTO '<path>' `` — rendered text pinned, identical to the
* legacy `` db.exec(`VACUUM INTO '${dbSnap.replaceAll("'", "''")}'`) ``
* (BK2, reached through the backup port's `snapshot()`): single quotes in `path` are
* escaped by doubling, the same escaping the legacy statement used — a
* consistent point-in-time snapshot under concurrent writers, with no
* MikroORM/Kysely equivalent.
*/
async vacuumInto(path: string): Promise<void> {
this.validateRequestContext();
const escaped = path.replaceAll("'", "''");
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
}
/**
* Plan 4 Task 4 — UC1 (`UserCleanupService.erasePluginUserData`): a
* best-effort per-user delete across the three plugin-held tables
* (`plugin_user_config`/`plugin_oauth_tokens`/`plugin_oauth_state`). These
* are `nest/plugins`' own tables (Plan 3b Task 5's "stays raw" ruling,
* narrowed by Plan 4 Task 8a to just this trio — UC2/UC3 already moved to
* `PluginsRepository`/`PluginUserErasureQueueRepository`), not an entity
* managed by any repository, so `connection.execute()` (rule 4's last
* tier) is the correct tool here, same shape as `walCheckpoint`/
* `vacuumInto` above — with ONE difference those two don't need: this is
* the first `MaintenanceRepository` caller ever invoked from INSIDE an
* open `UnitOfWork.transactional(...)` (`deleteUserCompletely`'s
* transaction). `Connection#execute(query, params, method, ctx)`'s `ctx`
* defaults to the plain (non-transactional) Kysely client when omitted —
* `walCheckpoint`/`vacuumInto` always omit it because their callers
* (the backup port) never run inside a transaction, but
* omitting it here deadlocked: this repo's `better-sqlite3` connection is
* single-connection/single-writer, the open transaction holds it for its
* whole lifetime, and a bare `execute()` tries to check out that SAME
* connection fresh instead of reusing the held one — a checkout that only
* frees once the transaction (which is itself awaiting THIS call)
* completes. `em.getTransactionContext()` returns the open transaction's
* driver handle when called from inside one (resolved the same
* `TransactionContext`-async-local-storage way `getContext()` is, so
* `this.em` need not be the transactional fork itself) and `undefined`
* otherwise, so passing it through is correct both inside and outside a
* transaction. Each statement is independently best-effort: a
* slimmed-down test schema may not carry one or more of these tables at
* all, matching the legacy per-table try/catch exactly.
* The per-user erasure of the three plugin-held tables (`plugin_user_config`,
* `plugin_oauth_tokens`, `plugin_oauth_state`), called inside the erasure's
* own `UnitOfWork.transactional(...)`.
*
* A table that does not exist (a slimmed-down test schema) is skipped; any
* other failure propagates. Each statement runs in its own savepoint, since
* on Postgres a failed statement aborts the whole transaction it is in.
* `em.getTransactionContext()` hands the statement the open transaction's
* connection, which a bare `execute()` would wait on forever.
*/
async deletePluginUserData(userId: number): Promise<void> {
this.validateRequestContext();
const ctx = this.em.getTransactionContext();
for (const table of ['plugin_user_config', 'plugin_oauth_tokens', 'plugin_oauth_state']) {
try {
await this.em.getConnection().execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', ctx);
} catch {
/* table absent (slim schema) */
await this.em.transactional((em) =>
em
.getConnection()
.execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', em.getTransactionContext()),
);
} catch (err: unknown) {
if (!isMissingTableError(err)) throw err;
}
}
}
@@ -1,6 +1,6 @@
import type { Places } from '../entities/Places.entity';
import { wrapOptionalLongitude } from '@trek/shared';
import { absDifference, coalesceParam, columnRef, currentTimestamp, lowerTrim } from '../dialect/sql-functions';
import { absDifference, coalesceParam, columnRef, currentTimestamp, ilike, lowerTrim } from '../dialect/sql-functions';
import { type AssertRowKeys } from './_shared/rows';
import { TrekRepository } from './_shared/trek-repository';
import { tourPlaceTextPatch, type TourPlaceWrite } from './tour-place-write';
@@ -659,8 +659,8 @@ export class PlacesRepository extends TrekRepository<Places> {
*/
async listForTrip(trip_id: string, filters: {
searchPattern?: string;
category?: string;
tag?: string;
category?: number;
tag?: number;
assignment?: 'all' | 'unassigned' | 'assigned';
}): Promise<PlaceListRow[]> {
const platform = this.getEntityManager().getPlatform();
@@ -680,15 +680,13 @@ export class PlacesRepository extends TrekRepository<Places> {
.where('p.trip_id = ?', [trip_id]);
if (filters.searchPattern) {
qb.andWhere(
"(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\' OR p.description LIKE ? ESCAPE '\\')",
[filters.searchPattern, filters.searchPattern, filters.searchPattern],
);
const fields = ['p.name', 'p.address', 'p.description'];
qb.andWhere(ilike(platform, fields), fields.map(() => filters.searchPattern));
}
if (filters.category) {
if (filters.category !== undefined) {
qb.andWhere('p.category_id = ?', [filters.category]);
}
if (filters.tag) {
if (filters.tag !== undefined) {
qb.andWhere('p.id IN (SELECT place_id FROM place_tags WHERE tag_id = ?)', [filters.tag]);
}
if (filters.assignment === 'unassigned') {
@@ -52,10 +52,14 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
* position) SELECT r.id, d.id, ? FROM reservations r JOIN days d ON
* d.trip_id = r.trip_id WHERE r.id = ? AND d.id = ? AND r.trip_id = ?`.
*
* Kysely `insertInto().orReplace().columns().expression(selectFrom…)`
* (the MikroORM QueryBuilder's `insertFrom` has no `OR REPLACE` flag —
* `.onConflict().merge()` is semantically different, a real UPDATE rather
* than REPLACE's delete-then-insert). The trip scoping lives IN the
* Kysely `insertInto().columns().expression(selectFrom…)` with an
* `ON CONFLICT (reservation_id, day_id) DO UPDATE SET position`, which both
* engines read (`INSERT OR REPLACE` is SQLite only). The pair is the
* table's whole primary key and `position` its only other column, so the
* update leaves the same row REPLACE's delete-then-insert did, and no
* foreign key points at this table for the delete to have cascaded to.
* The SELECT keeps its WHERE, which SQLite needs to read the trailing
* ON CONFLICT as the upsert clause. The trip scoping lives IN the
* SELECT (a foreign `reservation_id`/`day_id` materialises zero rows to
* insert, never a foreign-key error) — preserved exactly, not hoisted
* into a pre-check.
@@ -68,7 +72,6 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
): Promise<void> {
await this.kysely<ReservationDayPositionsKyselyDB>()
.insertInto('reservation_day_positions')
.orReplace()
.columns(['reservation_id', 'day_id', 'position'])
.expression((eb) =>
eb
@@ -79,6 +82,9 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
.where('d.id', '=', day_id)
.where('r.trip_id', '=', trip_id as number),
)
.onConflict((oc) =>
oc.columns(['reservation_id', 'day_id']).doUpdateSet((eb) => ({ position: eb.ref('excluded.position') })),
)
.execute();
}
}
@@ -2,7 +2,8 @@ import type { Reservations } from '../entities/Reservations.entity';
import { DayAssignments } from '../entities/DayAssignments.entity';
import { Days } from '../entities/Days.entity';
import { coalesceOverride, columnRef, dayDistance } from '../dialect/sql-functions';
import { castIntegerKysely, concatKysely, startsWithIsoDateKysely, substringKysely } from '../dialect/kysely-functions';
import { concatKysely, startsWithIsoDateKysely, substringKysely } from '../dialect/kysely-functions';
import { accommodationIdMatches } from './_shared/reservation-accommodation';
import { publicReservationExpr, publicStayExists, type ReservationVisibilityKyselyDB } from './_shared/reservation-visibility';
import type { DayAssignmentRow } from './day-assignment-rows';
import { TrekRepository } from './_shared/trek-repository';
@@ -32,18 +33,6 @@ export type {
PublicApiUnscheduledReservationRow,
};
/**
* Kysely typing for the RS18/RR1 join (`joinedQuery` below) — `reservations
* .accommodation_id` is a bare `p.text().nullable()` column with no FK to
* `day_accommodations` (§18.1 of the inventory), so the QueryBuilder's
* relation-path `.join()` cannot express `LEFT JOIN day_accommodations ap ON
* r.accommodation_id = ap.id` at all — the same escape hatch
* `DayAssignmentsRepository.effectiveStart`/`TripsRepository.tripSelectQuery`
* use. The `id`/`trip_id` the callers pass stay `number | string` (raw-bind,
* D4's T5 seam) and are cast to the column type only at the bind, so this
* join keeps the legacy statement's exact raw-bind flexibility.
*/
/**
* `reservations` as the two aliases below bind it: `accommodation_id` is a
* TEXT column, widened to `string | number | null` so the plain `number`
@@ -364,7 +353,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
findPassContext(id: number, tripId: number) { return passContextQuery(this.kysely<PassContextDB>(), id, tripId); }
private joinedQuery() {
return joinedReservationQuery(this.kysely<ReservationJoinDB>());
return joinedReservationQuery(this.kysely<ReservationJoinDB>(), this.getEntityManager().getPlatform());
}
/**
@@ -875,10 +864,13 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
/** CL2 — see {@link CalendarReservationRow}'s docstring. */
async listForCalendar(trip_id: number | string): Promise<CalendarReservationRow[]> {
const platform = this.getEntityManager().getPlatform();
const rows = await this.kysely<CalendarReservationKyselyDB>()
.selectFrom('reservations as r')
.leftJoin('places as pl', 'pl.id', 'r.place_id')
.leftJoin('day_accommodations as a', 'a.id', 'r.accommodation_id')
.leftJoin('day_accommodations as a', (join) =>
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'a.id')),
)
.leftJoin('days as sd', 'sd.id', 'a.start_day_id')
.leftJoin('days as ed', 'ed.id', 'a.end_day_id')
.leftJoin('days as rd', 'rd.id', 'r.day_id')
@@ -894,7 +886,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
eb
.selectFrom('reservations as r2')
.select((eb2) => eb2.fn.min<number | null>('r2.id').as('c'))
.whereRef('r2.accommodation_id', '=', 'a.id')
.where((eb2) => accommodationIdMatches(platform, eb2, 'r2.accommodation_id', 'a.id'))
.as('stay_first_reservation_id'),
'rd.date as day_date',
'red.date as end_day_date',
@@ -932,6 +924,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
/** CL7 — see {@link CalendarStayRow}'s docstring. */
async listPublicStaysForCalendar(trip_id: number | string): Promise<CalendarStayRow[]> {
const platform = this.getEntityManager().getPlatform();
const rows = await this.kysely<CalendarStayKyselyDB>()
.selectFrom('day_accommodations as a')
.where('a.trip_id', '=', trip_id as number)
@@ -953,7 +946,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
eb
.selectFrom('reservations as r')
.select('r.title')
.whereRef('r.accommodation_id', '=', 'a.id')
.where((eb2) => accommodationIdMatches(platform, eb2, 'r.accommodation_id', 'a.id'))
.where((eb2) => publicReservationExpr(eb2, 'r.ingest_state'))
.orderBy('r.id', 'asc')
.limit(1)
@@ -1039,7 +1032,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
eb
.selectFrom('reservations as res')
.select('res.title')
.where((eb2) => eb2(castIntegerKysely(platform, eb2, 'res.accommodation_id'), '=', eb2.ref('a.id')))
.where((eb2) => accommodationIdMatches(platform, eb2, 'res.accommodation_id', 'a.id'))
.where('res.status', '!=', 'cancelled')
.orderBy('res.id', 'asc')
.limit(1),
@@ -1080,7 +1073,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
eb
.selectFrom('reservations as res')
.select('res.title')
.where((eb2) => eb2(castIntegerKysely(platform, eb2, 'res.accommodation_id'), '=', eb2.ref('a.id')))
.where((eb2) => accommodationIdMatches(platform, eb2, 'res.accommodation_id', 'a.id'))
.where('res.status', '!=', 'cancelled')
.orderBy('res.id', 'asc')
.limit(1),
@@ -80,12 +80,14 @@ export class RouteUsageDailyRepository extends TrekRepository<RouteUsageDaily> {
await this.kysely<RouteUsageDailyKyselyDB>()
.insertInto('route_usage_daily')
.values(entry)
// The target row's columns are qualified: Postgres reads a bare name as
// ambiguous between it and `excluded` (42702).
.onConflict((oc) =>
oc.columns(['day', 'profile', 'surface', 'self_hosted']).doUpdateSet({
requests: (eb) => eb('requests', '+', eb.ref('excluded.requests')),
waypoints: (eb) => eb('waypoints', '+', eb.ref('excluded.waypoints')),
km: (eb) => eb('km', '+', eb.ref('excluded.km')),
failed: (eb) => eb('failed', '+', eb.ref('excluded.failed')),
requests: (eb) => eb(eb.ref('route_usage_daily.requests'), '+', eb.ref('excluded.requests')),
waypoints: (eb) => eb(eb.ref('route_usage_daily.waypoints'), '+', eb.ref('excluded.waypoints')),
km: (eb) => eb(eb.ref('route_usage_daily.km'), '+', eb.ref('excluded.km')),
failed: (eb) => eb(eb.ref('route_usage_daily.failed'), '+', eb.ref('excluded.failed')),
}),
)
.execute();
@@ -1,6 +1,6 @@
import { QueryFlag } from '@mikro-orm/core';
import type { TTripsCostsMode, Trips } from '../entities/Trips.entity';
import { coalesceParam, currentTimestamp, nowDateOffset } from '../dialect/sql-functions';
import { coalesceParam, currentTimestamp, ilike, nowDateOffset } from '../dialect/sql-functions';
import type { AssertRowKeys } from './_shared/rows';
import { TrekRepository } from './_shared/trek-repository';
import { tripAccessExpr } from './_shared/trip-access';
@@ -175,7 +175,7 @@ export class TripsRepository extends TrekRepository<Trips> {
return await this.accessibleTripsQuery(user_id)
.join('t.places_collection', 'p')
.select(['p.trip', 'p.name'])
.andWhere("(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\')", [likePattern, likePattern])
.andWhere(ilike(this.getEntityManager().getPlatform(), ['p.name', 'p.address']), [likePattern, likePattern])
.orderBy([{ 'p.trip': 'asc' }, { 'p.name': 'asc' }])
.limit(500)
// The limit is on the joined place rows, like the legacy statement's.
@@ -0,0 +1,22 @@
import { castIntegerKysely } from '../../dialect/kysely-functions';
import type { Platform } from '@mikro-orm/core';
import type { ExpressionBuilder, ExpressionWrapper, SqlBool, StringReference } from 'kysely';
/**
* The join condition between a reservation and the stay it is booked against.
*
* `reservations.accommodation_id` is TEXT with no foreign key, while
* `day_accommodations.id` is INTEGER, so the comparison is written as
* `CAST(<reservation ref> AS INTEGER) = <stay ref>`. SQLite coerces a bare
* text-to-integer comparison and Postgres refuses it (42883), so the cast is
* needed on both engines. A stored `'14.0'` casts to 14 on both.
*/
export function accommodationIdMatches<DB, TB extends keyof DB>(
platform: Platform,
eb: ExpressionBuilder<DB, TB>,
reservationAccommodationId: StringReference<DB, TB>,
stayId: StringReference<DB, TB>,
): ExpressionWrapper<DB, TB, SqlBool> {
return eb(castIntegerKysely(platform, eb, reservationAccommodationId), '=', eb.ref(stayId).$castTo<number>());
}
@@ -1,4 +1,6 @@
import type { DB } from '../../kysely/db';
import { accommodationIdMatches } from './reservation-accommodation';
import type { Platform } from '@mikro-orm/core';
import type { Kysely } from 'kysely';
@@ -9,14 +11,18 @@ export type ReservationJoinDB = Pick<DB, 'reservations' | 'days' | 'places' | 'd
* A booking with its day number, its place's name and its stay's place and
* times: the projection `ReservationsRepository.listForTrip` and
* `findWithJoins` narrow with their own `WHERE`. Kept out of the repository
* file, which is at its size baseline.
* file, which is at its size baseline. `accommodation_id` is TEXT and `ap.id`
* INTEGER, so the stay is matched through accommodationIdMatches: SQLite
* coerces the bare comparison, Postgres refuses it.
*/
export function joinedReservationQuery(kysely: Kysely<ReservationJoinDB>) {
export function joinedReservationQuery(kysely: Kysely<ReservationJoinDB>, platform: Platform) {
return kysely
.selectFrom('reservations as r')
.leftJoin('days as d', 'd.id', 'r.day_id')
.leftJoin('places as p', 'p.id', 'r.place_id')
.leftJoin('day_accommodations as ap', 'ap.id', 'r.accommodation_id')
.leftJoin('day_accommodations as ap', (join) =>
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'ap.id')),
)
.leftJoin('places as acc_p', 'acc_p.id', 'ap.place_id')
.selectAll('r')
.select([
+32
View File
@@ -0,0 +1,32 @@
import mikroOrmConfig from '../mikro-orm.config';
import type { AnyOrm } from './orm';
import { SharedSqliteDriver } from './orm-driver';
import { MikroORM as SqliteMikroORM } from '@mikro-orm/sqlite';
/**
* An ORM instance outside Nest, for the vitest global setup that builds the
* schema snapshot. Production uses the container-owned one from
* `MikroOrmModule.forRoot()` instead of creating a second.
*
* SQLite only: the snapshot is the SQLite schema and the init below is the
* SQLite driver's, so a config that names another engine is refused.
*/
export function createStandaloneOrm(): Promise<AnyOrm> {
if (mikroOrmConfig.driver !== SharedSqliteDriver) {
throw new Error('createStandaloneOrm builds the SQLite schema snapshot; unset TREK_DB_URL to run it.');
}
return SqliteMikroORM.init({
...mikroOrmConfig,
// Entity discovery is skipped deliberately. Migrations and seeders are raw
// SQL and need no metadata, and the global setup that calls this runs as ESM
// in vitest's main process, where discovering `*.entity.ts` fails on Node's
// extensionless import resolution.
entities: [],
entitiesTs: [],
discovery: { warnWhenNoEntities: false },
// The migrator's per-migration progress log is production boot output; the
// global setup already reports the snapshot in one line of its own, and a
// failing migration still throws.
migrations: { ...mikroOrmConfig.migrations, silent: true },
});
}