mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
feat(db): run TREK on PostgreSQL
TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before otherwise: an engine port under src/db/engine, the dialect helpers under src/db/dialect, a Postgres baseline for the migrations up to the cut, pg_dump/pg_restore backups, a probe of every repository method against a real server and a Postgres test lane in CI.
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
import { isPostgres, isSqlite, unsupported } from './platform';
|
||||
import type { Platform } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* The schema catalog, for the migrations' helpers (`db/migration-utils.ts`):
|
||||
* which tables and columns exist, each spelled the way its engine keeps them.
|
||||
* Both statements take the table name as their one bound parameter and answer
|
||||
* one row per hit with the name in `name`. Postgres reads the current schema
|
||||
* only, the one TREK's own tables live in.
|
||||
*/
|
||||
|
||||
/** Lists the columns of the table bound as `?`, one `name` per row; none for a missing table. */
|
||||
export function columnNamesSql(platform: Platform): string {
|
||||
if (isSqlite(platform)) return 'select name from pragma_table_info(?)';
|
||||
if (isPostgres(platform)) {
|
||||
return 'select column_name as name from information_schema.columns where table_schema = current_schema() and table_name = ?';
|
||||
}
|
||||
return unsupported(platform);
|
||||
}
|
||||
|
||||
/** One row when the table bound as `?` exists, none otherwise. */
|
||||
export function tableExistsSql(platform: Platform): string {
|
||||
if (isSqlite(platform)) return "select name from sqlite_master where type = 'table' and name = ?";
|
||||
if (isPostgres(platform)) {
|
||||
return 'select table_name as name from information_schema.tables where table_schema = current_schema() and table_name = ?';
|
||||
}
|
||||
return unsupported(platform);
|
||||
}
|
||||
|
||||
/**
|
||||
* The tables in the current schema, one `table_name` per row in name order:
|
||||
* the schema TREK's own tables live in. Postgres only, and the one definition
|
||||
* of that question: the bootstrap reads it through the connection, the client
|
||||
* tools' table listing hands it to psql.
|
||||
*/
|
||||
export const CURRENT_SCHEMA_TABLES_SQL =
|
||||
'select table_name from information_schema.tables where table_schema = current_schema() order by table_name';
|
||||
@@ -620,6 +620,17 @@ export function collateNoCase(platform: Platform, ref: string): RawQueryFragment
|
||||
return unsupported(platform);
|
||||
}
|
||||
|
||||
/**
|
||||
* A search that ignores case over `refs`: `(a LIKE ? ESCAPE '\' OR b LIKE ?
|
||||
* ...)`, one bound pattern per column, the caller escaping `%`, `_` and `\`.
|
||||
* SQLite's `LIKE` already ignores case (ASCII letters only); Postgres's is
|
||||
* exact, so it gets `ILIKE`, which folds every letter.
|
||||
*/
|
||||
export function ilike(platform: Platform, refs: readonly string[]): string {
|
||||
const operator = isSqlite(platform) ? 'LIKE' : isPostgres(platform) ? 'ILIKE' : unsupported(platform);
|
||||
return `(${refs.map((ref) => `${column(ref)} ${operator} ? ESCAPE '\\'`).join(' OR ')})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* `datetime('now', '-N hours')` — the hour-granularity sibling of
|
||||
* `nowMinusDays()` above, for `reminder-jobs.service.ts#todoTick`'s
|
||||
|
||||
@@ -18,13 +18,22 @@ export async function ensureEmailCaseIndex(connection: Connection): Promise<void
|
||||
const pending: unknown[] = await connection.execute(`SELECT value FROM app_settings WHERE key = ?`, [PENDING_KEY]);
|
||||
if (pending.length === 0) return;
|
||||
|
||||
const duplicates: Array<{ ids: string }> = await connection.execute(
|
||||
`SELECT group_concat(id, ', ') AS ids FROM (SELECT id, email FROM users ORDER BY id) GROUP BY lower(email) HAVING COUNT(*) > 1`,
|
||||
// Grouped here rather than with group_concat, which Postgres spells string_agg.
|
||||
const rows: Array<{ id: number; email_key: string }> = await connection.execute(
|
||||
`SELECT id, lower(email) AS email_key FROM users WHERE lower(email) IN ` +
|
||||
`(SELECT lower(email) FROM users GROUP BY lower(email) HAVING COUNT(*) > 1) ORDER BY id`,
|
||||
);
|
||||
if (duplicates.length > 0) {
|
||||
const duplicates = new Map<string, number[]>();
|
||||
for (const row of rows) {
|
||||
const group = duplicates.get(row.email_key);
|
||||
if (group) group.push(row.id);
|
||||
else duplicates.set(row.email_key, [row.id]);
|
||||
}
|
||||
if (duplicates.size > 0) {
|
||||
const ids = [...duplicates.values()].map((group) => group.join(', '));
|
||||
console.warn(
|
||||
`[DB] ${duplicates.length} email address(es) still belong to more than one account when case is ignored ` +
|
||||
`(user ids ${duplicates.map((row) => row.ids).join('; ')}), so users.email has no case-insensitive unique ` +
|
||||
`[DB] ${duplicates.size} email address(es) still belong to more than one account when case is ignored ` +
|
||||
`(user ids ${ids.join('; ')}), so users.email has no case-insensitive unique ` +
|
||||
'index yet. Give each of those accounts its own address under Admin > Users; the next start creates it.',
|
||||
);
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { EngineId } from '../../app-config/db-settings';
|
||||
import type { AnyOrm } from '../orm';
|
||||
import type { EntityManager, Options } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* One database engine: what the ORM needs to reach it, how the connection
|
||||
* opens, closes and comes back after a restore, and how a database is brought
|
||||
* to the current schema. Code that differs by engine dispatches through this,
|
||||
* or through the helpers in `db/dialect/`, never through its own platform
|
||||
* check (`lint:engine-dispatch`).
|
||||
*
|
||||
* Free of Nest like every other file under `db/`: `src/db` does not import
|
||||
* `src/nest` (`lint:boundaries`). `DatabaseLifecycle` is the provider that
|
||||
* calls it.
|
||||
*/
|
||||
export interface DatabaseEngine {
|
||||
readonly id: EngineId;
|
||||
/** For log lines and errors: the file path, or the URL without its password. */
|
||||
describe(): string;
|
||||
/** What MikroORM needs to reach the database: the driver and its target, the pool, driver options. */
|
||||
ormOptions(): Partial<Options>;
|
||||
/**
|
||||
* Called once at boot, before the schema bootstrap. `onReopened` runs after
|
||||
* each later reopen, once the ORM is pointed at the reopened database: the
|
||||
* engine does what its own reopen needs first (SQLite hands the ORM a new
|
||||
* handle, so the ORM's connection is closed and connected again), then calls
|
||||
* it, and the caller migrates the database and runs whatever follows.
|
||||
*/
|
||||
open(orm: AnyOrm, onReopened: () => Promise<void>): void;
|
||||
/** Closes what the engine itself holds; the ORM's own pool is closed with the Nest container. */
|
||||
close(): void;
|
||||
/**
|
||||
* Comes back after the database was replaced by a restore, then runs the
|
||||
* `onReopened` that `open` registered. Throws when `open` has not run.
|
||||
*/
|
||||
reopen(): Promise<void>;
|
||||
/**
|
||||
* Brings the database to the current schema, then seeds it. `snapshot:
|
||||
* false` skips the copy taken before pending migrations (a restore, whose
|
||||
* archive is the copy).
|
||||
*/
|
||||
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void>;
|
||||
/**
|
||||
* Runs first in every `UnitOfWork` transaction, on its EntityManager: makes
|
||||
* the transactions that write run one at a time, as SQLite's single writer
|
||||
* does, so a `MAX(...) + 1` read inside one cannot race another.
|
||||
*/
|
||||
serializeWrites(em: EntityManager): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { readEnv } from '../../app-config';
|
||||
import type { DbEngineSettings } from '../../app-config/db-settings';
|
||||
import type { DbSettings } from '../../app-config/derive';
|
||||
import { ALL_ENTITIES } from '../entities';
|
||||
import type { DatabaseEngine } from './engine';
|
||||
import { PostgresEngine } from './postgres/postgres-engine';
|
||||
import { SqliteEngine } from './sqlite/sqlite-engine';
|
||||
import { defineConfig, type Options } from '@mikro-orm/core';
|
||||
import { Migrator } from '@mikro-orm/migrations';
|
||||
import { SeedManager } from '@mikro-orm/seeder';
|
||||
|
||||
export type { DatabaseEngine } from './engine';
|
||||
|
||||
/**
|
||||
* The engine this process runs on, from `TREK_DB_URL` (`readEnv().db`): the
|
||||
* SQLite file unless the URL names a Postgres database. Read afresh on every
|
||||
* call, like everything `readEnv()` answers, so nothing caches a choice a test
|
||||
* or a restart changes.
|
||||
*/
|
||||
export function resolveEngine(db: DbEngineSettings = readEnv().db): DatabaseEngine {
|
||||
if (db.engine === 'postgres') return new PostgresEngine(db);
|
||||
return new SqliteEngine();
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres URL and the directory of its client tools, when the settings
|
||||
* name a Postgres database; null on SQLite. For the code that drives pg_dump
|
||||
* and pg_restore, which needs both and must not read them from the environment
|
||||
* again.
|
||||
*/
|
||||
export function postgresTarget(db: DbSettings): { url: string; binDir: string | undefined } | null {
|
||||
if (db.engine !== 'postgres') return null;
|
||||
return { url: db.url, binDir: db.pgBinDir };
|
||||
}
|
||||
|
||||
/**
|
||||
* The MikroORM config for `engine`: what every engine shares (entities,
|
||||
* migrations, seeders) and what the engine adds. `src/mikro-orm.config.ts`
|
||||
* exports it for the CLI and the Nest container, so both reach the same
|
||||
* database.
|
||||
*/
|
||||
export function buildOrmConfig(engine: DatabaseEngine): Options {
|
||||
return defineConfig({
|
||||
entities: ALL_ENTITIES,
|
||||
extensions: [Migrator, SeedManager],
|
||||
migrations: {
|
||||
path: 'dist/db/migrations',
|
||||
pathTs: 'src/db/migrations',
|
||||
snapshot: false,
|
||||
},
|
||||
seeder: {
|
||||
path: 'dist/db/seeders',
|
||||
pathTs: 'src/db/seeders',
|
||||
defaultSeeder: 'DatabaseSeeder',
|
||||
},
|
||||
...engine.ormOptions(),
|
||||
}) as Options;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { splitStatements } from './sql-split';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* Where the Postgres baseline lives: `server/assets/postgres/`, which the
|
||||
* Docker image copies beside `dist/` (the same place the airports and atlas
|
||||
* data are read from), reached the same way from `src/` and from `dist/`.
|
||||
*/
|
||||
const BASELINE_DIR = path.join(__dirname, '..', '..', '..', '..', 'assets', 'postgres');
|
||||
const SQL_FILE = 'baseline.sql';
|
||||
const MANIFEST_FILE = 'baseline.json';
|
||||
|
||||
/** The manifest beside the SQL: the last migration the baseline stands for, and every migration it replaces. */
|
||||
interface BaselineManifest {
|
||||
through: string;
|
||||
migrations: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The schema a fresh Postgres database starts from.
|
||||
*
|
||||
* The migrations up to `through` are written in SQLite and frozen, so a
|
||||
* Postgres database does not replay them: it runs these statements, generated
|
||||
* from the entities (`npm run gen:pg-baseline`), and records every migration
|
||||
* in `migrations` as executed in the same transaction. Migrations after
|
||||
* `through` run on both engines through the migrator.
|
||||
*/
|
||||
export interface PostgresBaseline extends BaselineManifest {
|
||||
statements: string[];
|
||||
}
|
||||
|
||||
/** The two files a baseline is written as, ready to compare or write. */
|
||||
export function renderBaseline(baseline: PostgresBaseline): { sql: string; manifest: string } {
|
||||
return {
|
||||
sql:
|
||||
'-- The schema a fresh Postgres database starts from. Generated by `npm run gen:pg-baseline`; do not edit.\n' +
|
||||
baseline.statements.map((statement) => `${statement};\n`).join(''),
|
||||
manifest: `${JSON.stringify({ through: baseline.through, migrations: baseline.migrations }, null, 2)}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `data` is a manifest a baseline can be read from: a `through` naming
|
||||
* one of a non-empty list of `migrations`, every entry a string.
|
||||
*/
|
||||
function isBaselineManifest(data: unknown): data is BaselineManifest {
|
||||
if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false;
|
||||
const { through, migrations } = data;
|
||||
return (
|
||||
typeof through === 'string' &&
|
||||
Array.isArray(migrations) &&
|
||||
migrations.length > 0 &&
|
||||
migrations.every((name): name is string => typeof name === 'string') &&
|
||||
migrations.includes(through)
|
||||
);
|
||||
}
|
||||
|
||||
function parseManifest(text: string, file: string): BaselineManifest {
|
||||
const data: unknown = JSON.parse(text);
|
||||
if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`);
|
||||
return { through: data.through, migrations: data.migrations };
|
||||
}
|
||||
|
||||
/** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */
|
||||
export function readPostgresBaseline(dir = BASELINE_DIR): PostgresBaseline {
|
||||
const manifestFile = path.join(dir, MANIFEST_FILE);
|
||||
const manifest = parseManifest(fs.readFileSync(manifestFile, 'utf8'), manifestFile);
|
||||
const statements = splitStatements(fs.readFileSync(path.join(dir, SQL_FILE), 'utf8').replace(/^--[^\n]*\n/gm, ''));
|
||||
if (statements.length === 0) throw new Error(`${path.join(dir, SQL_FILE)} holds no statements`);
|
||||
// The boot sends the baseline as one statement without parameters, where a `?` would be read as a binding.
|
||||
if (statements.some((statement) => statement.includes('?'))) {
|
||||
throw new Error(`${path.join(dir, SQL_FILE)} holds a "?", which the boot would read as a parameter`);
|
||||
}
|
||||
return { ...manifest, statements };
|
||||
}
|
||||
|
||||
/** Writes the two files (the generator's `--write`). */
|
||||
export function writePostgresBaseline(baseline: PostgresBaseline, dir = BASELINE_DIR): void {
|
||||
const { sql, manifest } = renderBaseline(baseline);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, SQL_FILE), sql);
|
||||
fs.writeFileSync(path.join(dir, MANIFEST_FILE), manifest);
|
||||
}
|
||||
|
||||
/** Whether the committed files are what `baseline` renders to (the generator's `--check`). */
|
||||
export function baselineMatches(baseline: PostgresBaseline, dir = BASELINE_DIR): boolean {
|
||||
const { sql, manifest } = renderBaseline(baseline);
|
||||
const read = (file: string) =>
|
||||
fs.existsSync(path.join(dir, file)) ? fs.readFileSync(path.join(dir, file), 'utf8') : null;
|
||||
return read(SQL_FILE) === sql && read(MANIFEST_FILE) === manifest;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Drops every table, view, sequence, routine, collation and standalone type in
|
||||
* the current schema, except an extension's, before a dump is loaded: a dump
|
||||
* only recreates what it holds, so whatever a later migration added would
|
||||
* otherwise survive the restore and break the migrations the reopen runs. It
|
||||
* runs in the same transaction as the load (`pgRestore`), so a restore that
|
||||
* fails leaves the database as it was.
|
||||
*/
|
||||
export const CLEAR_SCHEMA_SQL = `DO $clear$
|
||||
DECLARE r record;
|
||||
BEGIN
|
||||
FOR r IN SELECT c.oid::regclass AS name, c.relkind FROM pg_class c
|
||||
WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype IN ('e', 'a', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP %s IF EXISTS %s CASCADE',
|
||||
CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' WHEN 'S' THEN 'SEQUENCE'
|
||||
WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END, r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT p.oid::regprocedure AS sig FROM pg_proc p
|
||||
WHERE p.pronamespace = current_schema()::regnamespace
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP ROUTINE IF EXISTS %s CASCADE', r.sig);
|
||||
END LOOP;
|
||||
FOR r IN SELECT format('%I.%I', n.nspname, c.collname) AS name FROM pg_collation c
|
||||
JOIN pg_namespace n ON n.oid = c.collnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP COLLATION IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT t.oid::regtype AS name FROM pg_type t
|
||||
WHERE t.typnamespace = current_schema()::regnamespace AND t.typtype IN ('e', 'd', 'r')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = t.oid AND d.deptype IN ('e', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP TYPE IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
END
|
||||
$clear$;
|
||||
`;
|
||||
@@ -0,0 +1,119 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** The Postgres client tools a backup runs. */
|
||||
export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
|
||||
|
||||
/**
|
||||
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
|
||||
* Debian's versioned directories newest first (the Docker image installs 17),
|
||||
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
|
||||
*/
|
||||
const DEFAULT_BIN_DIRS = [
|
||||
'/usr/lib/postgresql/17/bin',
|
||||
'/usr/lib/postgresql/16/bin',
|
||||
'/usr/lib/postgresql/15/bin',
|
||||
'/usr/lib/postgresql/14/bin',
|
||||
'/usr/local/bin',
|
||||
'/usr/bin',
|
||||
];
|
||||
|
||||
/** A dump or a restore of a large database takes minutes, not hours. */
|
||||
const TOOL_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
||||
/** An unreachable host fails the tool in seconds instead of holding a boot or a restore. */
|
||||
const CONNECT_TIMEOUT_SECONDS = '10';
|
||||
/** How much of a tool's stderr an error keeps. */
|
||||
const STDERR_LIMIT = 2000;
|
||||
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
|
||||
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
|
||||
|
||||
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
|
||||
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
|
||||
const dirs = binDir ? [binDir] : DEFAULT_BIN_DIRS;
|
||||
const executable = (candidate: string): boolean => {
|
||||
try {
|
||||
fs.accessSync(candidate, fs.constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false; // missing or not executable: the next directory may have it
|
||||
}
|
||||
};
|
||||
return dirs.map((dir) => path.join(dir, tool)).find(executable) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The libpq environment for `url`: the tools read the password from
|
||||
* PGPASSWORD, so it never appears in an argument list another process can
|
||||
* read. An sslmode that asks for SSL verifies the server's certificate, as the
|
||||
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
|
||||
* else the system's trusted CAs.
|
||||
*/
|
||||
export function pgConnectionEnv(url: string): Record<string, string> {
|
||||
const parsed = new URL(url);
|
||||
const env: Record<string, string> = {
|
||||
PGHOST: parsed.hostname.replace(/^\[(.*)\]$/, '$1'),
|
||||
PGPORT: parsed.port || '5432',
|
||||
PGDATABASE: decodeURIComponent(parsed.pathname.replace(/^\//, '')),
|
||||
PGCONNECT_TIMEOUT: CONNECT_TIMEOUT_SECONDS,
|
||||
};
|
||||
if (parsed.username) env.PGUSER = decodeURIComponent(parsed.username);
|
||||
if (parsed.password) env.PGPASSWORD = decodeURIComponent(parsed.password);
|
||||
const sslmode = parsed.searchParams.get('sslmode');
|
||||
if (sslmode === 'disable') env.PGSSLMODE = 'disable';
|
||||
else if (sslmode) {
|
||||
env.PGSSLMODE = 'verify-full';
|
||||
env.PGSSLROOTCERT = parsed.searchParams.get('sslrootcert') ?? 'system';
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
/** A client tool, refused with a clear message when it is not installed. */
|
||||
export class PgToolMissingError extends Error {
|
||||
constructor(tool: PgTool) {
|
||||
super(
|
||||
`${tool} was not found, so a Postgres database cannot be backed up or restored. ` +
|
||||
'Install the PostgreSQL client tools (the Docker image has them) or point TREK_DB_PG_BIN_DIR at them.',
|
||||
);
|
||||
this.name = 'PgToolMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
|
||||
function summarizeStderr(stderr: string | undefined): string {
|
||||
const kept = (stderr ?? '')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim() !== '' && !DATA_LINE.test(line))
|
||||
.join('\n');
|
||||
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `tool` by its absolute path and resolves with its stdout. A failure
|
||||
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
|
||||
* the cause.
|
||||
*/
|
||||
export async function run(
|
||||
tool: PgTool,
|
||||
binDir: string | undefined,
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, args, {
|
||||
env,
|
||||
timeout: TOOL_TIMEOUT_MS,
|
||||
maxBuffer: TOOL_MAX_OUTPUT_BYTES,
|
||||
});
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const message = summarizeStderr((err as { stderr?: string }).stderr);
|
||||
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
|
||||
import { CLEAR_SCHEMA_SQL } from './clear-schema';
|
||||
import { pgConnectionEnv, run } from './pg-process';
|
||||
import { refuseToolsWithoutRestrict } from './restore-safety';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import readline from 'node:readline';
|
||||
|
||||
/**
|
||||
* Restore's session settings: a restore waiting on a lock another session
|
||||
* holds gives up, and rolls back, instead of hanging; the NOTICEs every
|
||||
* CASCADE prints stay out of its stderr.
|
||||
*/
|
||||
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
|
||||
|
||||
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
|
||||
function createPrivateFile(file: string): void {
|
||||
fs.writeFileSync(file, '', { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The schema TREK's tables live in: its name, and the name as SQL quotes it (`quote_ident`). */
|
||||
interface PgSchema {
|
||||
name: string;
|
||||
quoted: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The current schema of the database at `url`, the one TREK creates its
|
||||
* tables in. A backup and a restore stay inside it: a hosted database
|
||||
* (Supabase) keeps the platform's own schemas, `auth` among them, beside it,
|
||||
* and they are neither TREK's to copy nor to replace.
|
||||
*/
|
||||
async function pgCurrentSchema(url: string, binDir: string | undefined): Promise<PgSchema> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--no-align',
|
||||
'--tuples-only',
|
||||
"--command=select concat_ws(E'\\t', current_schema(), quote_ident(current_schema()))",
|
||||
],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
const [name, quoted] = out.replace(/\n$/, '').split('\t');
|
||||
if (!name || !quoted) {
|
||||
throw new Error('The database has no current schema: set the search_path of the role in TREK_DB_URL.');
|
||||
}
|
||||
return { name, quoted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a custom-format dump of the current schema of the database at `url`
|
||||
* to `target`, readable by its owner only. It is written beside the target and
|
||||
* renamed into place, so a dump that fails or is killed never leaves a file
|
||||
* that passes for a copy.
|
||||
*/
|
||||
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
|
||||
const partial = `${target}.partial`;
|
||||
try {
|
||||
createPrivateFile(partial);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
await run(
|
||||
'pg_dump',
|
||||
binDir,
|
||||
[
|
||||
'--format=custom',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
`--file=${partial}`,
|
||||
`--schema="${schema.name.replaceAll('"', '""')}"`,
|
||||
],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
fs.renameSync(partial, target);
|
||||
} catch (err) {
|
||||
fs.rmSync(partial, { force: true });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Escapes every character a RegExp would read as syntax. */
|
||||
function escapeRegExp(text: string): string {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the restore script creates TREK's migration table in `schema`: a
|
||||
* dump of TREK's own schema does. The script is read line by line and the
|
||||
* read stops at the first match: the COPY data that fills a large script is
|
||||
* never loaded, and a script past V8's maximum string length would not load
|
||||
* at all.
|
||||
*/
|
||||
async function scriptHoldsTrekSchema(script: string, schema: PgSchema): Promise<boolean> {
|
||||
const createsHistory = new RegExp(`^CREATE TABLE ${escapeRegExp(`${schema.quoted}.mikro_orm_migrations`)} \\(`);
|
||||
const input = fs.createReadStream(script, { encoding: 'utf8' });
|
||||
const lines = readline.createInterface({ input, crlfDelay: Infinity });
|
||||
try {
|
||||
for await (const line of lines) {
|
||||
if (createsHistory.test(line)) return true;
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
lines.close();
|
||||
input.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the current schema of the database at `url` with the dump at
|
||||
* `source`: pg_restore turns the dump's objects in that schema into a script
|
||||
* beside it (anything else the dump holds is left out), then psql clears the
|
||||
* schema and runs the script in one transaction, so a restore that fails
|
||||
* leaves the database as it was. A dump without TREK's tables in that schema
|
||||
* is refused before anything is cleared.
|
||||
*/
|
||||
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
|
||||
await refuseToolsWithoutRestrict(binDir);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
|
||||
try {
|
||||
const clear = path.join(workDir, 'clear-schema.sql');
|
||||
const script = path.join(workDir, 'restore.sql');
|
||||
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
|
||||
createPrivateFile(script);
|
||||
await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--no-owner', '--no-privileges', `--schema=${schema.name}`, `--file=${script}`, '--', source],
|
||||
{},
|
||||
);
|
||||
if (!(await scriptHoldsTrekSchema(script, schema))) {
|
||||
throw new Error(`The backup holds no TREK tables in the schema "${schema.name}"; nothing was replaced.`);
|
||||
}
|
||||
await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--quiet',
|
||||
'--single-transaction',
|
||||
'--set=ON_ERROR_STOP=1',
|
||||
`--output=${os.devNull}`,
|
||||
`--file=${clear}`,
|
||||
`--file=${script}`,
|
||||
],
|
||||
{ ...pgConnectionEnv(url), PGOPTIONS: RESTORE_SESSION_OPTIONS },
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** The tables in the current schema of the database at `url`, read before TREK opens it. */
|
||||
export async function pgTableNames(url: string, binDir: string | undefined): Promise<string[]> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
['--no-psqlrc', '--no-align', '--tuples-only', `--command=${CURRENT_SCHEMA_TABLES_SQL}`],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
return out.split('\n').filter((line) => line !== '');
|
||||
}
|
||||
|
||||
/**
|
||||
* The migration names in a restore script's COPY block for
|
||||
* `mikro_orm_migrations`, read from its `name` column; none when the script
|
||||
* has no such block.
|
||||
*/
|
||||
function migrationNamesFromCopy(script: string): string[] {
|
||||
const lines = script.split('\n');
|
||||
const copy = lines.findIndex((line) => /^COPY \S*mikro_orm_migrations \(/.test(line));
|
||||
if (copy < 0) return [];
|
||||
const columns =
|
||||
/\(([^)]*)\)/
|
||||
.exec(lines[copy])?.[1]
|
||||
.split(',')
|
||||
.map((column) => column.trim()) ?? [];
|
||||
const nameAt = columns.indexOf('name');
|
||||
if (nameAt < 0) return [];
|
||||
const names: string[] = [];
|
||||
for (const line of lines.slice(copy + 1)) {
|
||||
if (line === '\\.' || line === '') break;
|
||||
const name = line.split('\t')[nameAt];
|
||||
if (name) names.push(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* The migrations a dump records as executed, read from the dump itself (its
|
||||
* `mikro_orm_migrations` rows as COPY text), so a restore can refuse a newer
|
||||
* TREK's backup before anything is replaced. Throws on a file that is not a
|
||||
* dump; an empty list means the dump holds no TREK history.
|
||||
*/
|
||||
export async function pgDumpMigrations(source: string, binDir: string | undefined): Promise<string[]> {
|
||||
const script = await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--data-only', '--table=mikro_orm_migrations', '--file=-', '--', source],
|
||||
{},
|
||||
);
|
||||
return migrationNamesFromCopy(script);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { type Configuration } from '@mikro-orm/core';
|
||||
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
|
||||
|
||||
/** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */
|
||||
const INT8_OID = 20;
|
||||
/** `numeric`: what `sum()` of a bigint and `avg()` answer. */
|
||||
const NUMERIC_OID = 1700;
|
||||
|
||||
/**
|
||||
* Reads a `bigint` as a JavaScript number, the way better-sqlite3 hands
|
||||
* SQLite's 64-bit INTEGER to the services. `pg` returns it as a string to
|
||||
* keep precision past 2^53; no TREK value comes near that (ids, counts,
|
||||
* epoch milliseconds, cents), so one that does is refused rather than
|
||||
* rounded.
|
||||
*/
|
||||
export function parseInt8(text: string): number {
|
||||
const value = Number(text);
|
||||
if (!Number.isSafeInteger(value)) throw new Error(`A bigint of ${text} is beyond what TREK reads exactly (2^53).`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a `numeric` as a number, as SQLite answers `sum()` and `avg()`. `NaN`
|
||||
* and the infinities are refused, as is an integral result past 2^53, which is
|
||||
* refused like a bigint; a fraction is a double, the precision SQLite's REAL has.
|
||||
*/
|
||||
export function parseNumeric(text: string): number {
|
||||
const value = Number(text);
|
||||
if (!Number.isFinite(value)) {
|
||||
throw new Error(`A numeric of ${text} is not a finite number, which TREK does not read.`);
|
||||
}
|
||||
if (/^-?\d+$/.test(text) && !Number.isSafeInteger(value)) {
|
||||
throw new Error(`A numeric of ${text} is beyond what TREK reads exactly (2^53).`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's `PostgreSqlConnection`, with the pool's own type parsers replaced
|
||||
* for `int8` and `numeric`. MikroORM keeps dates as text, so only these two
|
||||
* change. `mapOptions` builds the `pg` pool options, whose `types` is the
|
||||
* `pg` TypeOverrides, the object that holds the parsers.
|
||||
*/
|
||||
class NumberTypesConnection extends PostgreSqlConnection {
|
||||
override mapOptions(overrides: Parameters<PostgreSqlConnection['mapOptions']>[0]) {
|
||||
const options = super.mapOptions(overrides);
|
||||
const types = options.types as { setTypeParser(oid: number, parse: (text: string) => unknown): void };
|
||||
types.setTypeParser(INT8_OID, parseInt8);
|
||||
types.setTypeParser(NUMERIC_OID, parseNumeric);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
|
||||
* as numbers. The parsers belong to this pool only: nothing global in `pg`
|
||||
* changes.
|
||||
*/
|
||||
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
|
||||
constructor(config: Configuration) {
|
||||
super(config);
|
||||
// PostgreSqlDriver's constructor hardcodes its connection class, as
|
||||
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
|
||||
// field, which is not public, so it is replaced here, before anything connects.
|
||||
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { DbEngineSettings } from '../../../app-config/db-settings';
|
||||
import { NO_SAFETY_NET } from '../../legacy-baseline';
|
||||
import { finishSchemaBootstrap, type AnyOrm } from '../../orm';
|
||||
import type { DatabaseEngine } from '../engine';
|
||||
import { redactUrl } from '../redact-url';
|
||||
import { TrekPostgreSqlDriver } from './postgres-driver';
|
||||
import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap';
|
||||
import { sslOptionFor } from './ssl';
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */
|
||||
type PostgresSettings = Extract<DbEngineSettings, { engine: 'postgres' }>;
|
||||
|
||||
/** The advisory lock key every write transaction takes: "TREK" in ASCII. */
|
||||
const TREK_WRITE_LOCK = 0x5452454b;
|
||||
|
||||
/**
|
||||
* The Postgres engine: MikroORM's own `pg` driver on `TREK_DB_URL`, with a
|
||||
* pool of at most `TREK_DB_POOL_MAX` connections that keeps none open while
|
||||
* idle. The pool belongs to the ORM, which the Nest container closes; the
|
||||
* schema comes from the baseline plus the migrations after it
|
||||
* (`schema-bootstrap.ts`).
|
||||
*/
|
||||
export class PostgresEngine implements DatabaseEngine {
|
||||
readonly id = 'postgres';
|
||||
|
||||
constructor(
|
||||
private readonly settings: PostgresSettings,
|
||||
private readonly bootstrapDeps: (url: string) => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
|
||||
) {}
|
||||
|
||||
private onReopened: (() => Promise<void>) | null = null;
|
||||
|
||||
/**
|
||||
* Nothing of its own to open: the ORM connects the pool. Keeps what a reopen
|
||||
* runs. A reopen replaces the database's contents, which the pool's own
|
||||
* connections keep reaching, so the ORM is not pointed at anything new and
|
||||
* the pool is never closed (ending a pool waits for every connection it has
|
||||
* lent out, and one lent to a job during the restore would never come back).
|
||||
*/
|
||||
open(_orm: AnyOrm, onReopened: () => Promise<void>): void {
|
||||
this.onReopened = onReopened;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
// Nothing of its own to close: the pool goes with the ORM.
|
||||
}
|
||||
|
||||
/**
|
||||
* After `pg_restore` replaced the database's contents: the restored
|
||||
* database is migrated forward on the same pool (`onReopened`, which
|
||||
* `DatabaseLifecycle` hands to `open`).
|
||||
*/
|
||||
async reopen(): Promise<void> {
|
||||
if (!this.onReopened) throw new Error('The Postgres engine was reopened before it was opened.');
|
||||
await this.onReopened();
|
||||
}
|
||||
|
||||
/**
|
||||
* `pg_advisory_xact_lock` on one key for the whole database: held until the
|
||||
* transaction commits or rolls back, taken again by a savepoint in the same
|
||||
* session without waiting, and shared by every process on the database.
|
||||
*/
|
||||
async serializeWrites(em: EntityManager): Promise<void> {
|
||||
await em
|
||||
.getConnection()
|
||||
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
|
||||
}
|
||||
|
||||
/** `snapshot: false` (a restore, whose archive is the copy) migrates without a pg_dump first. */
|
||||
async schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
|
||||
const deps = this.bootstrapDeps(this.settings.url);
|
||||
await bootstrapPostgresSchema(orm, options.snapshot ? deps : { ...deps, safetyNet: NO_SAFETY_NET });
|
||||
await finishSchemaBootstrap(orm);
|
||||
}
|
||||
|
||||
describe(): string {
|
||||
return `Postgres ${redactUrl(this.settings.url)}`;
|
||||
}
|
||||
|
||||
ormOptions() {
|
||||
const ssl = sslOptionFor(this.settings.url);
|
||||
return {
|
||||
driver: TrekPostgreSqlDriver,
|
||||
clientUrl: this.settings.url,
|
||||
pool: { min: 0, max: this.settings.poolMax },
|
||||
driverOptions: ssl === undefined ? {} : { ssl },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* The SQLite schema's triggers as Postgres writes them. The entities describe
|
||||
* tables only, so the baseline (`scripts/generate-pg-baseline.ts`) adds these
|
||||
* after the tables, and `check:pg-baseline` fails while a trigger in the
|
||||
* migrated SQLite schema has no entry here. Each entry is the trigger's
|
||||
* function and the trigger that runs it, under the SQLite trigger's name.
|
||||
*/
|
||||
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
|
||||
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
|
||||
trg_release_stop_on_stay_delete: [
|
||||
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
|
||||
RETURN OLD;
|
||||
END
|
||||
$$`,
|
||||
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
|
||||
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
|
||||
],
|
||||
// A place that moved or was renamed drops the region it was resolved to (#2527).
|
||||
trg_place_regions_follow_place: [
|
||||
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
DELETE FROM place_regions WHERE place_id = NEW.id;
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`,
|
||||
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
|
||||
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
|
||||
'EXECUTE FUNCTION trg_place_regions_follow_place()',
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,50 @@
|
||||
import { run } from './pg-process';
|
||||
|
||||
/**
|
||||
* The client-tool versions a restore accepts (CVE-2025-8714). A script from a
|
||||
* dump an admin uploaded is run by psql, which would also run a psql
|
||||
* meta-command (`\!` runs a shell command) smuggled into an object name. A
|
||||
* fixed pg_restore opens the script with `\restrict <random key>`, and a fixed
|
||||
* psql then refuses every meta-command until the matching `\unrestrict`.
|
||||
*
|
||||
* RESTRICT_SINCE holds, per major, the first minor that carries the fix. Every
|
||||
* major after the last one listed has it from its first release.
|
||||
*/
|
||||
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
|
||||
const LAST_MAJOR_WITHOUT_IT = Math.max(...Object.keys(RESTRICT_SINCE).map(Number));
|
||||
|
||||
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
|
||||
export function hasRestrictFix(version: string): boolean {
|
||||
const match = /\(PostgreSQL\) (\d+)\.(\d+)/.exec(version);
|
||||
if (!match) return false;
|
||||
const major = Number(match[1]);
|
||||
const minor = Number(match[2]);
|
||||
if (major > LAST_MAJOR_WITHOUT_IT) return true;
|
||||
const firstFixedMinor = RESTRICT_SINCE[major];
|
||||
return firstFixedMinor !== undefined && minor >= firstFixedMinor;
|
||||
}
|
||||
|
||||
/** The minimum versions as the error names them, newest major first: "17.6, 16.10, 15.14 or 14.19". */
|
||||
function minimumVersionsText(): string {
|
||||
const floors = Object.keys(RESTRICT_SINCE)
|
||||
.map(Number)
|
||||
.sort((a, b) => b - a)
|
||||
.map((major) => `${major}.${RESTRICT_SINCE[major]}`);
|
||||
const last = floors.pop() ?? '';
|
||||
return floors.length > 0 ? `${floors.join(', ')} or ${last}` : last;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses the restore when pg_restore or psql predates the fix. The tools'
|
||||
* own versions are checked, not the script, whose text the dump controls.
|
||||
*/
|
||||
export async function refuseToolsWithoutRestrict(binDir: string | undefined): Promise<void> {
|
||||
const tools = ['pg_restore', 'psql'] as const;
|
||||
const versions = await Promise.all(tools.map((tool) => run(tool, binDir, ['--version'], {})));
|
||||
const stale = tools.findIndex((_, i) => !hasRestrictFix(versions[i]));
|
||||
if (stale < 0) return;
|
||||
throw new Error(
|
||||
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least ${minimumVersionsText()} ` +
|
||||
'(the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* How the entities' schema is written in Postgres: the one definition the
|
||||
* Postgres baseline (`scripts/generate-pg-baseline.ts`), the schema parity
|
||||
* check and the probe (`scripts/pg-probe.ts`) share.
|
||||
*
|
||||
* The entities are generated from the migrated SQLite schema and a parity
|
||||
* test pins them to it (`npm run check:entities`), so they are the one
|
||||
* description of the tables both engines can read. MikroORM's schema
|
||||
* generator turns them into Postgres DDL after the adjustments below, which
|
||||
* keep the storage model the services and the dialect helpers are written
|
||||
* for (server/CLAUDE.md, "Storage model"):
|
||||
*
|
||||
* - timestamps stay TEXT;
|
||||
* - an INTEGER column is `bigint`: SQLite's INTEGER holds 64 bits, so a
|
||||
* column written with epoch milliseconds or a byte count past 2 GB would
|
||||
* overflow Postgres's 32-bit `integer` (the engine's driver reads
|
||||
* `bigint` back as a number, `postgres-driver.ts`);
|
||||
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`,
|
||||
* `CURRENT_TIMESTAMP` in a TEXT column) gets its Postgres equivalent, and
|
||||
* a column declared `COLLATE NOCASE` uses the `nocase` collation
|
||||
* `collateNoCase()` names on Postgres.
|
||||
*
|
||||
* The baseline keeps the foreign keys; the probe leaves them out, so a
|
||||
* statement it sends is never refused only because the sample row it writes
|
||||
* points at nothing. The triggers the SQLite schema has are in
|
||||
* `postgres-triggers.ts`.
|
||||
*/
|
||||
import { pgTimestampText, PG_UTC_NOW } from '../../dialect/platform';
|
||||
import { DbTimestampType } from '../../types';
|
||||
|
||||
/** SQLite's timestamp text on Postgres, for a column default. */
|
||||
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
|
||||
|
||||
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
|
||||
export const NOCASE_COLLATION_SQL =
|
||||
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
|
||||
|
||||
/** The Postgres spelling of a column default the entities carry in SQLite's. */
|
||||
export function portableDefault(raw: string): string {
|
||||
const text = raw
|
||||
.trim()
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
|
||||
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
|
||||
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** A column type SQLite stores as its 64-bit INTEGER. */
|
||||
const INTEGER_TYPE = /^(integer|int)$/i;
|
||||
|
||||
/**
|
||||
* The subset of MikroORM's property metadata the adjustments read and write.
|
||||
* An entity's `props` satisfy it as they are, so no cast is needed.
|
||||
*/
|
||||
export interface ColumnMeta {
|
||||
name: string;
|
||||
defaultRaw?: string;
|
||||
collation?: string;
|
||||
columnTypes?: string[];
|
||||
customType?: unknown;
|
||||
}
|
||||
|
||||
/** A timestamp column becomes text. Returns whether it changed. */
|
||||
function timestampAsText(column: ColumnMeta): boolean {
|
||||
if (!(column.customType instanceof DbTimestampType) || !column.columnTypes) return false;
|
||||
column.columnTypes = column.columnTypes.map(() => 'text');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** An integer column becomes bigint. Returns whether it changed. */
|
||||
function integerAsBigint(column: ColumnMeta): boolean {
|
||||
if (!column.columnTypes?.some((type) => INTEGER_TYPE.test(type))) return false;
|
||||
column.columnTypes = column.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A SQLite-only default gets its Postgres spelling. Returns whether it changed. */
|
||||
function defaultAsPostgres(column: ColumnMeta): boolean {
|
||||
if (typeof column.defaultRaw !== 'string') return false;
|
||||
const next = portableDefault(column.defaultRaw);
|
||||
if (next === column.defaultRaw) return false;
|
||||
column.defaultRaw = next;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A NOCASE column uses the `nocase` collation. Returns whether it changed. */
|
||||
function nocaseAsCollation(column: ColumnMeta): boolean {
|
||||
if (column.collation?.toUpperCase() !== 'NOCASE') return false;
|
||||
column.collation = 'nocase';
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the adjustments above to an entity's properties, in place, and
|
||||
* returns the names of the properties it changed, for the log. A timestamp
|
||||
* column is text, so the integer adjustment never sees it.
|
||||
*/
|
||||
export function applyPostgresColumnAdjustments(properties: readonly ColumnMeta[]): string[] {
|
||||
const changed: string[] = [];
|
||||
for (const property of properties) {
|
||||
const typeChanged = timestampAsText(property) || integerAsBigint(property);
|
||||
const defaultChanged = defaultAsPostgres(property);
|
||||
const collationChanged = nocaseAsCollation(property);
|
||||
if (typeChanged || defaultChanged || collationChanged) changed.push(property.name);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import { readEnv, type AppEnv } from '../../../app-config';
|
||||
import { resolveDataPaths } from '../../../app-config/data-paths';
|
||||
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
|
||||
import { knownMigrationNames } from '../../known-migrations';
|
||||
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
|
||||
import type { AnyOrm } from '../../orm';
|
||||
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
|
||||
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
|
||||
import { pgDump } from './pg-tools';
|
||||
import type { Connection } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const logger = new Logger('DB');
|
||||
|
||||
/** A Postgres copy is a pg_dump custom-format archive. */
|
||||
const DUMP_EXTENSION = '.pgdump';
|
||||
|
||||
/** What a Postgres bootstrap reads, passed in so a test can supply its own. */
|
||||
export interface PostgresBootstrapDeps {
|
||||
baseline: () => PostgresBaseline;
|
||||
known: Set<string>;
|
||||
safetyNet: MigrationSafetyNet;
|
||||
}
|
||||
|
||||
/**
|
||||
* The tables in the current schema. A database with no TREK migration history
|
||||
* must hold none: TREK creates its schema in an empty database and never
|
||||
* adopts tables something else made. Read before anything is written, so a
|
||||
* refused database is left exactly as it was.
|
||||
*/
|
||||
async function currentTables(connection: Connection): Promise<string[]> {
|
||||
const rows: Array<{ table_name: string }> = await connection.execute(CURRENT_SCHEMA_TABLES_SQL);
|
||||
return rows.map((row) => row.table_name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the schema in an empty Postgres database: the baseline's statements
|
||||
* and the record of every migration it stands for, in one transaction (DDL is
|
||||
* transactional on Postgres, so the database ends with all of it or none). A
|
||||
* database holding tables beside no TREK history is refused before anything is
|
||||
* written.
|
||||
*/
|
||||
async function createFromBaseline(
|
||||
connection: Connection,
|
||||
storage: ReturnType<Migrator['getStorage']>,
|
||||
tables: readonly string[],
|
||||
historyTable: string,
|
||||
baseline: () => PostgresBaseline,
|
||||
): Promise<void> {
|
||||
const foreign = tables.filter((table) => table !== historyTable);
|
||||
if (foreign.length > 0) {
|
||||
throw new Error(
|
||||
`${REFUSAL} the database holds ${foreign.length} table(s) (${foreign.slice(0, 5).join(', ')}) but no TREK ` +
|
||||
'migration history. Point TREK_DB_URL at an empty database.',
|
||||
);
|
||||
}
|
||||
await storage.ensureTable();
|
||||
const schema = baseline();
|
||||
logger.log(`Creating the schema in an empty Postgres database (through ${migrationLabel(schema.through)})`);
|
||||
await connection.transactional(async (trx) => {
|
||||
await connection.execute(schema.statements.join(';\n'), [], 'run', trx);
|
||||
await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx)));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a database whose recorded history names a migration this build does
|
||||
* not know: a newer TREK migrated it. The refusal names the copy taken before
|
||||
* that migration, when there is one.
|
||||
*/
|
||||
async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDeps): Promise<void> {
|
||||
const thisRelease = migrationLabel([...deps.known].sort().at(-1) ?? '');
|
||||
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings a Postgres database to the current schema: inspect what it holds,
|
||||
* then create the schema or verify the history it has, then migrate.
|
||||
*
|
||||
* - No migration recorded and no table: the baseline is created
|
||||
* (`createFromBaseline`).
|
||||
* - No migration recorded but tables present: refused, naming a few of them.
|
||||
* - Migrations recorded: refused when one is unknown to this build, else the
|
||||
* pending migrations run, after the safety net. A database the baseline just
|
||||
* created runs the migrations after the baseline's cut without it: there is
|
||||
* nothing yet to lose.
|
||||
*/
|
||||
export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstrapDeps): Promise<void> {
|
||||
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
|
||||
const connection = orm.em.getConnection();
|
||||
const historyTable = orm.config.get('migrations').tableName ?? 'mikro_orm_migrations';
|
||||
|
||||
const tables = await currentTables(connection);
|
||||
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
|
||||
|
||||
if (executed.length === 0) {
|
||||
await createFromBaseline(connection, migrator.getStorage(), tables, historyTable, deps.baseline);
|
||||
} else {
|
||||
await refuseNewerHistory(executed, deps);
|
||||
}
|
||||
|
||||
const pending = await migrator.getPending();
|
||||
if (pending.length === 0) return;
|
||||
// Only a database with history has a last migration to label its copy by; a
|
||||
// database the baseline just created has nothing yet to lose.
|
||||
const latest = [...executed].sort().at(-1);
|
||||
if (latest !== undefined) await deps.safetyNet.beforeMigrate(migrationLabel(latest));
|
||||
logger.log(`Applying ${pending.length} pending migration(s)`);
|
||||
await migrator.up();
|
||||
}
|
||||
|
||||
/** Dumps a killed boot left half-written: never a copy, only disk taken. */
|
||||
function removeOrphanedPartials(dir: string): void {
|
||||
for (const file of fs.readdirSync(dir)) {
|
||||
if (file.startsWith(PREFIX) && file.endsWith(`${DUMP_EXTENSION}.partial`)) {
|
||||
fs.rmSync(path.join(dir, file), { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres safety net before pending migrations, as the SQLite file's
|
||||
* (`db/pre-migrate-snapshot.ts`): a `pg_dump` of the database to
|
||||
* `<data>/pre-migrate-<schema>-<time>.pgdump`, the newest
|
||||
* `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` kept. A copy that cannot be written
|
||||
* (pg_dump missing, the data dir full) refuses the boot before anything is
|
||||
* migrated, unless `TREK_DB_PRE_MIGRATE_SNAPSHOT=false`. Off under
|
||||
* NODE_ENV=test unless a suite asks for it.
|
||||
*/
|
||||
export function postgresSafetyNet(
|
||||
url: string,
|
||||
env: () => AppEnv = readEnv,
|
||||
now: () => Date = () => new Date(),
|
||||
dataDir: () => string = () => resolveDataPaths(env()).dataDir,
|
||||
): MigrationSafetyNet {
|
||||
return {
|
||||
async beforeMigrate(from) {
|
||||
const { db, app } = env();
|
||||
if (db.preMigrateSnapshot === undefined && app.isTest) return;
|
||||
if (db.preMigrateSnapshot === false) {
|
||||
logger.warn('TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating the Postgres database without a copy');
|
||||
return;
|
||||
}
|
||||
const target = path.join(dataDir(), `${PREFIX}${from ?? 'unversioned'}-${stamp(now())}${DUMP_EXTENSION}`);
|
||||
try {
|
||||
fs.mkdirSync(dataDir(), { recursive: true });
|
||||
removeOrphanedPartials(dataDir());
|
||||
await pgDump(url, target, db.pgBinDir);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
|
||||
`(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Fix that, ` +
|
||||
'or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
logger.log(`Copy of the database before migrating: ${target}`);
|
||||
pruneSnapshots(dataDir(), db.preMigrateSnapshotKeep, DUMP_EXTENSION);
|
||||
},
|
||||
|
||||
async restoreHint(thisRelease) {
|
||||
const snapshots = listSnapshots(dataDir(), DUMP_EXTENSION);
|
||||
const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
|
||||
if (match) {
|
||||
return (
|
||||
`The copy taken right before the newer TREK migrated it is ${match}: stop TREK, empty the database ` +
|
||||
'(drop and recreate it), and restore it with pg_restore --no-owner --no-privileges --single-transaction.'
|
||||
);
|
||||
}
|
||||
if (snapshots.length > 0) {
|
||||
return `Copies taken before earlier migrations are in ${dataDir()} (${PREFIX}*${DUMP_EXTENSION}), none of them from this release.`;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
|
||||
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
|
||||
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Splits generated DDL into single statements, so one statement Postgres
|
||||
* refuses does not take the rest of the schema with it.
|
||||
*
|
||||
* Understood: `;` outside quotes, `'` and `"` quoted text, and the `$$` dollar
|
||||
* quote a function body is written in, which is the only dollar quote TREK's
|
||||
* generated DDL and baseline contain. A tagged dollar quote (`$tag$`) is not
|
||||
* understood, so it is refused rather than split inside.
|
||||
*/
|
||||
|
||||
/** A tagged dollar quote (`$tag$`), matched where the scan stands. */
|
||||
const TAGGED_DOLLAR_QUOTE = /\$[A-Za-z_]\w*\$/y;
|
||||
|
||||
function refuseTaggedDollarQuote(sql: string, at: number): void {
|
||||
TAGGED_DOLLAR_QUOTE.lastIndex = at;
|
||||
const tag = TAGGED_DOLLAR_QUOTE.exec(sql)?.[0];
|
||||
if (tag) throw new Error(`splitStatements understands only the $$ dollar quote, not ${tag} (at ${at})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The statements in `sql`, trimmed and in order. A statement that sets a
|
||||
* session parameter (`SET ...`) is dropped: a dump's preamble, which a schema
|
||||
* the entities generate never needs.
|
||||
*/
|
||||
export function splitStatements(sql: string): string[] {
|
||||
const statements: string[] = [];
|
||||
let current = '';
|
||||
let quote: '"' | "'" | '$$' | null = null;
|
||||
for (let i = 0; i < sql.length; i++) {
|
||||
const char = sql[i];
|
||||
const dollar = sql.startsWith('$$', i);
|
||||
if (quote === '$$') {
|
||||
if (dollar) {
|
||||
quote = null;
|
||||
current += '$';
|
||||
i++;
|
||||
}
|
||||
current += char;
|
||||
} else if (quote) {
|
||||
if (char === quote) quote = null;
|
||||
current += char;
|
||||
} else if (dollar) {
|
||||
quote = '$$';
|
||||
current += '$$';
|
||||
i++;
|
||||
} else if (char === "'" || char === '"') {
|
||||
quote = char;
|
||||
current += char;
|
||||
} else if (char === ';') {
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
if (char === '$') refuseTaggedDollarQuote(sql, i);
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
return statements.filter((statement) => !/^set\s/i.test(statement));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
/** The sslmode values that verify the server, read the way the `pg` driver reads them. */
|
||||
const VERIFYING_MODES = new Set(['require', 'verify-ca', 'verify-full']);
|
||||
|
||||
/** The `ssl` option of a `pg` pool, as `sslOptionFor` builds it. */
|
||||
type PgSslOption = false | { rejectUnauthorized: true; ca?: string };
|
||||
|
||||
/**
|
||||
* The `ssl` option for the pool, from the URL's `sslmode` and `sslrootcert`.
|
||||
*
|
||||
* MikroORM builds the pool from the URL's host, port, user, password and
|
||||
* database alone and drops the rest of the query, so an `sslmode` in
|
||||
* `TREK_DB_URL` would be ignored without this. `require`, `verify-ca` and
|
||||
* `verify-full` all verify the server's certificate, as the `pg` driver itself
|
||||
* treats them; `disable` turns SSL off, and no `sslmode` leaves the driver's
|
||||
* default (no SSL). A mode that would connect without verifying (`prefer`,
|
||||
* `allow`, `no-verify`) or that is misspelt refuses, rather than quietly
|
||||
* connecting with less than was asked. `sslrootcert` names the CA file a host
|
||||
* that signs its own certificates needs (Supabase's, say).
|
||||
*/
|
||||
export function sslOptionFor(url: string): PgSslOption | undefined {
|
||||
const params = new URL(url).searchParams;
|
||||
const mode = params.get('sslmode');
|
||||
const rootCert = params.get('sslrootcert');
|
||||
if (mode === null) {
|
||||
if (rootCert !== null) throw new Error('TREK_DB_URL names an sslrootcert without sslmode=verify-full.');
|
||||
return undefined;
|
||||
}
|
||||
if (mode === 'disable') return false;
|
||||
if (!VERIFYING_MODES.has(mode)) {
|
||||
throw new Error(
|
||||
`TREK_DB_URL has sslmode "${mode}"; use disable, require, verify-ca or verify-full (all but disable verify the server).`,
|
||||
);
|
||||
}
|
||||
if (rootCert === null) return { rejectUnauthorized: true };
|
||||
try {
|
||||
return { rejectUnauthorized: true, ca: readFileSync(rootCert, 'utf8') };
|
||||
} catch (error) {
|
||||
throw new Error(`TREK_DB_URL names sslrootcert ${rootCert}, which cannot be read.`, { cause: error });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* A connection URL fit for a log line: the password masked and the query
|
||||
* dropped, since libpq reads a password from the query too
|
||||
* (`?password=...`). A value that does not parse is not echoed at all.
|
||||
*/
|
||||
export function redactUrl(url: string): string {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return '<unparseable URL>';
|
||||
}
|
||||
if (parsed.password) parsed.password = '***';
|
||||
parsed.search = '';
|
||||
parsed.hash = '';
|
||||
return parsed.toString();
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { closeDb, getRawConnection, registerReinitializeHook, reinitialize } from '../../database';
|
||||
import { resolveDbPath } from '../../db-path';
|
||||
import { runSchemaBootstrap, type AnyOrm } from '../../orm';
|
||||
import { SharedSqliteDriver } from '../../orm-driver';
|
||||
import type { DatabaseEngine } from '../engine';
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* The SQLite file engine, TREK's default: one better-sqlite3 handle that
|
||||
* `db/database.ts` owns and the ORM's bound driver shares (`db/orm-driver.ts`),
|
||||
* on `TREK_DB_FILE` or `data/travel.db` (`:memory:` under test). A restore
|
||||
* swaps the file and reopens the handle; the schema comes from the migration
|
||||
* chain (`db/orm.ts`).
|
||||
*/
|
||||
export class SqliteEngine implements DatabaseEngine {
|
||||
readonly id = 'sqlite';
|
||||
|
||||
describe(): string {
|
||||
return `SQLite file ${resolveDbPath()}`;
|
||||
}
|
||||
|
||||
ormOptions() {
|
||||
return { driver: SharedSqliteDriver, dbName: resolveDbPath() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the file (normally a no-op: the ORM's first connect inside
|
||||
* NestFactory.create() already did, said here so the boot does not depend on
|
||||
* that), and registers the reopen. A reopen swaps the file under the same
|
||||
* handle, which Kysely caches, so the ORM's connection is closed and connected
|
||||
* again first: the driver then reaches the new handle through
|
||||
* `createKyselyDialect()`. The bound driver's `destroy()` is a no-op
|
||||
* (`orm-driver.ts`), so the close never reaches the handle `db/database.ts` owns.
|
||||
*/
|
||||
open(orm: AnyOrm, onReopened: () => Promise<void>): void {
|
||||
getRawConnection();
|
||||
registerReinitializeHook(async () => {
|
||||
const connection = orm.em.getConnection();
|
||||
await connection.close(true);
|
||||
await connection.connect();
|
||||
await onReopened();
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {
|
||||
closeDb();
|
||||
}
|
||||
|
||||
reopen(): Promise<void> {
|
||||
return reinitialize();
|
||||
}
|
||||
|
||||
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
|
||||
return runSchemaBootstrap(orm, options);
|
||||
}
|
||||
|
||||
/** A no-op: SQLite has one connection and one writer, so its transactions already run one at a time. */
|
||||
serializeWrites(_em: EntityManager): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* The SQLite engine's whole-file maintenance: the two statements a backup,
|
||||
* a restore and the demo reset need that are about the database file rather
|
||||
* than any table (`PRAGMA wal_checkpoint`, `VACUUM INTO`). Neither has a
|
||||
* Postgres spelling, so they live under the engine that owns the file instead
|
||||
* of in `MaintenanceRepository`, whose remaining statements run on every
|
||||
* engine (and which the Postgres probe calls method by method).
|
||||
*
|
||||
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
|
||||
* factory over the context-resolving global `EntityManager`, and injected by
|
||||
* the SQLite backup port (`SqliteDatabaseBackup`). Free of Nest decorators
|
||||
* like every other file under `db/`, so a test can build one over its own
|
||||
* `EntityManager`.
|
||||
*/
|
||||
export class SqliteMaintenance {
|
||||
constructor(private readonly em: EntityManager) {}
|
||||
|
||||
/**
|
||||
* MikroORM's own validating `getContext()`: throws outside a request
|
||||
* context, the fail-closed check `TrekRepository` applies to every
|
||||
* repository, since a raw `connection.execute()` has no such guard.
|
||||
*/
|
||||
private validateRequestContext(): void {
|
||||
this.em.getContext();
|
||||
}
|
||||
|
||||
/**
|
||||
* `PRAGMA wal_checkpoint(TRUNCATE)`: flushes the WAL into the main file
|
||||
* before a snapshot or a swap. Reached through the backup port's
|
||||
* `checkpoint()`; every caller treats it as best effort.
|
||||
*/
|
||||
async walCheckpoint(): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/**
|
||||
* `` VACUUM INTO '<path>' ``: a consistent point-in-time copy of the file
|
||||
* under concurrent writers, reached through the backup port's
|
||||
* `snapshot()`. Single quotes in `path` are escaped by doubling.
|
||||
*/
|
||||
async vacuumInto(path: string): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
const escaped = path.replaceAll("'", "''");
|
||||
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { columnNamesSql, tableExistsSql } from './dialect/catalog';
|
||||
import { TableNotFoundException } from '@mikro-orm/core';
|
||||
import type { Migration } from '@mikro-orm/migrations';
|
||||
|
||||
/**
|
||||
@@ -8,7 +10,7 @@ import type { Migration } from '@mikro-orm/migrations';
|
||||
* a migration class.
|
||||
*/
|
||||
|
||||
/** A single `pragma_table_info` row — only the column name is ever read. */
|
||||
/** A catalog row (`dialect/catalog.ts`): only the name is ever read. */
|
||||
interface ColumnRow {
|
||||
name: string;
|
||||
}
|
||||
@@ -30,21 +32,18 @@ export async function addColumnIfMissing(
|
||||
column: string,
|
||||
definition: string,
|
||||
): Promise<void> {
|
||||
const existing = (await migration.execute('select name from pragma_table_info(?) where name = ?', [
|
||||
table,
|
||||
column,
|
||||
])) as ColumnRow[];
|
||||
if (existing.length > 0) return;
|
||||
await migration.execute(`alter table \`${table}\` add column ${definition}`);
|
||||
if ((await columnNames(migration, table)).has(column)) return;
|
||||
const platform = migration.getEntityManager().getPlatform();
|
||||
await migration.execute(`alter table ${platform.quoteIdentifier(table)} add column ${definition}`);
|
||||
}
|
||||
|
||||
/** True when `table` exists in the current database. */
|
||||
export async function tableExists(migration: Migration, table: string): Promise<boolean> {
|
||||
const rows = await migration.execute(`select name from sqlite_master where type = 'table' and name = ?`, [table]);
|
||||
const rows = await migration.execute(tableExistsSql(migration.getEntityManager().getPlatform()), [table]);
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/** The `CREATE TABLE` DDL SQLite stored for `table`, or null when it does not exist. */
|
||||
/** The `CREATE TABLE` DDL SQLite stored for `table`, or null when it does not exist. SQLite only: Postgres keeps no such text. */
|
||||
export async function tableSql(migration: Migration, table: string): Promise<string | null> {
|
||||
const rows = (await migration.execute(`select sql from sqlite_master where name = ?`, [table])) as {
|
||||
sql: string | null;
|
||||
@@ -68,7 +67,8 @@ export async function execBestEffort(migration: Migration, sql: string, params?:
|
||||
|
||||
/** The column names of `table`, or an empty set when the table does not exist. */
|
||||
export async function columnNames(migration: Migration, table: string): Promise<Set<string>> {
|
||||
const rows = (await migration.execute('select name from pragma_table_info(?)', [table])) as ColumnRow[];
|
||||
const sql = columnNamesSql(migration.getEntityManager().getPlatform());
|
||||
const rows = (await migration.execute(sql, [table])) as ColumnRow[];
|
||||
return new Set(rows.map((r) => r.name));
|
||||
}
|
||||
|
||||
@@ -80,10 +80,20 @@ export async function execUnlessTableMissing(migration: Migration, sql: string,
|
||||
try {
|
||||
await migration.execute(sql, params);
|
||||
} catch (err: unknown) {
|
||||
if (!(err instanceof Error) || !err.message.includes('no such table')) throw err;
|
||||
if (!isMissingTableError(err)) throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a statement failed because its table does not exist. The ORM maps
|
||||
* SQLite's "no such table" and Postgres's SQLSTATE 42P01 to
|
||||
* `TableNotFoundException`; the raw SQLite message is checked too, for an error
|
||||
* that reached the caller unconverted.
|
||||
*/
|
||||
export function isMissingTableError(err: unknown): boolean {
|
||||
return err instanceof TableNotFoundException || (err instanceof Error && err.message.includes('no such table'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a group of statements the legacy step wrapped in a single swallowing
|
||||
* try/catch. A failure part-way through leaves the earlier statements applied,
|
||||
|
||||
+15
-35
@@ -1,4 +1,3 @@
|
||||
import mikroOrmConfig from '../mikro-orm.config';
|
||||
import { withRequestContext } from '../nest/database/request-context';
|
||||
import { runDemoSeed } from './database';
|
||||
import { ensureEmailCaseIndex } from './email-case-index';
|
||||
@@ -6,16 +5,11 @@ import { migrateToHead, NO_SAFETY_NET } from './legacy-baseline';
|
||||
import { preMigrateSnapshot } from './pre-migrate-snapshot';
|
||||
import type { AnyEntity, EntityClass, EntityManager, EntitySchema, IDatabaseDriver, MikroORM } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { MikroORM as SqliteMikroORM } from '@mikro-orm/sqlite';
|
||||
|
||||
/**
|
||||
* Schema ownership at startup.
|
||||
*
|
||||
* This replaces `createTables()` → `runMigrations()` → `runSeeds()`, which used
|
||||
* to run synchronously inside `database.ts::initDb()` at module load. The
|
||||
* migrator is async, so the work moved to `buildApp()`, which is the first place
|
||||
* in the boot with an await and still runs before anything reads the database
|
||||
* (`bootstrap.ts` resolves SettingsService right after).
|
||||
* Schema ownership at startup. The migrator is async, so the schema is brought
|
||||
* to head from `DatabaseLifecycle.open()`, which `buildApp()` calls before
|
||||
* anything reads the database. Nothing runs at module load.
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -35,10 +29,10 @@ export type AnyOrm = MikroORM<
|
||||
* Migrate to head, then seed. Safe to call again, both halves are idempotent.
|
||||
*
|
||||
* `snapshot: false` skips the pre-migration copy (`pre-migrate-snapshot.ts`).
|
||||
* Only the restore hook passes it: the file it migrates was just unpacked from
|
||||
* a backup archive, which is still there to go back to.
|
||||
* Only the reopen after a restore passes it: the file it migrates was just
|
||||
* unpacked from a backup archive, which is still there to go back to.
|
||||
*/
|
||||
export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { snapshot?: boolean } = {}): Promise<void> {
|
||||
export async function runSchemaBootstrap(orm: AnyOrm, { snapshot }: { snapshot: boolean }): Promise<void> {
|
||||
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
|
||||
const connection = orm.em.getConnection();
|
||||
// A database the retired positional runner migrated has no migrator rows yet;
|
||||
@@ -52,8 +46,16 @@ export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { sna
|
||||
undefined,
|
||||
snapshot ? preMigrateSnapshot(connection) : NO_SAFETY_NET,
|
||||
);
|
||||
await finishSchemaBootstrap(orm);
|
||||
}
|
||||
|
||||
/**
|
||||
* What every engine does once its schema is at head: the deferred index, the
|
||||
* seeders, the demo seed. Idempotent, like the migration step before it.
|
||||
*/
|
||||
export async function finishSchemaBootstrap(orm: AnyOrm): Promise<void> {
|
||||
// The one index a migration may have had to defer (see email-case-index.ts).
|
||||
await ensureEmailCaseIndex(connection);
|
||||
await ensureEmailCaseIndex(orm.em.getConnection());
|
||||
|
||||
const defaultSeeder = orm.config.get('seeder').defaultSeeder;
|
||||
if (defaultSeeder) await orm.seeder.seedString(defaultSeeder);
|
||||
@@ -77,25 +79,3 @@ export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { sna
|
||||
// fully-synchronous version had by construction.
|
||||
await withRequestContext(orm, () => runDemoSeed());
|
||||
}
|
||||
|
||||
/**
|
||||
* An ORM instance outside Nest, for the vitest global setup that builds the
|
||||
* schema snapshot. Production uses the container-owned one from
|
||||
* `MikroOrmModule.forRoot()` instead of creating a second.
|
||||
*/
|
||||
export function createStandaloneOrm(): Promise<AnyOrm> {
|
||||
return SqliteMikroORM.init({
|
||||
...mikroOrmConfig,
|
||||
// Entity discovery is skipped deliberately. Migrations and seeders are raw
|
||||
// SQL and need no metadata, and the global setup that calls this runs as ESM
|
||||
// in vitest's main process, where discovering `*.entity.ts` fails on Node's
|
||||
// extensionless import resolution.
|
||||
entities: [],
|
||||
entitiesTs: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
// The migrator's per-migration progress log is production boot output; the
|
||||
// global setup already reports the snapshot in one line of its own, and a
|
||||
// failing migration still throws.
|
||||
migrations: { ...mikroOrmConfig.migrations, silent: true },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -25,11 +25,10 @@ import path from 'node:path';
|
||||
* migrated; `TREK_DB_PRE_MIGRATE_SNAPSHOT=false` is the way past that.
|
||||
*/
|
||||
|
||||
const PREFIX = 'pre-migrate-';
|
||||
const SNAPSHOT_FILE = /^pre-migrate-.+\.db$/;
|
||||
export const PREFIX = 'pre-migrate-';
|
||||
|
||||
/** `20261008T091500Z`: sortable, and free of the colons Windows refuses in a file name. */
|
||||
function stamp(now: Date): string {
|
||||
export function stamp(now: Date): string {
|
||||
return now
|
||||
.toISOString()
|
||||
.replace(/\.\d+Z$/, 'Z')
|
||||
@@ -51,11 +50,11 @@ async function holdsAnything(connection: Connection): Promise<boolean> {
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/** The snapshots in `dir`, newest first. */
|
||||
export function listSnapshots(dir: string): string[] {
|
||||
/** The snapshots in `dir`, newest first: the SQLite file's (`.db`) or a Postgres engine's (`.pgdump`). */
|
||||
export function listSnapshots(dir: string, extension = '.db'): string[] {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir).filter((entry) => SNAPSHOT_FILE.test(entry));
|
||||
entries = fs.readdirSync(dir).filter((entry) => entry.startsWith(PREFIX) && entry.endsWith(extension));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
@@ -66,8 +65,8 @@ export function listSnapshots(dir: string): string[] {
|
||||
}
|
||||
|
||||
/** Deletes all but the newest `keep`. A failure only warns: the new snapshot is already safe. */
|
||||
export function pruneSnapshots(dir: string, keep: number): void {
|
||||
for (const file of listSnapshots(dir).slice(keep)) {
|
||||
export function pruneSnapshots(dir: string, keep: number, extension = '.db'): void {
|
||||
for (const file of listSnapshots(dir, extension).slice(keep)) {
|
||||
try {
|
||||
fs.rmSync(file, { force: true });
|
||||
} catch (err) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { coalesceOverride, columnRef } from '../dialect/sql-functions';
|
||||
import type { DayAccommodations } from '../entities/DayAccommodations.entity';
|
||||
import type { DB } from '../kysely/db';
|
||||
import { accommodationIdMatches } from './_shared/reservation-accommodation';
|
||||
import { type AssertRowKeys } from './_shared/rows';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
|
||||
@@ -249,10 +250,13 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
|
||||
* parity keeps the duplicates, no `DISTINCT`.
|
||||
*/
|
||||
async listForTripWithPlaceAndBooking(trip_id: number | string): Promise<DayAccommodationListRow[]> {
|
||||
const platform = this.getEntityManager().getPlatform();
|
||||
const rows = await this.kysely<DayAccommodationListKyselyDB>()
|
||||
.selectFrom('day_accommodations as a')
|
||||
.leftJoin('places as p', 'p.id', 'a.place_id')
|
||||
.leftJoin('reservations as r', 'r.accommodation_id', 'a.id')
|
||||
.leftJoin('reservations as r', (join) =>
|
||||
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'a.id')),
|
||||
)
|
||||
.selectAll('a')
|
||||
.select([
|
||||
'p.name as place_name',
|
||||
@@ -469,6 +473,7 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
|
||||
* this table (see {@link DayAccommodationListKyselyDB}).
|
||||
*/
|
||||
async listRoadtripStays(trip_id: number): Promise<RoadtripStayRow[]> {
|
||||
const platform = this.getEntityManager().getPlatform();
|
||||
const rows = await this.kysely<RoadtripStaysKyselyDB>()
|
||||
.selectFrom('day_accommodations as a')
|
||||
.leftJoin('places as p', 'p.id', 'a.place_id')
|
||||
@@ -485,7 +490,7 @@ export class DayAccommodationsRepository extends TrekRepository<DayAccommodation
|
||||
eb
|
||||
.selectFrom('reservations as r')
|
||||
.select((eb2) => eb2.fn.min<number | null>('r.id').as('m'))
|
||||
.whereRef('r.accommodation_id', '=', 'a.id')
|
||||
.where((eb2) => accommodationIdMatches(platform, eb2, 'r.accommodation_id', 'a.id'))
|
||||
.as('reservation_id'),
|
||||
])
|
||||
.where('a.trip_id', '=', trip_id)
|
||||
|
||||
@@ -43,7 +43,9 @@ export class GoogleApiUsageRepository extends TrekRepository<GoogleApiUsage> {
|
||||
await this.kysely<GoogleApiUsageKyselyDB>()
|
||||
.insertInto('google_api_usage')
|
||||
.values({ day, calls: 1 })
|
||||
.onConflict((oc) => oc.column('day').doUpdateSet({ calls: (eb) => eb('calls', '+', 1) }))
|
||||
// The target row's column is qualified: Postgres reads a bare name as
|
||||
// ambiguous between it and `excluded` (42702).
|
||||
.onConflict((oc) => oc.column('day').doUpdateSet({ calls: (eb) => eb(eb.ref('google_api_usage.calls'), '+', 1) }))
|
||||
.execute();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,27 +1,18 @@
|
||||
import { isMissingTableError } from '../migration-utils';
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* Plan 3i Task 3 — R1's raw-connection exception, named ahead of this plan by
|
||||
* the program brief's own rule 4 ("connection.execute() (only in
|
||||
* MaintenanceRepository/DemoRepository, with a comment)").
|
||||
*
|
||||
* Not a `TrekRepository<Entity>` — it backs no table and no entity of its
|
||||
* own. It exists purely as the one place two whole-database-file statements
|
||||
* (a `PRAGMA` and a `VACUUM INTO`) run against the ORM's own bound driver
|
||||
* connection instead of scattering raw `better-sqlite3` calls across
|
||||
* `backup.impl.ts`/`demo-reset.ts`. `find/findOne → create/assign/flush →
|
||||
* nativeUpdate/nativeDelete → em.upsert → qb() → em.getKysely()` all fail
|
||||
* for these two statements — neither is expressible against an entity at
|
||||
* all — so `connection.execute()` (the last tier in the program's query-API
|
||||
* order) is the correct tool, not a shortcut around it.
|
||||
* Raw-connection statements that run on every engine and that no entity can
|
||||
* express: the readiness ping and the plugin-table erasure. Not a
|
||||
* `TrekRepository<Entity>`, since it backs no entity. The SQLite file's own
|
||||
* maintenance (`PRAGMA wal_checkpoint`, `VACUUM INTO`) lives in
|
||||
* `SqliteMaintenance`, under `db/engine/sqlite/`.
|
||||
*
|
||||
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
|
||||
* factory over the context-resolving global `EntityManager`; its consumers
|
||||
* inject it: the readiness probe, `UserCleanupService` and the SQLite backup
|
||||
* port (`SqliteDatabaseBackup`), which the backup, the restore and the demo
|
||||
* reset all go through. The class stays free of Nest decorators like every
|
||||
* other file under `db/`, so a test can still build one over its own
|
||||
* `EntityManager`.
|
||||
* inject it: the readiness probe and `UserCleanupService`. The class stays
|
||||
* free of Nest decorators like every other file under `db/`, so a test can
|
||||
* still build one over its own `EntityManager`.
|
||||
*/
|
||||
export class MaintenanceRepository {
|
||||
constructor(private readonly em: EntityManager) {}
|
||||
@@ -30,7 +21,7 @@ export class MaintenanceRepository {
|
||||
* MikroORM's own validating `getContext()` — throws
|
||||
* `ValidationError.cannotUseGlobalContext()` outside a request context (the
|
||||
* production default), the same fail-closed ratchet `TrekRepository`
|
||||
* applies to every entity-bound repository. Called first by both methods
|
||||
* applies to every entity-bound repository. Called first by every method
|
||||
* below, since raw `connection.execute()` has no such guard of its own.
|
||||
*/
|
||||
private validateRequestContext(): void {
|
||||
@@ -44,69 +35,27 @@ export class MaintenanceRepository {
|
||||
}
|
||||
|
||||
/**
|
||||
* `PRAGMA wal_checkpoint(TRUNCATE)` — rendered text pinned, identical to
|
||||
* the legacy `db.exec('PRAGMA wal_checkpoint(TRUNCATE)')` it replaces.
|
||||
* Reached through the backup port's `checkpoint()`, which the backup
|
||||
* (BK1) and the demo reset (DMR3) call. Flushes the WAL into the main db
|
||||
* file before a snapshot or a swap; every caller treats it as best effort.
|
||||
*/
|
||||
async walCheckpoint(): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/**
|
||||
* `` VACUUM INTO '<path>' `` — rendered text pinned, identical to the
|
||||
* legacy `` db.exec(`VACUUM INTO '${dbSnap.replaceAll("'", "''")}'`) ``
|
||||
* (BK2, reached through the backup port's `snapshot()`): single quotes in `path` are
|
||||
* escaped by doubling, the same escaping the legacy statement used — a
|
||||
* consistent point-in-time snapshot under concurrent writers, with no
|
||||
* MikroORM/Kysely equivalent.
|
||||
*/
|
||||
async vacuumInto(path: string): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
const escaped = path.replaceAll("'", "''");
|
||||
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plan 4 Task 4 — UC1 (`UserCleanupService.erasePluginUserData`): a
|
||||
* best-effort per-user delete across the three plugin-held tables
|
||||
* (`plugin_user_config`/`plugin_oauth_tokens`/`plugin_oauth_state`). These
|
||||
* are `nest/plugins`' own tables (Plan 3b Task 5's "stays raw" ruling,
|
||||
* narrowed by Plan 4 Task 8a to just this trio — UC2/UC3 already moved to
|
||||
* `PluginsRepository`/`PluginUserErasureQueueRepository`), not an entity
|
||||
* managed by any repository, so `connection.execute()` (rule 4's last
|
||||
* tier) is the correct tool here, same shape as `walCheckpoint`/
|
||||
* `vacuumInto` above — with ONE difference those two don't need: this is
|
||||
* the first `MaintenanceRepository` caller ever invoked from INSIDE an
|
||||
* open `UnitOfWork.transactional(...)` (`deleteUserCompletely`'s
|
||||
* transaction). `Connection#execute(query, params, method, ctx)`'s `ctx`
|
||||
* defaults to the plain (non-transactional) Kysely client when omitted —
|
||||
* `walCheckpoint`/`vacuumInto` always omit it because their callers
|
||||
* (the backup port) never run inside a transaction, but
|
||||
* omitting it here deadlocked: this repo's `better-sqlite3` connection is
|
||||
* single-connection/single-writer, the open transaction holds it for its
|
||||
* whole lifetime, and a bare `execute()` tries to check out that SAME
|
||||
* connection fresh instead of reusing the held one — a checkout that only
|
||||
* frees once the transaction (which is itself awaiting THIS call)
|
||||
* completes. `em.getTransactionContext()` returns the open transaction's
|
||||
* driver handle when called from inside one (resolved the same
|
||||
* `TransactionContext`-async-local-storage way `getContext()` is, so
|
||||
* `this.em` need not be the transactional fork itself) and `undefined`
|
||||
* otherwise, so passing it through is correct both inside and outside a
|
||||
* transaction. Each statement is independently best-effort: a
|
||||
* slimmed-down test schema may not carry one or more of these tables at
|
||||
* all, matching the legacy per-table try/catch exactly.
|
||||
* The per-user erasure of the three plugin-held tables (`plugin_user_config`,
|
||||
* `plugin_oauth_tokens`, `plugin_oauth_state`), called inside the erasure's
|
||||
* own `UnitOfWork.transactional(...)`.
|
||||
*
|
||||
* A table that does not exist (a slimmed-down test schema) is skipped; any
|
||||
* other failure propagates. Each statement runs in its own savepoint, since
|
||||
* on Postgres a failed statement aborts the whole transaction it is in.
|
||||
* `em.getTransactionContext()` hands the statement the open transaction's
|
||||
* connection, which a bare `execute()` would wait on forever.
|
||||
*/
|
||||
async deletePluginUserData(userId: number): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
const ctx = this.em.getTransactionContext();
|
||||
for (const table of ['plugin_user_config', 'plugin_oauth_tokens', 'plugin_oauth_state']) {
|
||||
try {
|
||||
await this.em.getConnection().execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', ctx);
|
||||
} catch {
|
||||
/* table absent (slim schema) */
|
||||
await this.em.transactional((em) =>
|
||||
em
|
||||
.getConnection()
|
||||
.execute(`DELETE FROM ${table} WHERE user_id = ?`, [userId], 'run', em.getTransactionContext()),
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
if (!isMissingTableError(err)) throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { Places } from '../entities/Places.entity';
|
||||
import { wrapOptionalLongitude } from '@trek/shared';
|
||||
import { absDifference, coalesceParam, columnRef, currentTimestamp, lowerTrim } from '../dialect/sql-functions';
|
||||
import { absDifference, coalesceParam, columnRef, currentTimestamp, ilike, lowerTrim } from '../dialect/sql-functions';
|
||||
import { type AssertRowKeys } from './_shared/rows';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
import { tourPlaceTextPatch, type TourPlaceWrite } from './tour-place-write';
|
||||
@@ -659,8 +659,8 @@ export class PlacesRepository extends TrekRepository<Places> {
|
||||
*/
|
||||
async listForTrip(trip_id: string, filters: {
|
||||
searchPattern?: string;
|
||||
category?: string;
|
||||
tag?: string;
|
||||
category?: number;
|
||||
tag?: number;
|
||||
assignment?: 'all' | 'unassigned' | 'assigned';
|
||||
}): Promise<PlaceListRow[]> {
|
||||
const platform = this.getEntityManager().getPlatform();
|
||||
@@ -680,15 +680,13 @@ export class PlacesRepository extends TrekRepository<Places> {
|
||||
.where('p.trip_id = ?', [trip_id]);
|
||||
|
||||
if (filters.searchPattern) {
|
||||
qb.andWhere(
|
||||
"(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\' OR p.description LIKE ? ESCAPE '\\')",
|
||||
[filters.searchPattern, filters.searchPattern, filters.searchPattern],
|
||||
);
|
||||
const fields = ['p.name', 'p.address', 'p.description'];
|
||||
qb.andWhere(ilike(platform, fields), fields.map(() => filters.searchPattern));
|
||||
}
|
||||
if (filters.category) {
|
||||
if (filters.category !== undefined) {
|
||||
qb.andWhere('p.category_id = ?', [filters.category]);
|
||||
}
|
||||
if (filters.tag) {
|
||||
if (filters.tag !== undefined) {
|
||||
qb.andWhere('p.id IN (SELECT place_id FROM place_tags WHERE tag_id = ?)', [filters.tag]);
|
||||
}
|
||||
if (filters.assignment === 'unassigned') {
|
||||
|
||||
@@ -52,10 +52,14 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
|
||||
* position) SELECT r.id, d.id, ? FROM reservations r JOIN days d ON
|
||||
* d.trip_id = r.trip_id WHERE r.id = ? AND d.id = ? AND r.trip_id = ?`.
|
||||
*
|
||||
* Kysely `insertInto().orReplace().columns().expression(selectFrom…)`
|
||||
* (the MikroORM QueryBuilder's `insertFrom` has no `OR REPLACE` flag —
|
||||
* `.onConflict().merge()` is semantically different, a real UPDATE rather
|
||||
* than REPLACE's delete-then-insert). The trip scoping lives IN the
|
||||
* Kysely `insertInto().columns().expression(selectFrom…)` with an
|
||||
* `ON CONFLICT (reservation_id, day_id) DO UPDATE SET position`, which both
|
||||
* engines read (`INSERT OR REPLACE` is SQLite only). The pair is the
|
||||
* table's whole primary key and `position` its only other column, so the
|
||||
* update leaves the same row REPLACE's delete-then-insert did, and no
|
||||
* foreign key points at this table for the delete to have cascaded to.
|
||||
* The SELECT keeps its WHERE, which SQLite needs to read the trailing
|
||||
* ON CONFLICT as the upsert clause. The trip scoping lives IN the
|
||||
* SELECT (a foreign `reservation_id`/`day_id` materialises zero rows to
|
||||
* insert, never a foreign-key error) — preserved exactly, not hoisted
|
||||
* into a pre-check.
|
||||
@@ -68,7 +72,6 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
|
||||
): Promise<void> {
|
||||
await this.kysely<ReservationDayPositionsKyselyDB>()
|
||||
.insertInto('reservation_day_positions')
|
||||
.orReplace()
|
||||
.columns(['reservation_id', 'day_id', 'position'])
|
||||
.expression((eb) =>
|
||||
eb
|
||||
@@ -79,6 +82,9 @@ export class ReservationDayPositionsRepository extends TrekRepository<Reservatio
|
||||
.where('d.id', '=', day_id)
|
||||
.where('r.trip_id', '=', trip_id as number),
|
||||
)
|
||||
.onConflict((oc) =>
|
||||
oc.columns(['reservation_id', 'day_id']).doUpdateSet((eb) => ({ position: eb.ref('excluded.position') })),
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@ import type { Reservations } from '../entities/Reservations.entity';
|
||||
import { DayAssignments } from '../entities/DayAssignments.entity';
|
||||
import { Days } from '../entities/Days.entity';
|
||||
import { coalesceOverride, columnRef, dayDistance } from '../dialect/sql-functions';
|
||||
import { castIntegerKysely, concatKysely, startsWithIsoDateKysely, substringKysely } from '../dialect/kysely-functions';
|
||||
import { concatKysely, startsWithIsoDateKysely, substringKysely } from '../dialect/kysely-functions';
|
||||
import { accommodationIdMatches } from './_shared/reservation-accommodation';
|
||||
import { publicReservationExpr, publicStayExists, type ReservationVisibilityKyselyDB } from './_shared/reservation-visibility';
|
||||
import type { DayAssignmentRow } from './day-assignment-rows';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
@@ -32,18 +33,6 @@ export type {
|
||||
PublicApiUnscheduledReservationRow,
|
||||
};
|
||||
|
||||
/**
|
||||
* Kysely typing for the RS18/RR1 join (`joinedQuery` below) — `reservations
|
||||
* .accommodation_id` is a bare `p.text().nullable()` column with no FK to
|
||||
* `day_accommodations` (§18.1 of the inventory), so the QueryBuilder's
|
||||
* relation-path `.join()` cannot express `LEFT JOIN day_accommodations ap ON
|
||||
* r.accommodation_id = ap.id` at all — the same escape hatch
|
||||
* `DayAssignmentsRepository.effectiveStart`/`TripsRepository.tripSelectQuery`
|
||||
* use. The `id`/`trip_id` the callers pass stay `number | string` (raw-bind,
|
||||
* D4's T5 seam) and are cast to the column type only at the bind, so this
|
||||
* join keeps the legacy statement's exact raw-bind flexibility.
|
||||
*/
|
||||
|
||||
/**
|
||||
* `reservations` as the two aliases below bind it: `accommodation_id` is a
|
||||
* TEXT column, widened to `string | number | null` so the plain `number`
|
||||
@@ -364,7 +353,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
findPassContext(id: number, tripId: number) { return passContextQuery(this.kysely<PassContextDB>(), id, tripId); }
|
||||
|
||||
private joinedQuery() {
|
||||
return joinedReservationQuery(this.kysely<ReservationJoinDB>());
|
||||
return joinedReservationQuery(this.kysely<ReservationJoinDB>(), this.getEntityManager().getPlatform());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -875,10 +864,13 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
|
||||
/** CL2 — see {@link CalendarReservationRow}'s docstring. */
|
||||
async listForCalendar(trip_id: number | string): Promise<CalendarReservationRow[]> {
|
||||
const platform = this.getEntityManager().getPlatform();
|
||||
const rows = await this.kysely<CalendarReservationKyselyDB>()
|
||||
.selectFrom('reservations as r')
|
||||
.leftJoin('places as pl', 'pl.id', 'r.place_id')
|
||||
.leftJoin('day_accommodations as a', 'a.id', 'r.accommodation_id')
|
||||
.leftJoin('day_accommodations as a', (join) =>
|
||||
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'a.id')),
|
||||
)
|
||||
.leftJoin('days as sd', 'sd.id', 'a.start_day_id')
|
||||
.leftJoin('days as ed', 'ed.id', 'a.end_day_id')
|
||||
.leftJoin('days as rd', 'rd.id', 'r.day_id')
|
||||
@@ -894,7 +886,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
eb
|
||||
.selectFrom('reservations as r2')
|
||||
.select((eb2) => eb2.fn.min<number | null>('r2.id').as('c'))
|
||||
.whereRef('r2.accommodation_id', '=', 'a.id')
|
||||
.where((eb2) => accommodationIdMatches(platform, eb2, 'r2.accommodation_id', 'a.id'))
|
||||
.as('stay_first_reservation_id'),
|
||||
'rd.date as day_date',
|
||||
'red.date as end_day_date',
|
||||
@@ -932,6 +924,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
|
||||
/** CL7 — see {@link CalendarStayRow}'s docstring. */
|
||||
async listPublicStaysForCalendar(trip_id: number | string): Promise<CalendarStayRow[]> {
|
||||
const platform = this.getEntityManager().getPlatform();
|
||||
const rows = await this.kysely<CalendarStayKyselyDB>()
|
||||
.selectFrom('day_accommodations as a')
|
||||
.where('a.trip_id', '=', trip_id as number)
|
||||
@@ -953,7 +946,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
eb
|
||||
.selectFrom('reservations as r')
|
||||
.select('r.title')
|
||||
.whereRef('r.accommodation_id', '=', 'a.id')
|
||||
.where((eb2) => accommodationIdMatches(platform, eb2, 'r.accommodation_id', 'a.id'))
|
||||
.where((eb2) => publicReservationExpr(eb2, 'r.ingest_state'))
|
||||
.orderBy('r.id', 'asc')
|
||||
.limit(1)
|
||||
@@ -1039,7 +1032,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
eb
|
||||
.selectFrom('reservations as res')
|
||||
.select('res.title')
|
||||
.where((eb2) => eb2(castIntegerKysely(platform, eb2, 'res.accommodation_id'), '=', eb2.ref('a.id')))
|
||||
.where((eb2) => accommodationIdMatches(platform, eb2, 'res.accommodation_id', 'a.id'))
|
||||
.where('res.status', '!=', 'cancelled')
|
||||
.orderBy('res.id', 'asc')
|
||||
.limit(1),
|
||||
@@ -1080,7 +1073,7 @@ export class ReservationsRepository extends TrekRepository<Reservations> {
|
||||
eb
|
||||
.selectFrom('reservations as res')
|
||||
.select('res.title')
|
||||
.where((eb2) => eb2(castIntegerKysely(platform, eb2, 'res.accommodation_id'), '=', eb2.ref('a.id')))
|
||||
.where((eb2) => accommodationIdMatches(platform, eb2, 'res.accommodation_id', 'a.id'))
|
||||
.where('res.status', '!=', 'cancelled')
|
||||
.orderBy('res.id', 'asc')
|
||||
.limit(1),
|
||||
|
||||
@@ -80,12 +80,14 @@ export class RouteUsageDailyRepository extends TrekRepository<RouteUsageDaily> {
|
||||
await this.kysely<RouteUsageDailyKyselyDB>()
|
||||
.insertInto('route_usage_daily')
|
||||
.values(entry)
|
||||
// The target row's columns are qualified: Postgres reads a bare name as
|
||||
// ambiguous between it and `excluded` (42702).
|
||||
.onConflict((oc) =>
|
||||
oc.columns(['day', 'profile', 'surface', 'self_hosted']).doUpdateSet({
|
||||
requests: (eb) => eb('requests', '+', eb.ref('excluded.requests')),
|
||||
waypoints: (eb) => eb('waypoints', '+', eb.ref('excluded.waypoints')),
|
||||
km: (eb) => eb('km', '+', eb.ref('excluded.km')),
|
||||
failed: (eb) => eb('failed', '+', eb.ref('excluded.failed')),
|
||||
requests: (eb) => eb(eb.ref('route_usage_daily.requests'), '+', eb.ref('excluded.requests')),
|
||||
waypoints: (eb) => eb(eb.ref('route_usage_daily.waypoints'), '+', eb.ref('excluded.waypoints')),
|
||||
km: (eb) => eb(eb.ref('route_usage_daily.km'), '+', eb.ref('excluded.km')),
|
||||
failed: (eb) => eb(eb.ref('route_usage_daily.failed'), '+', eb.ref('excluded.failed')),
|
||||
}),
|
||||
)
|
||||
.execute();
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { QueryFlag } from '@mikro-orm/core';
|
||||
import type { TTripsCostsMode, Trips } from '../entities/Trips.entity';
|
||||
import { coalesceParam, currentTimestamp, nowDateOffset } from '../dialect/sql-functions';
|
||||
import { coalesceParam, currentTimestamp, ilike, nowDateOffset } from '../dialect/sql-functions';
|
||||
import type { AssertRowKeys } from './_shared/rows';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
import { tripAccessExpr } from './_shared/trip-access';
|
||||
@@ -175,7 +175,7 @@ export class TripsRepository extends TrekRepository<Trips> {
|
||||
return await this.accessibleTripsQuery(user_id)
|
||||
.join('t.places_collection', 'p')
|
||||
.select(['p.trip', 'p.name'])
|
||||
.andWhere("(p.name LIKE ? ESCAPE '\\' OR p.address LIKE ? ESCAPE '\\')", [likePattern, likePattern])
|
||||
.andWhere(ilike(this.getEntityManager().getPlatform(), ['p.name', 'p.address']), [likePattern, likePattern])
|
||||
.orderBy([{ 'p.trip': 'asc' }, { 'p.name': 'asc' }])
|
||||
.limit(500)
|
||||
// The limit is on the joined place rows, like the legacy statement's.
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { castIntegerKysely } from '../../dialect/kysely-functions';
|
||||
import type { Platform } from '@mikro-orm/core';
|
||||
|
||||
import type { ExpressionBuilder, ExpressionWrapper, SqlBool, StringReference } from 'kysely';
|
||||
|
||||
/**
|
||||
* The join condition between a reservation and the stay it is booked against.
|
||||
*
|
||||
* `reservations.accommodation_id` is TEXT with no foreign key, while
|
||||
* `day_accommodations.id` is INTEGER, so the comparison is written as
|
||||
* `CAST(<reservation ref> AS INTEGER) = <stay ref>`. SQLite coerces a bare
|
||||
* text-to-integer comparison and Postgres refuses it (42883), so the cast is
|
||||
* needed on both engines. A stored `'14.0'` casts to 14 on both.
|
||||
*/
|
||||
export function accommodationIdMatches<DB, TB extends keyof DB>(
|
||||
platform: Platform,
|
||||
eb: ExpressionBuilder<DB, TB>,
|
||||
reservationAccommodationId: StringReference<DB, TB>,
|
||||
stayId: StringReference<DB, TB>,
|
||||
): ExpressionWrapper<DB, TB, SqlBool> {
|
||||
return eb(castIntegerKysely(platform, eb, reservationAccommodationId), '=', eb.ref(stayId).$castTo<number>());
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import type { DB } from '../../kysely/db';
|
||||
import { accommodationIdMatches } from './reservation-accommodation';
|
||||
import type { Platform } from '@mikro-orm/core';
|
||||
|
||||
import type { Kysely } from 'kysely';
|
||||
|
||||
@@ -9,14 +11,18 @@ export type ReservationJoinDB = Pick<DB, 'reservations' | 'days' | 'places' | 'd
|
||||
* A booking with its day number, its place's name and its stay's place and
|
||||
* times: the projection `ReservationsRepository.listForTrip` and
|
||||
* `findWithJoins` narrow with their own `WHERE`. Kept out of the repository
|
||||
* file, which is at its size baseline.
|
||||
* file, which is at its size baseline. `accommodation_id` is TEXT and `ap.id`
|
||||
* INTEGER, so the stay is matched through accommodationIdMatches: SQLite
|
||||
* coerces the bare comparison, Postgres refuses it.
|
||||
*/
|
||||
export function joinedReservationQuery(kysely: Kysely<ReservationJoinDB>) {
|
||||
export function joinedReservationQuery(kysely: Kysely<ReservationJoinDB>, platform: Platform) {
|
||||
return kysely
|
||||
.selectFrom('reservations as r')
|
||||
.leftJoin('days as d', 'd.id', 'r.day_id')
|
||||
.leftJoin('places as p', 'p.id', 'r.place_id')
|
||||
.leftJoin('day_accommodations as ap', 'ap.id', 'r.accommodation_id')
|
||||
.leftJoin('day_accommodations as ap', (join) =>
|
||||
join.on((eb) => accommodationIdMatches(platform, eb, 'r.accommodation_id', 'ap.id')),
|
||||
)
|
||||
.leftJoin('places as acc_p', 'acc_p.id', 'ap.place_id')
|
||||
.selectAll('r')
|
||||
.select([
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import mikroOrmConfig from '../mikro-orm.config';
|
||||
import type { AnyOrm } from './orm';
|
||||
import { SharedSqliteDriver } from './orm-driver';
|
||||
import { MikroORM as SqliteMikroORM } from '@mikro-orm/sqlite';
|
||||
|
||||
/**
|
||||
* An ORM instance outside Nest, for the vitest global setup that builds the
|
||||
* schema snapshot. Production uses the container-owned one from
|
||||
* `MikroOrmModule.forRoot()` instead of creating a second.
|
||||
*
|
||||
* SQLite only: the snapshot is the SQLite schema and the init below is the
|
||||
* SQLite driver's, so a config that names another engine is refused.
|
||||
*/
|
||||
export function createStandaloneOrm(): Promise<AnyOrm> {
|
||||
if (mikroOrmConfig.driver !== SharedSqliteDriver) {
|
||||
throw new Error('createStandaloneOrm builds the SQLite schema snapshot; unset TREK_DB_URL to run it.');
|
||||
}
|
||||
return SqliteMikroORM.init({
|
||||
...mikroOrmConfig,
|
||||
// Entity discovery is skipped deliberately. Migrations and seeders are raw
|
||||
// SQL and need no metadata, and the global setup that calls this runs as ESM
|
||||
// in vitest's main process, where discovering `*.entity.ts` fails on Node's
|
||||
// extensionless import resolution.
|
||||
entities: [],
|
||||
entitiesTs: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
// The migrator's per-migration progress log is production boot output; the
|
||||
// global setup already reports the snapshot in one line of its own, and a
|
||||
// failing migration still throws.
|
||||
migrations: { ...mikroOrmConfig.migrations, silent: true },
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user