feat(db): run TREK on PostgreSQL

TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before
otherwise: an engine port under src/db/engine, the dialect helpers under
src/db/dialect, a Postgres baseline for the migrations up to the cut,
pg_dump/pg_restore backups, a probe of every repository method against a
real server and a Postgres test lane in CI.
This commit is contained in:
Isaias Tavares
2026-11-05 01:15:21 -03:00
committed by Maurice
parent 17b47b38b9
commit 2e3762afe1
176 changed files with 6728 additions and 777 deletions
+49
View File
@@ -0,0 +1,49 @@
import type { EngineId } from '../../app-config/db-settings';
import type { AnyOrm } from '../orm';
import type { EntityManager, Options } from '@mikro-orm/core';
/**
* One database engine: what the ORM needs to reach it, how the connection
* opens, closes and comes back after a restore, and how a database is brought
* to the current schema. Code that differs by engine dispatches through this,
* or through the helpers in `db/dialect/`, never through its own platform
* check (`lint:engine-dispatch`).
*
* Free of Nest like every other file under `db/`: `src/db` does not import
* `src/nest` (`lint:boundaries`). `DatabaseLifecycle` is the provider that
* calls it.
*/
export interface DatabaseEngine {
readonly id: EngineId;
/** For log lines and errors: the file path, or the URL without its password. */
describe(): string;
/** What MikroORM needs to reach the database: the driver and its target, the pool, driver options. */
ormOptions(): Partial<Options>;
/**
* Called once at boot, before the schema bootstrap. `onReopened` runs after
* each later reopen, once the ORM is pointed at the reopened database: the
* engine does what its own reopen needs first (SQLite hands the ORM a new
* handle, so the ORM's connection is closed and connected again), then calls
* it, and the caller migrates the database and runs whatever follows.
*/
open(orm: AnyOrm, onReopened: () => Promise<void>): void;
/** Closes what the engine itself holds; the ORM's own pool is closed with the Nest container. */
close(): void;
/**
* Comes back after the database was replaced by a restore, then runs the
* `onReopened` that `open` registered. Throws when `open` has not run.
*/
reopen(): Promise<void>;
/**
* Brings the database to the current schema, then seeds it. `snapshot:
* false` skips the copy taken before pending migrations (a restore, whose
* archive is the copy).
*/
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void>;
/**
* Runs first in every `UnitOfWork` transaction, on its EntityManager: makes
* the transactions that write run one at a time, as SQLite's single writer
* does, so a `MAX(...) + 1` read inside one cannot race another.
*/
serializeWrites(em: EntityManager): Promise<void>;
}
+58
View File
@@ -0,0 +1,58 @@
import { readEnv } from '../../app-config';
import type { DbEngineSettings } from '../../app-config/db-settings';
import type { DbSettings } from '../../app-config/derive';
import { ALL_ENTITIES } from '../entities';
import type { DatabaseEngine } from './engine';
import { PostgresEngine } from './postgres/postgres-engine';
import { SqliteEngine } from './sqlite/sqlite-engine';
import { defineConfig, type Options } from '@mikro-orm/core';
import { Migrator } from '@mikro-orm/migrations';
import { SeedManager } from '@mikro-orm/seeder';
export type { DatabaseEngine } from './engine';
/**
* The engine this process runs on, from `TREK_DB_URL` (`readEnv().db`): the
* SQLite file unless the URL names a Postgres database. Read afresh on every
* call, like everything `readEnv()` answers, so nothing caches a choice a test
* or a restart changes.
*/
export function resolveEngine(db: DbEngineSettings = readEnv().db): DatabaseEngine {
if (db.engine === 'postgres') return new PostgresEngine(db);
return new SqliteEngine();
}
/**
* The Postgres URL and the directory of its client tools, when the settings
* name a Postgres database; null on SQLite. For the code that drives pg_dump
* and pg_restore, which needs both and must not read them from the environment
* again.
*/
export function postgresTarget(db: DbSettings): { url: string; binDir: string | undefined } | null {
if (db.engine !== 'postgres') return null;
return { url: db.url, binDir: db.pgBinDir };
}
/**
* The MikroORM config for `engine`: what every engine shares (entities,
* migrations, seeders) and what the engine adds. `src/mikro-orm.config.ts`
* exports it for the CLI and the Nest container, so both reach the same
* database.
*/
export function buildOrmConfig(engine: DatabaseEngine): Options {
return defineConfig({
entities: ALL_ENTITIES,
extensions: [Migrator, SeedManager],
migrations: {
path: 'dist/db/migrations',
pathTs: 'src/db/migrations',
snapshot: false,
},
seeder: {
path: 'dist/db/seeders',
pathTs: 'src/db/seeders',
defaultSeeder: 'DatabaseSeeder',
},
...engine.ormOptions(),
}) as Options;
}
+93
View File
@@ -0,0 +1,93 @@
import { splitStatements } from './sql-split';
import fs from 'node:fs';
import path from 'node:path';
/**
* Where the Postgres baseline lives: `server/assets/postgres/`, which the
* Docker image copies beside `dist/` (the same place the airports and atlas
* data are read from), reached the same way from `src/` and from `dist/`.
*/
const BASELINE_DIR = path.join(__dirname, '..', '..', '..', '..', 'assets', 'postgres');
const SQL_FILE = 'baseline.sql';
const MANIFEST_FILE = 'baseline.json';
/** The manifest beside the SQL: the last migration the baseline stands for, and every migration it replaces. */
interface BaselineManifest {
through: string;
migrations: string[];
}
/**
* The schema a fresh Postgres database starts from.
*
* The migrations up to `through` are written in SQLite and frozen, so a
* Postgres database does not replay them: it runs these statements, generated
* from the entities (`npm run gen:pg-baseline`), and records every migration
* in `migrations` as executed in the same transaction. Migrations after
* `through` run on both engines through the migrator.
*/
export interface PostgresBaseline extends BaselineManifest {
statements: string[];
}
/** The two files a baseline is written as, ready to compare or write. */
export function renderBaseline(baseline: PostgresBaseline): { sql: string; manifest: string } {
return {
sql:
'-- The schema a fresh Postgres database starts from. Generated by `npm run gen:pg-baseline`; do not edit.\n' +
baseline.statements.map((statement) => `${statement};\n`).join(''),
manifest: `${JSON.stringify({ through: baseline.through, migrations: baseline.migrations }, null, 2)}\n`,
};
}
/**
* Whether `data` is a manifest a baseline can be read from: a `through` naming
* one of a non-empty list of `migrations`, every entry a string.
*/
function isBaselineManifest(data: unknown): data is BaselineManifest {
if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false;
const { through, migrations } = data;
return (
typeof through === 'string' &&
Array.isArray(migrations) &&
migrations.length > 0 &&
migrations.every((name): name is string => typeof name === 'string') &&
migrations.includes(through)
);
}
function parseManifest(text: string, file: string): BaselineManifest {
const data: unknown = JSON.parse(text);
if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`);
return { through: data.through, migrations: data.migrations };
}
/** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */
export function readPostgresBaseline(dir = BASELINE_DIR): PostgresBaseline {
const manifestFile = path.join(dir, MANIFEST_FILE);
const manifest = parseManifest(fs.readFileSync(manifestFile, 'utf8'), manifestFile);
const statements = splitStatements(fs.readFileSync(path.join(dir, SQL_FILE), 'utf8').replace(/^--[^\n]*\n/gm, ''));
if (statements.length === 0) throw new Error(`${path.join(dir, SQL_FILE)} holds no statements`);
// The boot sends the baseline as one statement without parameters, where a `?` would be read as a binding.
if (statements.some((statement) => statement.includes('?'))) {
throw new Error(`${path.join(dir, SQL_FILE)} holds a "?", which the boot would read as a parameter`);
}
return { ...manifest, statements };
}
/** Writes the two files (the generator's `--write`). */
export function writePostgresBaseline(baseline: PostgresBaseline, dir = BASELINE_DIR): void {
const { sql, manifest } = renderBaseline(baseline);
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, SQL_FILE), sql);
fs.writeFileSync(path.join(dir, MANIFEST_FILE), manifest);
}
/** Whether the committed files are what `baseline` renders to (the generator's `--check`). */
export function baselineMatches(baseline: PostgresBaseline, dir = BASELINE_DIR): boolean {
const { sql, manifest } = renderBaseline(baseline);
const read = (file: string) =>
fs.existsSync(path.join(dir, file)) ? fs.readFileSync(path.join(dir, file), 'utf8') : null;
return read(SQL_FILE) === sql && read(MANIFEST_FILE) === manifest;
}
@@ -0,0 +1,41 @@
/**
* Drops every table, view, sequence, routine, collation and standalone type in
* the current schema, except an extension's, before a dump is loaded: a dump
* only recreates what it holds, so whatever a later migration added would
* otherwise survive the restore and break the migrations the reopen runs. It
* runs in the same transaction as the load (`pgRestore`), so a restore that
* fails leaves the database as it was.
*/
export const CLEAR_SCHEMA_SQL = `DO $clear$
DECLARE r record;
BEGIN
FOR r IN SELECT c.oid::regclass AS name, c.relkind FROM pg_class c
WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype IN ('e', 'a', 'i'))
LOOP
EXECUTE format('DROP %s IF EXISTS %s CASCADE',
CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' WHEN 'S' THEN 'SEQUENCE'
WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END, r.name);
END LOOP;
FOR r IN SELECT p.oid::regprocedure AS sig FROM pg_proc p
WHERE p.pronamespace = current_schema()::regnamespace
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
LOOP
EXECUTE format('DROP ROUTINE IF EXISTS %s CASCADE', r.sig);
END LOOP;
FOR r IN SELECT format('%I.%I', n.nspname, c.collname) AS name FROM pg_collation c
JOIN pg_namespace n ON n.oid = c.collnamespace
WHERE n.nspname = current_schema()
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype = 'e')
LOOP
EXECUTE format('DROP COLLATION IF EXISTS %s CASCADE', r.name);
END LOOP;
FOR r IN SELECT t.oid::regtype AS name FROM pg_type t
WHERE t.typnamespace = current_schema()::regnamespace AND t.typtype IN ('e', 'd', 'r')
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = t.oid AND d.deptype IN ('e', 'i'))
LOOP
EXECUTE format('DROP TYPE IF EXISTS %s CASCADE', r.name);
END LOOP;
END
$clear$;
`;
+119
View File
@@ -0,0 +1,119 @@
import { execFile } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { promisify } from 'node:util';
const execFileAsync = promisify(execFile);
/** The Postgres client tools a backup runs. */
export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
/**
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
* Debian's versioned directories newest first (the Docker image installs 17),
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
*/
const DEFAULT_BIN_DIRS = [
'/usr/lib/postgresql/17/bin',
'/usr/lib/postgresql/16/bin',
'/usr/lib/postgresql/15/bin',
'/usr/lib/postgresql/14/bin',
'/usr/local/bin',
'/usr/bin',
];
/** A dump or a restore of a large database takes minutes, not hours. */
const TOOL_TIMEOUT_MS = 30 * 60 * 1000;
const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
/** An unreachable host fails the tool in seconds instead of holding a boot or a restore. */
const CONNECT_TIMEOUT_SECONDS = '10';
/** How much of a tool's stderr an error keeps. */
const STDERR_LIMIT = 2000;
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
const dirs = binDir ? [binDir] : DEFAULT_BIN_DIRS;
const executable = (candidate: string): boolean => {
try {
fs.accessSync(candidate, fs.constants.X_OK);
return true;
} catch {
return false; // missing or not executable: the next directory may have it
}
};
return dirs.map((dir) => path.join(dir, tool)).find(executable) ?? null;
}
/**
* The libpq environment for `url`: the tools read the password from
* PGPASSWORD, so it never appears in an argument list another process can
* read. An sslmode that asks for SSL verifies the server's certificate, as the
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
* else the system's trusted CAs.
*/
export function pgConnectionEnv(url: string): Record<string, string> {
const parsed = new URL(url);
const env: Record<string, string> = {
PGHOST: parsed.hostname.replace(/^\[(.*)\]$/, '$1'),
PGPORT: parsed.port || '5432',
PGDATABASE: decodeURIComponent(parsed.pathname.replace(/^\//, '')),
PGCONNECT_TIMEOUT: CONNECT_TIMEOUT_SECONDS,
};
if (parsed.username) env.PGUSER = decodeURIComponent(parsed.username);
if (parsed.password) env.PGPASSWORD = decodeURIComponent(parsed.password);
const sslmode = parsed.searchParams.get('sslmode');
if (sslmode === 'disable') env.PGSSLMODE = 'disable';
else if (sslmode) {
env.PGSSLMODE = 'verify-full';
env.PGSSLROOTCERT = parsed.searchParams.get('sslrootcert') ?? 'system';
}
return env;
}
/** A client tool, refused with a clear message when it is not installed. */
export class PgToolMissingError extends Error {
constructor(tool: PgTool) {
super(
`${tool} was not found, so a Postgres database cannot be backed up or restored. ` +
'Install the PostgreSQL client tools (the Docker image has them) or point TREK_DB_PG_BIN_DIR at them.',
);
this.name = 'PgToolMissingError';
}
}
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
function summarizeStderr(stderr: string | undefined): string {
const kept = (stderr ?? '')
.split('\n')
.filter((line) => line.trim() !== '' && !DATA_LINE.test(line))
.join('\n');
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
}
/**
* Runs `tool` by its absolute path and resolves with its stdout. A failure
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
* the cause.
*/
export async function run(
tool: PgTool,
binDir: string | undefined,
args: string[],
env: Record<string, string>,
): Promise<string> {
const binary = resolvePgTool(tool, binDir);
if (!binary) throw new PgToolMissingError(tool);
try {
const { stdout } = await execFileAsync(binary, args, {
env,
timeout: TOOL_TIMEOUT_MS,
maxBuffer: TOOL_MAX_OUTPUT_BYTES,
});
return stdout;
} catch (err) {
const message = summarizeStderr((err as { stderr?: string }).stderr);
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
}
}
+206
View File
@@ -0,0 +1,206 @@
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
import { CLEAR_SCHEMA_SQL } from './clear-schema';
import { pgConnectionEnv, run } from './pg-process';
import { refuseToolsWithoutRestrict } from './restore-safety';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import readline from 'node:readline';
/**
* Restore's session settings: a restore waiting on a lock another session
* holds gives up, and rolls back, instead of hanging; the NOTICEs every
* CASCADE prints stay out of its stderr.
*/
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
function createPrivateFile(file: string): void {
fs.writeFileSync(file, '', { mode: 0o600 });
}
/** The schema TREK's tables live in: its name, and the name as SQL quotes it (`quote_ident`). */
interface PgSchema {
name: string;
quoted: string;
}
/**
* The current schema of the database at `url`, the one TREK creates its
* tables in. A backup and a restore stay inside it: a hosted database
* (Supabase) keeps the platform's own schemas, `auth` among them, beside it,
* and they are neither TREK's to copy nor to replace.
*/
async function pgCurrentSchema(url: string, binDir: string | undefined): Promise<PgSchema> {
const out = await run(
'psql',
binDir,
[
'--no-psqlrc',
'--no-align',
'--tuples-only',
"--command=select concat_ws(E'\\t', current_schema(), quote_ident(current_schema()))",
],
pgConnectionEnv(url),
);
const [name, quoted] = out.replace(/\n$/, '').split('\t');
if (!name || !quoted) {
throw new Error('The database has no current schema: set the search_path of the role in TREK_DB_URL.');
}
return { name, quoted };
}
/**
* Writes a custom-format dump of the current schema of the database at `url`
* to `target`, readable by its owner only. It is written beside the target and
* renamed into place, so a dump that fails or is killed never leaves a file
* that passes for a copy.
*/
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
const partial = `${target}.partial`;
try {
createPrivateFile(partial);
const schema = await pgCurrentSchema(url, binDir);
await run(
'pg_dump',
binDir,
[
'--format=custom',
'--no-owner',
'--no-privileges',
`--file=${partial}`,
`--schema="${schema.name.replaceAll('"', '""')}"`,
],
pgConnectionEnv(url),
);
fs.renameSync(partial, target);
} catch (err) {
fs.rmSync(partial, { force: true });
throw err;
}
}
/** Escapes every character a RegExp would read as syntax. */
function escapeRegExp(text: string): string {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
/**
* Whether the restore script creates TREK's migration table in `schema`: a
* dump of TREK's own schema does. The script is read line by line and the
* read stops at the first match: the COPY data that fills a large script is
* never loaded, and a script past V8's maximum string length would not load
* at all.
*/
async function scriptHoldsTrekSchema(script: string, schema: PgSchema): Promise<boolean> {
const createsHistory = new RegExp(`^CREATE TABLE ${escapeRegExp(`${schema.quoted}.mikro_orm_migrations`)} \\(`);
const input = fs.createReadStream(script, { encoding: 'utf8' });
const lines = readline.createInterface({ input, crlfDelay: Infinity });
try {
for await (const line of lines) {
if (createsHistory.test(line)) return true;
}
return false;
} finally {
lines.close();
input.destroy();
}
}
/**
* Replaces the current schema of the database at `url` with the dump at
* `source`: pg_restore turns the dump's objects in that schema into a script
* beside it (anything else the dump holds is left out), then psql clears the
* schema and runs the script in one transaction, so a restore that fails
* leaves the database as it was. A dump without TREK's tables in that schema
* is refused before anything is cleared.
*/
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
await refuseToolsWithoutRestrict(binDir);
const schema = await pgCurrentSchema(url, binDir);
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
try {
const clear = path.join(workDir, 'clear-schema.sql');
const script = path.join(workDir, 'restore.sql');
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
createPrivateFile(script);
await run(
'pg_restore',
binDir,
['--no-owner', '--no-privileges', `--schema=${schema.name}`, `--file=${script}`, '--', source],
{},
);
if (!(await scriptHoldsTrekSchema(script, schema))) {
throw new Error(`The backup holds no TREK tables in the schema "${schema.name}"; nothing was replaced.`);
}
await run(
'psql',
binDir,
[
'--no-psqlrc',
'--quiet',
'--single-transaction',
'--set=ON_ERROR_STOP=1',
`--output=${os.devNull}`,
`--file=${clear}`,
`--file=${script}`,
],
{ ...pgConnectionEnv(url), PGOPTIONS: RESTORE_SESSION_OPTIONS },
);
} finally {
fs.rmSync(workDir, { recursive: true, force: true });
}
}
/** The tables in the current schema of the database at `url`, read before TREK opens it. */
export async function pgTableNames(url: string, binDir: string | undefined): Promise<string[]> {
const out = await run(
'psql',
binDir,
['--no-psqlrc', '--no-align', '--tuples-only', `--command=${CURRENT_SCHEMA_TABLES_SQL}`],
pgConnectionEnv(url),
);
return out.split('\n').filter((line) => line !== '');
}
/**
* The migration names in a restore script's COPY block for
* `mikro_orm_migrations`, read from its `name` column; none when the script
* has no such block.
*/
function migrationNamesFromCopy(script: string): string[] {
const lines = script.split('\n');
const copy = lines.findIndex((line) => /^COPY \S*mikro_orm_migrations \(/.test(line));
if (copy < 0) return [];
const columns =
/\(([^)]*)\)/
.exec(lines[copy])?.[1]
.split(',')
.map((column) => column.trim()) ?? [];
const nameAt = columns.indexOf('name');
if (nameAt < 0) return [];
const names: string[] = [];
for (const line of lines.slice(copy + 1)) {
if (line === '\\.' || line === '') break;
const name = line.split('\t')[nameAt];
if (name) names.push(name);
}
return names;
}
/**
* The migrations a dump records as executed, read from the dump itself (its
* `mikro_orm_migrations` rows as COPY text), so a restore can refuse a newer
* TREK's backup before anything is replaced. Throws on a file that is not a
* dump; an empty list means the dump holds no TREK history.
*/
export async function pgDumpMigrations(source: string, binDir: string | undefined): Promise<string[]> {
const script = await run(
'pg_restore',
binDir,
['--data-only', '--table=mikro_orm_migrations', '--file=-', '--', source],
{},
);
return migrationNamesFromCopy(script);
}
@@ -0,0 +1,67 @@
import { type Configuration } from '@mikro-orm/core';
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
/** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */
const INT8_OID = 20;
/** `numeric`: what `sum()` of a bigint and `avg()` answer. */
const NUMERIC_OID = 1700;
/**
* Reads a `bigint` as a JavaScript number, the way better-sqlite3 hands
* SQLite's 64-bit INTEGER to the services. `pg` returns it as a string to
* keep precision past 2^53; no TREK value comes near that (ids, counts,
* epoch milliseconds, cents), so one that does is refused rather than
* rounded.
*/
export function parseInt8(text: string): number {
const value = Number(text);
if (!Number.isSafeInteger(value)) throw new Error(`A bigint of ${text} is beyond what TREK reads exactly (2^53).`);
return value;
}
/**
* Reads a `numeric` as a number, as SQLite answers `sum()` and `avg()`. `NaN`
* and the infinities are refused, as is an integral result past 2^53, which is
* refused like a bigint; a fraction is a double, the precision SQLite's REAL has.
*/
export function parseNumeric(text: string): number {
const value = Number(text);
if (!Number.isFinite(value)) {
throw new Error(`A numeric of ${text} is not a finite number, which TREK does not read.`);
}
if (/^-?\d+$/.test(text) && !Number.isSafeInteger(value)) {
throw new Error(`A numeric of ${text} is beyond what TREK reads exactly (2^53).`);
}
return value;
}
/**
* MikroORM's `PostgreSqlConnection`, with the pool's own type parsers replaced
* for `int8` and `numeric`. MikroORM keeps dates as text, so only these two
* change. `mapOptions` builds the `pg` pool options, whose `types` is the
* `pg` TypeOverrides, the object that holds the parsers.
*/
class NumberTypesConnection extends PostgreSqlConnection {
override mapOptions(overrides: Parameters<PostgreSqlConnection['mapOptions']>[0]) {
const options = super.mapOptions(overrides);
const types = options.types as { setTypeParser(oid: number, parse: (text: string) => unknown): void };
types.setTypeParser(INT8_OID, parseInt8);
types.setTypeParser(NUMERIC_OID, parseNumeric);
return options;
}
}
/**
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
* as numbers. The parsers belong to this pool only: nothing global in `pg`
* changes.
*/
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
constructor(config: Configuration) {
super(config);
// PostgreSqlDriver's constructor hardcodes its connection class, as
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
// field, which is not public, so it is replaced here, before anything connects.
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
}
}
@@ -0,0 +1,90 @@
import type { DbEngineSettings } from '../../../app-config/db-settings';
import { NO_SAFETY_NET } from '../../legacy-baseline';
import { finishSchemaBootstrap, type AnyOrm } from '../../orm';
import type { DatabaseEngine } from '../engine';
import { redactUrl } from '../redact-url';
import { TrekPostgreSqlDriver } from './postgres-driver';
import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap';
import { sslOptionFor } from './ssl';
import type { EntityManager } from '@mikro-orm/core';
/** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */
type PostgresSettings = Extract<DbEngineSettings, { engine: 'postgres' }>;
/** The advisory lock key every write transaction takes: "TREK" in ASCII. */
const TREK_WRITE_LOCK = 0x5452454b;
/**
* The Postgres engine: MikroORM's own `pg` driver on `TREK_DB_URL`, with a
* pool of at most `TREK_DB_POOL_MAX` connections that keeps none open while
* idle. The pool belongs to the ORM, which the Nest container closes; the
* schema comes from the baseline plus the migrations after it
* (`schema-bootstrap.ts`).
*/
export class PostgresEngine implements DatabaseEngine {
readonly id = 'postgres';
constructor(
private readonly settings: PostgresSettings,
private readonly bootstrapDeps: (url: string) => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
) {}
private onReopened: (() => Promise<void>) | null = null;
/**
* Nothing of its own to open: the ORM connects the pool. Keeps what a reopen
* runs. A reopen replaces the database's contents, which the pool's own
* connections keep reaching, so the ORM is not pointed at anything new and
* the pool is never closed (ending a pool waits for every connection it has
* lent out, and one lent to a job during the restore would never come back).
*/
open(_orm: AnyOrm, onReopened: () => Promise<void>): void {
this.onReopened = onReopened;
}
close(): void {
// Nothing of its own to close: the pool goes with the ORM.
}
/**
* After `pg_restore` replaced the database's contents: the restored
* database is migrated forward on the same pool (`onReopened`, which
* `DatabaseLifecycle` hands to `open`).
*/
async reopen(): Promise<void> {
if (!this.onReopened) throw new Error('The Postgres engine was reopened before it was opened.');
await this.onReopened();
}
/**
* `pg_advisory_xact_lock` on one key for the whole database: held until the
* transaction commits or rolls back, taken again by a savepoint in the same
* session without waiting, and shared by every process on the database.
*/
async serializeWrites(em: EntityManager): Promise<void> {
await em
.getConnection()
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
}
/** `snapshot: false` (a restore, whose archive is the copy) migrates without a pg_dump first. */
async schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
const deps = this.bootstrapDeps(this.settings.url);
await bootstrapPostgresSchema(orm, options.snapshot ? deps : { ...deps, safetyNet: NO_SAFETY_NET });
await finishSchemaBootstrap(orm);
}
describe(): string {
return `Postgres ${redactUrl(this.settings.url)}`;
}
ormOptions() {
const ssl = sslOptionFor(this.settings.url);
return {
driver: TrekPostgreSqlDriver,
clientUrl: this.settings.url,
pool: { min: 0, max: this.settings.poolMax },
driverOptions: ssl === undefined ? {} : { ssl },
};
}
}
@@ -0,0 +1,32 @@
/**
* The SQLite schema's triggers as Postgres writes them. The entities describe
* tables only, so the baseline (`scripts/generate-pg-baseline.ts`) adds these
* after the tables, and `check:pg-baseline` fails while a trigger in the
* migrated SQLite schema has no entry here. Each entry is the trigger's
* function and the trigger that runs it, under the SQLite trigger's name.
*/
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
trg_release_stop_on_stay_delete: [
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
RETURN OLD;
END
$$`,
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
],
// A place that moved or was renamed drops the region it was resolved to (#2527).
trg_place_regions_follow_place: [
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM place_regions WHERE place_id = NEW.id;
RETURN NEW;
END
$$`,
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
'EXECUTE FUNCTION trg_place_regions_follow_place()',
],
};
@@ -0,0 +1,50 @@
import { run } from './pg-process';
/**
* The client-tool versions a restore accepts (CVE-2025-8714). A script from a
* dump an admin uploaded is run by psql, which would also run a psql
* meta-command (`\!` runs a shell command) smuggled into an object name. A
* fixed pg_restore opens the script with `\restrict <random key>`, and a fixed
* psql then refuses every meta-command until the matching `\unrestrict`.
*
* RESTRICT_SINCE holds, per major, the first minor that carries the fix. Every
* major after the last one listed has it from its first release.
*/
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
const LAST_MAJOR_WITHOUT_IT = Math.max(...Object.keys(RESTRICT_SINCE).map(Number));
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
export function hasRestrictFix(version: string): boolean {
const match = /\(PostgreSQL\) (\d+)\.(\d+)/.exec(version);
if (!match) return false;
const major = Number(match[1]);
const minor = Number(match[2]);
if (major > LAST_MAJOR_WITHOUT_IT) return true;
const firstFixedMinor = RESTRICT_SINCE[major];
return firstFixedMinor !== undefined && minor >= firstFixedMinor;
}
/** The minimum versions as the error names them, newest major first: "17.6, 16.10, 15.14 or 14.19". */
function minimumVersionsText(): string {
const floors = Object.keys(RESTRICT_SINCE)
.map(Number)
.sort((a, b) => b - a)
.map((major) => `${major}.${RESTRICT_SINCE[major]}`);
const last = floors.pop() ?? '';
return floors.length > 0 ? `${floors.join(', ')} or ${last}` : last;
}
/**
* Refuses the restore when pg_restore or psql predates the fix. The tools'
* own versions are checked, not the script, whose text the dump controls.
*/
export async function refuseToolsWithoutRestrict(binDir: string | undefined): Promise<void> {
const tools = ['pg_restore', 'psql'] as const;
const versions = await Promise.all(tools.map((tool) => run(tool, binDir, ['--version'], {})));
const stale = tools.findIndex((_, i) => !hasRestrictFix(versions[i]));
if (stale < 0) return;
throw new Error(
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least ${minimumVersionsText()} ` +
'(the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
);
}
@@ -0,0 +1,109 @@
/**
* How the entities' schema is written in Postgres: the one definition the
* Postgres baseline (`scripts/generate-pg-baseline.ts`), the schema parity
* check and the probe (`scripts/pg-probe.ts`) share.
*
* The entities are generated from the migrated SQLite schema and a parity
* test pins them to it (`npm run check:entities`), so they are the one
* description of the tables both engines can read. MikroORM's schema
* generator turns them into Postgres DDL after the adjustments below, which
* keep the storage model the services and the dialect helpers are written
* for (server/CLAUDE.md, "Storage model"):
*
* - timestamps stay TEXT;
* - an INTEGER column is `bigint`: SQLite's INTEGER holds 64 bits, so a
* column written with epoch milliseconds or a byte count past 2 GB would
* overflow Postgres's 32-bit `integer` (the engine's driver reads
* `bigint` back as a number, `postgres-driver.ts`);
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`,
* `CURRENT_TIMESTAMP` in a TEXT column) gets its Postgres equivalent, and
* a column declared `COLLATE NOCASE` uses the `nocase` collation
* `collateNoCase()` names on Postgres.
*
* The baseline keeps the foreign keys; the probe leaves them out, so a
* statement it sends is never refused only because the sample row it writes
* points at nothing. The triggers the SQLite schema has are in
* `postgres-triggers.ts`.
*/
import { pgTimestampText, PG_UTC_NOW } from '../../dialect/platform';
import { DbTimestampType } from '../../types';
/** SQLite's timestamp text on Postgres, for a column default. */
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
export const NOCASE_COLLATION_SQL =
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
/** The Postgres spelling of a column default the entities carry in SQLite's. */
export function portableDefault(raw: string): string {
const text = raw
.trim()
.replace(/^\((.*)\)$/s, '$1')
.trim();
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
return raw;
}
/** A column type SQLite stores as its 64-bit INTEGER. */
const INTEGER_TYPE = /^(integer|int)$/i;
/**
* The subset of MikroORM's property metadata the adjustments read and write.
* An entity's `props` satisfy it as they are, so no cast is needed.
*/
export interface ColumnMeta {
name: string;
defaultRaw?: string;
collation?: string;
columnTypes?: string[];
customType?: unknown;
}
/** A timestamp column becomes text. Returns whether it changed. */
function timestampAsText(column: ColumnMeta): boolean {
if (!(column.customType instanceof DbTimestampType) || !column.columnTypes) return false;
column.columnTypes = column.columnTypes.map(() => 'text');
return true;
}
/** An integer column becomes bigint. Returns whether it changed. */
function integerAsBigint(column: ColumnMeta): boolean {
if (!column.columnTypes?.some((type) => INTEGER_TYPE.test(type))) return false;
column.columnTypes = column.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
return true;
}
/** A SQLite-only default gets its Postgres spelling. Returns whether it changed. */
function defaultAsPostgres(column: ColumnMeta): boolean {
if (typeof column.defaultRaw !== 'string') return false;
const next = portableDefault(column.defaultRaw);
if (next === column.defaultRaw) return false;
column.defaultRaw = next;
return true;
}
/** A NOCASE column uses the `nocase` collation. Returns whether it changed. */
function nocaseAsCollation(column: ColumnMeta): boolean {
if (column.collation?.toUpperCase() !== 'NOCASE') return false;
column.collation = 'nocase';
return true;
}
/**
* Applies the adjustments above to an entity's properties, in place, and
* returns the names of the properties it changed, for the log. A timestamp
* column is text, so the integer adjustment never sees it.
*/
export function applyPostgresColumnAdjustments(properties: readonly ColumnMeta[]): string[] {
const changed: string[] = [];
for (const property of properties) {
const typeChanged = timestampAsText(property) || integerAsBigint(property);
const defaultChanged = defaultAsPostgres(property);
const collationChanged = nocaseAsCollation(property);
if (typeChanged || defaultChanged || collationChanged) changed.push(property.name);
}
return changed;
}
@@ -0,0 +1,185 @@
import { readEnv, type AppEnv } from '../../../app-config';
import { resolveDataPaths } from '../../../app-config/data-paths';
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
import { knownMigrationNames } from '../../known-migrations';
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
import type { AnyOrm } from '../../orm';
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
import { pgDump } from './pg-tools';
import type { Connection } from '@mikro-orm/core';
import type { Migrator } from '@mikro-orm/migrations';
import { Logger } from '@nestjs/common';
import fs from 'node:fs';
import path from 'node:path';
const logger = new Logger('DB');
/** A Postgres copy is a pg_dump custom-format archive. */
const DUMP_EXTENSION = '.pgdump';
/** What a Postgres bootstrap reads, passed in so a test can supply its own. */
export interface PostgresBootstrapDeps {
baseline: () => PostgresBaseline;
known: Set<string>;
safetyNet: MigrationSafetyNet;
}
/**
* The tables in the current schema. A database with no TREK migration history
* must hold none: TREK creates its schema in an empty database and never
* adopts tables something else made. Read before anything is written, so a
* refused database is left exactly as it was.
*/
async function currentTables(connection: Connection): Promise<string[]> {
const rows: Array<{ table_name: string }> = await connection.execute(CURRENT_SCHEMA_TABLES_SQL);
return rows.map((row) => row.table_name);
}
/**
* Creates the schema in an empty Postgres database: the baseline's statements
* and the record of every migration it stands for, in one transaction (DDL is
* transactional on Postgres, so the database ends with all of it or none). A
* database holding tables beside no TREK history is refused before anything is
* written.
*/
async function createFromBaseline(
connection: Connection,
storage: ReturnType<Migrator['getStorage']>,
tables: readonly string[],
historyTable: string,
baseline: () => PostgresBaseline,
): Promise<void> {
const foreign = tables.filter((table) => table !== historyTable);
if (foreign.length > 0) {
throw new Error(
`${REFUSAL} the database holds ${foreign.length} table(s) (${foreign.slice(0, 5).join(', ')}) but no TREK ` +
'migration history. Point TREK_DB_URL at an empty database.',
);
}
await storage.ensureTable();
const schema = baseline();
logger.log(`Creating the schema in an empty Postgres database (through ${migrationLabel(schema.through)})`);
await connection.transactional(async (trx) => {
await connection.execute(schema.statements.join(';\n'), [], 'run', trx);
await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx)));
});
}
/**
* Refuses a database whose recorded history names a migration this build does
* not know: a newer TREK migrated it. The refusal names the copy taken before
* that migration, when there is one.
*/
async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDeps): Promise<void> {
const thisRelease = migrationLabel([...deps.known].sort().at(-1) ?? '');
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
}
/**
* Brings a Postgres database to the current schema: inspect what it holds,
* then create the schema or verify the history it has, then migrate.
*
* - No migration recorded and no table: the baseline is created
* (`createFromBaseline`).
* - No migration recorded but tables present: refused, naming a few of them.
* - Migrations recorded: refused when one is unknown to this build, else the
* pending migrations run, after the safety net. A database the baseline just
* created runs the migrations after the baseline's cut without it: there is
* nothing yet to lose.
*/
export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstrapDeps): Promise<void> {
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
const connection = orm.em.getConnection();
const historyTable = orm.config.get('migrations').tableName ?? 'mikro_orm_migrations';
const tables = await currentTables(connection);
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
if (executed.length === 0) {
await createFromBaseline(connection, migrator.getStorage(), tables, historyTable, deps.baseline);
} else {
await refuseNewerHistory(executed, deps);
}
const pending = await migrator.getPending();
if (pending.length === 0) return;
// Only a database with history has a last migration to label its copy by; a
// database the baseline just created has nothing yet to lose.
const latest = [...executed].sort().at(-1);
if (latest !== undefined) await deps.safetyNet.beforeMigrate(migrationLabel(latest));
logger.log(`Applying ${pending.length} pending migration(s)`);
await migrator.up();
}
/** Dumps a killed boot left half-written: never a copy, only disk taken. */
function removeOrphanedPartials(dir: string): void {
for (const file of fs.readdirSync(dir)) {
if (file.startsWith(PREFIX) && file.endsWith(`${DUMP_EXTENSION}.partial`)) {
fs.rmSync(path.join(dir, file), { force: true });
}
}
}
/**
* The Postgres safety net before pending migrations, as the SQLite file's
* (`db/pre-migrate-snapshot.ts`): a `pg_dump` of the database to
* `<data>/pre-migrate-<schema>-<time>.pgdump`, the newest
* `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` kept. A copy that cannot be written
* (pg_dump missing, the data dir full) refuses the boot before anything is
* migrated, unless `TREK_DB_PRE_MIGRATE_SNAPSHOT=false`. Off under
* NODE_ENV=test unless a suite asks for it.
*/
export function postgresSafetyNet(
url: string,
env: () => AppEnv = readEnv,
now: () => Date = () => new Date(),
dataDir: () => string = () => resolveDataPaths(env()).dataDir,
): MigrationSafetyNet {
return {
async beforeMigrate(from) {
const { db, app } = env();
if (db.preMigrateSnapshot === undefined && app.isTest) return;
if (db.preMigrateSnapshot === false) {
logger.warn('TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating the Postgres database without a copy');
return;
}
const target = path.join(dataDir(), `${PREFIX}${from ?? 'unversioned'}-${stamp(now())}${DUMP_EXTENSION}`);
try {
fs.mkdirSync(dataDir(), { recursive: true });
removeOrphanedPartials(dataDir());
await pgDump(url, target, db.pgBinDir);
} catch (err) {
throw new Error(
`${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
`(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Fix that, ` +
'or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
{ cause: err },
);
}
logger.log(`Copy of the database before migrating: ${target}`);
pruneSnapshots(dataDir(), db.preMigrateSnapshotKeep, DUMP_EXTENSION);
},
async restoreHint(thisRelease) {
const snapshots = listSnapshots(dataDir(), DUMP_EXTENSION);
const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
if (match) {
return (
`The copy taken right before the newer TREK migrated it is ${match}: stop TREK, empty the database ` +
'(drop and recreate it), and restore it with pg_restore --no-owner --no-privileges --single-transaction.'
);
}
if (snapshots.length > 0) {
return `Copies taken before earlier migrations are in ${dataDir()} (${PREFIX}*${DUMP_EXTENSION}), none of them from this release.`;
}
return null;
},
};
}
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
}
@@ -0,0 +1,59 @@
/**
* Splits generated DDL into single statements, so one statement Postgres
* refuses does not take the rest of the schema with it.
*
* Understood: `;` outside quotes, `'` and `"` quoted text, and the `$$` dollar
* quote a function body is written in, which is the only dollar quote TREK's
* generated DDL and baseline contain. A tagged dollar quote (`$tag$`) is not
* understood, so it is refused rather than split inside.
*/
/** A tagged dollar quote (`$tag$`), matched where the scan stands. */
const TAGGED_DOLLAR_QUOTE = /\$[A-Za-z_]\w*\$/y;
function refuseTaggedDollarQuote(sql: string, at: number): void {
TAGGED_DOLLAR_QUOTE.lastIndex = at;
const tag = TAGGED_DOLLAR_QUOTE.exec(sql)?.[0];
if (tag) throw new Error(`splitStatements understands only the $$ dollar quote, not ${tag} (at ${at})`);
}
/**
* The statements in `sql`, trimmed and in order. A statement that sets a
* session parameter (`SET ...`) is dropped: a dump's preamble, which a schema
* the entities generate never needs.
*/
export function splitStatements(sql: string): string[] {
const statements: string[] = [];
let current = '';
let quote: '"' | "'" | '$$' | null = null;
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const dollar = sql.startsWith('$$', i);
if (quote === '$$') {
if (dollar) {
quote = null;
current += '$';
i++;
}
current += char;
} else if (quote) {
if (char === quote) quote = null;
current += char;
} else if (dollar) {
quote = '$$';
current += '$$';
i++;
} else if (char === "'" || char === '"') {
quote = char;
current += char;
} else if (char === ';') {
if (current.trim()) statements.push(current.trim());
current = '';
} else {
if (char === '$') refuseTaggedDollarQuote(sql, i);
current += char;
}
}
if (current.trim()) statements.push(current.trim());
return statements.filter((statement) => !/^set\s/i.test(statement));
}
+42
View File
@@ -0,0 +1,42 @@
import { readFileSync } from 'node:fs';
/** The sslmode values that verify the server, read the way the `pg` driver reads them. */
const VERIFYING_MODES = new Set(['require', 'verify-ca', 'verify-full']);
/** The `ssl` option of a `pg` pool, as `sslOptionFor` builds it. */
type PgSslOption = false | { rejectUnauthorized: true; ca?: string };
/**
* The `ssl` option for the pool, from the URL's `sslmode` and `sslrootcert`.
*
* MikroORM builds the pool from the URL's host, port, user, password and
* database alone and drops the rest of the query, so an `sslmode` in
* `TREK_DB_URL` would be ignored without this. `require`, `verify-ca` and
* `verify-full` all verify the server's certificate, as the `pg` driver itself
* treats them; `disable` turns SSL off, and no `sslmode` leaves the driver's
* default (no SSL). A mode that would connect without verifying (`prefer`,
* `allow`, `no-verify`) or that is misspelt refuses, rather than quietly
* connecting with less than was asked. `sslrootcert` names the CA file a host
* that signs its own certificates needs (Supabase's, say).
*/
export function sslOptionFor(url: string): PgSslOption | undefined {
const params = new URL(url).searchParams;
const mode = params.get('sslmode');
const rootCert = params.get('sslrootcert');
if (mode === null) {
if (rootCert !== null) throw new Error('TREK_DB_URL names an sslrootcert without sslmode=verify-full.');
return undefined;
}
if (mode === 'disable') return false;
if (!VERIFYING_MODES.has(mode)) {
throw new Error(
`TREK_DB_URL has sslmode "${mode}"; use disable, require, verify-ca or verify-full (all but disable verify the server).`,
);
}
if (rootCert === null) return { rejectUnauthorized: true };
try {
return { rejectUnauthorized: true, ca: readFileSync(rootCert, 'utf8') };
} catch (error) {
throw new Error(`TREK_DB_URL names sslrootcert ${rootCert}, which cannot be read.`, { cause: error });
}
}
+17
View File
@@ -0,0 +1,17 @@
/**
* A connection URL fit for a log line: the password masked and the query
* dropped, since libpq reads a password from the query too
* (`?password=...`). A value that does not parse is not echoed at all.
*/
export function redactUrl(url: string): string {
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return '<unparseable URL>';
}
if (parsed.password) parsed.password = '***';
parsed.search = '';
parsed.hash = '';
return parsed.toString();
}
@@ -0,0 +1,61 @@
import { closeDb, getRawConnection, registerReinitializeHook, reinitialize } from '../../database';
import { resolveDbPath } from '../../db-path';
import { runSchemaBootstrap, type AnyOrm } from '../../orm';
import { SharedSqliteDriver } from '../../orm-driver';
import type { DatabaseEngine } from '../engine';
import type { EntityManager } from '@mikro-orm/core';
/**
* The SQLite file engine, TREK's default: one better-sqlite3 handle that
* `db/database.ts` owns and the ORM's bound driver shares (`db/orm-driver.ts`),
* on `TREK_DB_FILE` or `data/travel.db` (`:memory:` under test). A restore
* swaps the file and reopens the handle; the schema comes from the migration
* chain (`db/orm.ts`).
*/
export class SqliteEngine implements DatabaseEngine {
readonly id = 'sqlite';
describe(): string {
return `SQLite file ${resolveDbPath()}`;
}
ormOptions() {
return { driver: SharedSqliteDriver, dbName: resolveDbPath() };
}
/**
* Opens the file (normally a no-op: the ORM's first connect inside
* NestFactory.create() already did, said here so the boot does not depend on
* that), and registers the reopen. A reopen swaps the file under the same
* handle, which Kysely caches, so the ORM's connection is closed and connected
* again first: the driver then reaches the new handle through
* `createKyselyDialect()`. The bound driver's `destroy()` is a no-op
* (`orm-driver.ts`), so the close never reaches the handle `db/database.ts` owns.
*/
open(orm: AnyOrm, onReopened: () => Promise<void>): void {
getRawConnection();
registerReinitializeHook(async () => {
const connection = orm.em.getConnection();
await connection.close(true);
await connection.connect();
await onReopened();
});
}
close(): void {
closeDb();
}
reopen(): Promise<void> {
return reinitialize();
}
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
return runSchemaBootstrap(orm, options);
}
/** A no-op: SQLite has one connection and one writer, so its transactions already run one at a time. */
serializeWrites(_em: EntityManager): Promise<void> {
return Promise.resolve();
}
}
@@ -0,0 +1,49 @@
import type { EntityManager } from '@mikro-orm/core';
/**
* The SQLite engine's whole-file maintenance: the two statements a backup,
* a restore and the demo reset need that are about the database file rather
* than any table (`PRAGMA wal_checkpoint`, `VACUUM INTO`). Neither has a
* Postgres spelling, so they live under the engine that owns the file instead
* of in `MaintenanceRepository`, whose remaining statements run on every
* engine (and which the Postgres probe calls method by method).
*
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
* factory over the context-resolving global `EntityManager`, and injected by
* the SQLite backup port (`SqliteDatabaseBackup`). Free of Nest decorators
* like every other file under `db/`, so a test can build one over its own
* `EntityManager`.
*/
export class SqliteMaintenance {
constructor(private readonly em: EntityManager) {}
/**
* MikroORM's own validating `getContext()`: throws outside a request
* context, the fail-closed check `TrekRepository` applies to every
* repository, since a raw `connection.execute()` has no such guard.
*/
private validateRequestContext(): void {
this.em.getContext();
}
/**
* `PRAGMA wal_checkpoint(TRUNCATE)`: flushes the WAL into the main file
* before a snapshot or a swap. Reached through the backup port's
* `checkpoint()`; every caller treats it as best effort.
*/
async walCheckpoint(): Promise<void> {
this.validateRequestContext();
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
}
/**
* `` VACUUM INTO '<path>' ``: a consistent point-in-time copy of the file
* under concurrent writers, reached through the backup port's
* `snapshot()`. Single quotes in `path` are escaped by doubling.
*/
async vacuumInto(path: string): Promise<void> {
this.validateRequestContext();
const escaped = path.replaceAll("'", "''");
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
}
}