mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
feat(db): run TREK on PostgreSQL
TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before otherwise: an engine port under src/db/engine, the dialect helpers under src/db/dialect, a Postgres baseline for the migrations up to the cut, pg_dump/pg_restore backups, a probe of every repository method against a real server and a Postgres test lane in CI.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
import type { EngineId } from '../../app-config/db-settings';
|
||||
import type { AnyOrm } from '../orm';
|
||||
import type { EntityManager, Options } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* One database engine: what the ORM needs to reach it, how the connection
|
||||
* opens, closes and comes back after a restore, and how a database is brought
|
||||
* to the current schema. Code that differs by engine dispatches through this,
|
||||
* or through the helpers in `db/dialect/`, never through its own platform
|
||||
* check (`lint:engine-dispatch`).
|
||||
*
|
||||
* Free of Nest like every other file under `db/`: `src/db` does not import
|
||||
* `src/nest` (`lint:boundaries`). `DatabaseLifecycle` is the provider that
|
||||
* calls it.
|
||||
*/
|
||||
export interface DatabaseEngine {
|
||||
readonly id: EngineId;
|
||||
/** For log lines and errors: the file path, or the URL without its password. */
|
||||
describe(): string;
|
||||
/** What MikroORM needs to reach the database: the driver and its target, the pool, driver options. */
|
||||
ormOptions(): Partial<Options>;
|
||||
/**
|
||||
* Called once at boot, before the schema bootstrap. `onReopened` runs after
|
||||
* each later reopen, once the ORM is pointed at the reopened database: the
|
||||
* engine does what its own reopen needs first (SQLite hands the ORM a new
|
||||
* handle, so the ORM's connection is closed and connected again), then calls
|
||||
* it, and the caller migrates the database and runs whatever follows.
|
||||
*/
|
||||
open(orm: AnyOrm, onReopened: () => Promise<void>): void;
|
||||
/** Closes what the engine itself holds; the ORM's own pool is closed with the Nest container. */
|
||||
close(): void;
|
||||
/**
|
||||
* Comes back after the database was replaced by a restore, then runs the
|
||||
* `onReopened` that `open` registered. Throws when `open` has not run.
|
||||
*/
|
||||
reopen(): Promise<void>;
|
||||
/**
|
||||
* Brings the database to the current schema, then seeds it. `snapshot:
|
||||
* false` skips the copy taken before pending migrations (a restore, whose
|
||||
* archive is the copy).
|
||||
*/
|
||||
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void>;
|
||||
/**
|
||||
* Runs first in every `UnitOfWork` transaction, on its EntityManager: makes
|
||||
* the transactions that write run one at a time, as SQLite's single writer
|
||||
* does, so a `MAX(...) + 1` read inside one cannot race another.
|
||||
*/
|
||||
serializeWrites(em: EntityManager): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { readEnv } from '../../app-config';
|
||||
import type { DbEngineSettings } from '../../app-config/db-settings';
|
||||
import type { DbSettings } from '../../app-config/derive';
|
||||
import { ALL_ENTITIES } from '../entities';
|
||||
import type { DatabaseEngine } from './engine';
|
||||
import { PostgresEngine } from './postgres/postgres-engine';
|
||||
import { SqliteEngine } from './sqlite/sqlite-engine';
|
||||
import { defineConfig, type Options } from '@mikro-orm/core';
|
||||
import { Migrator } from '@mikro-orm/migrations';
|
||||
import { SeedManager } from '@mikro-orm/seeder';
|
||||
|
||||
export type { DatabaseEngine } from './engine';
|
||||
|
||||
/**
|
||||
* The engine this process runs on, from `TREK_DB_URL` (`readEnv().db`): the
|
||||
* SQLite file unless the URL names a Postgres database. Read afresh on every
|
||||
* call, like everything `readEnv()` answers, so nothing caches a choice a test
|
||||
* or a restart changes.
|
||||
*/
|
||||
export function resolveEngine(db: DbEngineSettings = readEnv().db): DatabaseEngine {
|
||||
if (db.engine === 'postgres') return new PostgresEngine(db);
|
||||
return new SqliteEngine();
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres URL and the directory of its client tools, when the settings
|
||||
* name a Postgres database; null on SQLite. For the code that drives pg_dump
|
||||
* and pg_restore, which needs both and must not read them from the environment
|
||||
* again.
|
||||
*/
|
||||
export function postgresTarget(db: DbSettings): { url: string; binDir: string | undefined } | null {
|
||||
if (db.engine !== 'postgres') return null;
|
||||
return { url: db.url, binDir: db.pgBinDir };
|
||||
}
|
||||
|
||||
/**
|
||||
* The MikroORM config for `engine`: what every engine shares (entities,
|
||||
* migrations, seeders) and what the engine adds. `src/mikro-orm.config.ts`
|
||||
* exports it for the CLI and the Nest container, so both reach the same
|
||||
* database.
|
||||
*/
|
||||
export function buildOrmConfig(engine: DatabaseEngine): Options {
|
||||
return defineConfig({
|
||||
entities: ALL_ENTITIES,
|
||||
extensions: [Migrator, SeedManager],
|
||||
migrations: {
|
||||
path: 'dist/db/migrations',
|
||||
pathTs: 'src/db/migrations',
|
||||
snapshot: false,
|
||||
},
|
||||
seeder: {
|
||||
path: 'dist/db/seeders',
|
||||
pathTs: 'src/db/seeders',
|
||||
defaultSeeder: 'DatabaseSeeder',
|
||||
},
|
||||
...engine.ormOptions(),
|
||||
}) as Options;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { splitStatements } from './sql-split';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* Where the Postgres baseline lives: `server/assets/postgres/`, which the
|
||||
* Docker image copies beside `dist/` (the same place the airports and atlas
|
||||
* data are read from), reached the same way from `src/` and from `dist/`.
|
||||
*/
|
||||
const BASELINE_DIR = path.join(__dirname, '..', '..', '..', '..', 'assets', 'postgres');
|
||||
const SQL_FILE = 'baseline.sql';
|
||||
const MANIFEST_FILE = 'baseline.json';
|
||||
|
||||
/** The manifest beside the SQL: the last migration the baseline stands for, and every migration it replaces. */
|
||||
interface BaselineManifest {
|
||||
through: string;
|
||||
migrations: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The schema a fresh Postgres database starts from.
|
||||
*
|
||||
* The migrations up to `through` are written in SQLite and frozen, so a
|
||||
* Postgres database does not replay them: it runs these statements, generated
|
||||
* from the entities (`npm run gen:pg-baseline`), and records every migration
|
||||
* in `migrations` as executed in the same transaction. Migrations after
|
||||
* `through` run on both engines through the migrator.
|
||||
*/
|
||||
export interface PostgresBaseline extends BaselineManifest {
|
||||
statements: string[];
|
||||
}
|
||||
|
||||
/** The two files a baseline is written as, ready to compare or write. */
|
||||
export function renderBaseline(baseline: PostgresBaseline): { sql: string; manifest: string } {
|
||||
return {
|
||||
sql:
|
||||
'-- The schema a fresh Postgres database starts from. Generated by `npm run gen:pg-baseline`; do not edit.\n' +
|
||||
baseline.statements.map((statement) => `${statement};\n`).join(''),
|
||||
manifest: `${JSON.stringify({ through: baseline.through, migrations: baseline.migrations }, null, 2)}\n`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `data` is a manifest a baseline can be read from: a `through` naming
|
||||
* one of a non-empty list of `migrations`, every entry a string.
|
||||
*/
|
||||
function isBaselineManifest(data: unknown): data is BaselineManifest {
|
||||
if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false;
|
||||
const { through, migrations } = data;
|
||||
return (
|
||||
typeof through === 'string' &&
|
||||
Array.isArray(migrations) &&
|
||||
migrations.length > 0 &&
|
||||
migrations.every((name): name is string => typeof name === 'string') &&
|
||||
migrations.includes(through)
|
||||
);
|
||||
}
|
||||
|
||||
function parseManifest(text: string, file: string): BaselineManifest {
|
||||
const data: unknown = JSON.parse(text);
|
||||
if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`);
|
||||
return { through: data.through, migrations: data.migrations };
|
||||
}
|
||||
|
||||
/** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */
|
||||
export function readPostgresBaseline(dir = BASELINE_DIR): PostgresBaseline {
|
||||
const manifestFile = path.join(dir, MANIFEST_FILE);
|
||||
const manifest = parseManifest(fs.readFileSync(manifestFile, 'utf8'), manifestFile);
|
||||
const statements = splitStatements(fs.readFileSync(path.join(dir, SQL_FILE), 'utf8').replace(/^--[^\n]*\n/gm, ''));
|
||||
if (statements.length === 0) throw new Error(`${path.join(dir, SQL_FILE)} holds no statements`);
|
||||
// The boot sends the baseline as one statement without parameters, where a `?` would be read as a binding.
|
||||
if (statements.some((statement) => statement.includes('?'))) {
|
||||
throw new Error(`${path.join(dir, SQL_FILE)} holds a "?", which the boot would read as a parameter`);
|
||||
}
|
||||
return { ...manifest, statements };
|
||||
}
|
||||
|
||||
/** Writes the two files (the generator's `--write`). */
|
||||
export function writePostgresBaseline(baseline: PostgresBaseline, dir = BASELINE_DIR): void {
|
||||
const { sql, manifest } = renderBaseline(baseline);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, SQL_FILE), sql);
|
||||
fs.writeFileSync(path.join(dir, MANIFEST_FILE), manifest);
|
||||
}
|
||||
|
||||
/** Whether the committed files are what `baseline` renders to (the generator's `--check`). */
|
||||
export function baselineMatches(baseline: PostgresBaseline, dir = BASELINE_DIR): boolean {
|
||||
const { sql, manifest } = renderBaseline(baseline);
|
||||
const read = (file: string) =>
|
||||
fs.existsSync(path.join(dir, file)) ? fs.readFileSync(path.join(dir, file), 'utf8') : null;
|
||||
return read(SQL_FILE) === sql && read(MANIFEST_FILE) === manifest;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* Drops every table, view, sequence, routine, collation and standalone type in
|
||||
* the current schema, except an extension's, before a dump is loaded: a dump
|
||||
* only recreates what it holds, so whatever a later migration added would
|
||||
* otherwise survive the restore and break the migrations the reopen runs. It
|
||||
* runs in the same transaction as the load (`pgRestore`), so a restore that
|
||||
* fails leaves the database as it was.
|
||||
*/
|
||||
export const CLEAR_SCHEMA_SQL = `DO $clear$
|
||||
DECLARE r record;
|
||||
BEGIN
|
||||
FOR r IN SELECT c.oid::regclass AS name, c.relkind FROM pg_class c
|
||||
WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype IN ('e', 'a', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP %s IF EXISTS %s CASCADE',
|
||||
CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' WHEN 'S' THEN 'SEQUENCE'
|
||||
WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END, r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT p.oid::regprocedure AS sig FROM pg_proc p
|
||||
WHERE p.pronamespace = current_schema()::regnamespace
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP ROUTINE IF EXISTS %s CASCADE', r.sig);
|
||||
END LOOP;
|
||||
FOR r IN SELECT format('%I.%I', n.nspname, c.collname) AS name FROM pg_collation c
|
||||
JOIN pg_namespace n ON n.oid = c.collnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP COLLATION IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT t.oid::regtype AS name FROM pg_type t
|
||||
WHERE t.typnamespace = current_schema()::regnamespace AND t.typtype IN ('e', 'd', 'r')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = t.oid AND d.deptype IN ('e', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP TYPE IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
END
|
||||
$clear$;
|
||||
`;
|
||||
@@ -0,0 +1,119 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** The Postgres client tools a backup runs. */
|
||||
export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
|
||||
|
||||
/**
|
||||
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
|
||||
* Debian's versioned directories newest first (the Docker image installs 17),
|
||||
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
|
||||
*/
|
||||
const DEFAULT_BIN_DIRS = [
|
||||
'/usr/lib/postgresql/17/bin',
|
||||
'/usr/lib/postgresql/16/bin',
|
||||
'/usr/lib/postgresql/15/bin',
|
||||
'/usr/lib/postgresql/14/bin',
|
||||
'/usr/local/bin',
|
||||
'/usr/bin',
|
||||
];
|
||||
|
||||
/** A dump or a restore of a large database takes minutes, not hours. */
|
||||
const TOOL_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
||||
/** An unreachable host fails the tool in seconds instead of holding a boot or a restore. */
|
||||
const CONNECT_TIMEOUT_SECONDS = '10';
|
||||
/** How much of a tool's stderr an error keeps. */
|
||||
const STDERR_LIMIT = 2000;
|
||||
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
|
||||
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
|
||||
|
||||
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
|
||||
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
|
||||
const dirs = binDir ? [binDir] : DEFAULT_BIN_DIRS;
|
||||
const executable = (candidate: string): boolean => {
|
||||
try {
|
||||
fs.accessSync(candidate, fs.constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false; // missing or not executable: the next directory may have it
|
||||
}
|
||||
};
|
||||
return dirs.map((dir) => path.join(dir, tool)).find(executable) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The libpq environment for `url`: the tools read the password from
|
||||
* PGPASSWORD, so it never appears in an argument list another process can
|
||||
* read. An sslmode that asks for SSL verifies the server's certificate, as the
|
||||
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
|
||||
* else the system's trusted CAs.
|
||||
*/
|
||||
export function pgConnectionEnv(url: string): Record<string, string> {
|
||||
const parsed = new URL(url);
|
||||
const env: Record<string, string> = {
|
||||
PGHOST: parsed.hostname.replace(/^\[(.*)\]$/, '$1'),
|
||||
PGPORT: parsed.port || '5432',
|
||||
PGDATABASE: decodeURIComponent(parsed.pathname.replace(/^\//, '')),
|
||||
PGCONNECT_TIMEOUT: CONNECT_TIMEOUT_SECONDS,
|
||||
};
|
||||
if (parsed.username) env.PGUSER = decodeURIComponent(parsed.username);
|
||||
if (parsed.password) env.PGPASSWORD = decodeURIComponent(parsed.password);
|
||||
const sslmode = parsed.searchParams.get('sslmode');
|
||||
if (sslmode === 'disable') env.PGSSLMODE = 'disable';
|
||||
else if (sslmode) {
|
||||
env.PGSSLMODE = 'verify-full';
|
||||
env.PGSSLROOTCERT = parsed.searchParams.get('sslrootcert') ?? 'system';
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
/** A client tool, refused with a clear message when it is not installed. */
|
||||
export class PgToolMissingError extends Error {
|
||||
constructor(tool: PgTool) {
|
||||
super(
|
||||
`${tool} was not found, so a Postgres database cannot be backed up or restored. ` +
|
||||
'Install the PostgreSQL client tools (the Docker image has them) or point TREK_DB_PG_BIN_DIR at them.',
|
||||
);
|
||||
this.name = 'PgToolMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
|
||||
function summarizeStderr(stderr: string | undefined): string {
|
||||
const kept = (stderr ?? '')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim() !== '' && !DATA_LINE.test(line))
|
||||
.join('\n');
|
||||
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `tool` by its absolute path and resolves with its stdout. A failure
|
||||
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
|
||||
* the cause.
|
||||
*/
|
||||
export async function run(
|
||||
tool: PgTool,
|
||||
binDir: string | undefined,
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, args, {
|
||||
env,
|
||||
timeout: TOOL_TIMEOUT_MS,
|
||||
maxBuffer: TOOL_MAX_OUTPUT_BYTES,
|
||||
});
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const message = summarizeStderr((err as { stderr?: string }).stderr);
|
||||
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
|
||||
import { CLEAR_SCHEMA_SQL } from './clear-schema';
|
||||
import { pgConnectionEnv, run } from './pg-process';
|
||||
import { refuseToolsWithoutRestrict } from './restore-safety';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import readline from 'node:readline';
|
||||
|
||||
/**
|
||||
* Restore's session settings: a restore waiting on a lock another session
|
||||
* holds gives up, and rolls back, instead of hanging; the NOTICEs every
|
||||
* CASCADE prints stay out of its stderr.
|
||||
*/
|
||||
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
|
||||
|
||||
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
|
||||
function createPrivateFile(file: string): void {
|
||||
fs.writeFileSync(file, '', { mode: 0o600 });
|
||||
}
|
||||
|
||||
/** The schema TREK's tables live in: its name, and the name as SQL quotes it (`quote_ident`). */
|
||||
interface PgSchema {
|
||||
name: string;
|
||||
quoted: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The current schema of the database at `url`, the one TREK creates its
|
||||
* tables in. A backup and a restore stay inside it: a hosted database
|
||||
* (Supabase) keeps the platform's own schemas, `auth` among them, beside it,
|
||||
* and they are neither TREK's to copy nor to replace.
|
||||
*/
|
||||
async function pgCurrentSchema(url: string, binDir: string | undefined): Promise<PgSchema> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--no-align',
|
||||
'--tuples-only',
|
||||
"--command=select concat_ws(E'\\t', current_schema(), quote_ident(current_schema()))",
|
||||
],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
const [name, quoted] = out.replace(/\n$/, '').split('\t');
|
||||
if (!name || !quoted) {
|
||||
throw new Error('The database has no current schema: set the search_path of the role in TREK_DB_URL.');
|
||||
}
|
||||
return { name, quoted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a custom-format dump of the current schema of the database at `url`
|
||||
* to `target`, readable by its owner only. It is written beside the target and
|
||||
* renamed into place, so a dump that fails or is killed never leaves a file
|
||||
* that passes for a copy.
|
||||
*/
|
||||
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
|
||||
const partial = `${target}.partial`;
|
||||
try {
|
||||
createPrivateFile(partial);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
await run(
|
||||
'pg_dump',
|
||||
binDir,
|
||||
[
|
||||
'--format=custom',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
`--file=${partial}`,
|
||||
`--schema="${schema.name.replaceAll('"', '""')}"`,
|
||||
],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
fs.renameSync(partial, target);
|
||||
} catch (err) {
|
||||
fs.rmSync(partial, { force: true });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Escapes every character a RegExp would read as syntax. */
|
||||
function escapeRegExp(text: string): string {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the restore script creates TREK's migration table in `schema`: a
|
||||
* dump of TREK's own schema does. The script is read line by line and the
|
||||
* read stops at the first match: the COPY data that fills a large script is
|
||||
* never loaded, and a script past V8's maximum string length would not load
|
||||
* at all.
|
||||
*/
|
||||
async function scriptHoldsTrekSchema(script: string, schema: PgSchema): Promise<boolean> {
|
||||
const createsHistory = new RegExp(`^CREATE TABLE ${escapeRegExp(`${schema.quoted}.mikro_orm_migrations`)} \\(`);
|
||||
const input = fs.createReadStream(script, { encoding: 'utf8' });
|
||||
const lines = readline.createInterface({ input, crlfDelay: Infinity });
|
||||
try {
|
||||
for await (const line of lines) {
|
||||
if (createsHistory.test(line)) return true;
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
lines.close();
|
||||
input.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the current schema of the database at `url` with the dump at
|
||||
* `source`: pg_restore turns the dump's objects in that schema into a script
|
||||
* beside it (anything else the dump holds is left out), then psql clears the
|
||||
* schema and runs the script in one transaction, so a restore that fails
|
||||
* leaves the database as it was. A dump without TREK's tables in that schema
|
||||
* is refused before anything is cleared.
|
||||
*/
|
||||
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
|
||||
await refuseToolsWithoutRestrict(binDir);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
|
||||
try {
|
||||
const clear = path.join(workDir, 'clear-schema.sql');
|
||||
const script = path.join(workDir, 'restore.sql');
|
||||
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
|
||||
createPrivateFile(script);
|
||||
await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--no-owner', '--no-privileges', `--schema=${schema.name}`, `--file=${script}`, '--', source],
|
||||
{},
|
||||
);
|
||||
if (!(await scriptHoldsTrekSchema(script, schema))) {
|
||||
throw new Error(`The backup holds no TREK tables in the schema "${schema.name}"; nothing was replaced.`);
|
||||
}
|
||||
await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--quiet',
|
||||
'--single-transaction',
|
||||
'--set=ON_ERROR_STOP=1',
|
||||
`--output=${os.devNull}`,
|
||||
`--file=${clear}`,
|
||||
`--file=${script}`,
|
||||
],
|
||||
{ ...pgConnectionEnv(url), PGOPTIONS: RESTORE_SESSION_OPTIONS },
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** The tables in the current schema of the database at `url`, read before TREK opens it. */
|
||||
export async function pgTableNames(url: string, binDir: string | undefined): Promise<string[]> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
['--no-psqlrc', '--no-align', '--tuples-only', `--command=${CURRENT_SCHEMA_TABLES_SQL}`],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
return out.split('\n').filter((line) => line !== '');
|
||||
}
|
||||
|
||||
/**
|
||||
* The migration names in a restore script's COPY block for
|
||||
* `mikro_orm_migrations`, read from its `name` column; none when the script
|
||||
* has no such block.
|
||||
*/
|
||||
function migrationNamesFromCopy(script: string): string[] {
|
||||
const lines = script.split('\n');
|
||||
const copy = lines.findIndex((line) => /^COPY \S*mikro_orm_migrations \(/.test(line));
|
||||
if (copy < 0) return [];
|
||||
const columns =
|
||||
/\(([^)]*)\)/
|
||||
.exec(lines[copy])?.[1]
|
||||
.split(',')
|
||||
.map((column) => column.trim()) ?? [];
|
||||
const nameAt = columns.indexOf('name');
|
||||
if (nameAt < 0) return [];
|
||||
const names: string[] = [];
|
||||
for (const line of lines.slice(copy + 1)) {
|
||||
if (line === '\\.' || line === '') break;
|
||||
const name = line.split('\t')[nameAt];
|
||||
if (name) names.push(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* The migrations a dump records as executed, read from the dump itself (its
|
||||
* `mikro_orm_migrations` rows as COPY text), so a restore can refuse a newer
|
||||
* TREK's backup before anything is replaced. Throws on a file that is not a
|
||||
* dump; an empty list means the dump holds no TREK history.
|
||||
*/
|
||||
export async function pgDumpMigrations(source: string, binDir: string | undefined): Promise<string[]> {
|
||||
const script = await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--data-only', '--table=mikro_orm_migrations', '--file=-', '--', source],
|
||||
{},
|
||||
);
|
||||
return migrationNamesFromCopy(script);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { type Configuration } from '@mikro-orm/core';
|
||||
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
|
||||
|
||||
/** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */
|
||||
const INT8_OID = 20;
|
||||
/** `numeric`: what `sum()` of a bigint and `avg()` answer. */
|
||||
const NUMERIC_OID = 1700;
|
||||
|
||||
/**
|
||||
* Reads a `bigint` as a JavaScript number, the way better-sqlite3 hands
|
||||
* SQLite's 64-bit INTEGER to the services. `pg` returns it as a string to
|
||||
* keep precision past 2^53; no TREK value comes near that (ids, counts,
|
||||
* epoch milliseconds, cents), so one that does is refused rather than
|
||||
* rounded.
|
||||
*/
|
||||
export function parseInt8(text: string): number {
|
||||
const value = Number(text);
|
||||
if (!Number.isSafeInteger(value)) throw new Error(`A bigint of ${text} is beyond what TREK reads exactly (2^53).`);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a `numeric` as a number, as SQLite answers `sum()` and `avg()`. `NaN`
|
||||
* and the infinities are refused, as is an integral result past 2^53, which is
|
||||
* refused like a bigint; a fraction is a double, the precision SQLite's REAL has.
|
||||
*/
|
||||
export function parseNumeric(text: string): number {
|
||||
const value = Number(text);
|
||||
if (!Number.isFinite(value)) {
|
||||
throw new Error(`A numeric of ${text} is not a finite number, which TREK does not read.`);
|
||||
}
|
||||
if (/^-?\d+$/.test(text) && !Number.isSafeInteger(value)) {
|
||||
throw new Error(`A numeric of ${text} is beyond what TREK reads exactly (2^53).`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's `PostgreSqlConnection`, with the pool's own type parsers replaced
|
||||
* for `int8` and `numeric`. MikroORM keeps dates as text, so only these two
|
||||
* change. `mapOptions` builds the `pg` pool options, whose `types` is the
|
||||
* `pg` TypeOverrides, the object that holds the parsers.
|
||||
*/
|
||||
class NumberTypesConnection extends PostgreSqlConnection {
|
||||
override mapOptions(overrides: Parameters<PostgreSqlConnection['mapOptions']>[0]) {
|
||||
const options = super.mapOptions(overrides);
|
||||
const types = options.types as { setTypeParser(oid: number, parse: (text: string) => unknown): void };
|
||||
types.setTypeParser(INT8_OID, parseInt8);
|
||||
types.setTypeParser(NUMERIC_OID, parseNumeric);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
|
||||
* as numbers. The parsers belong to this pool only: nothing global in `pg`
|
||||
* changes.
|
||||
*/
|
||||
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
|
||||
constructor(config: Configuration) {
|
||||
super(config);
|
||||
// PostgreSqlDriver's constructor hardcodes its connection class, as
|
||||
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
|
||||
// field, which is not public, so it is replaced here, before anything connects.
|
||||
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { DbEngineSettings } from '../../../app-config/db-settings';
|
||||
import { NO_SAFETY_NET } from '../../legacy-baseline';
|
||||
import { finishSchemaBootstrap, type AnyOrm } from '../../orm';
|
||||
import type { DatabaseEngine } from '../engine';
|
||||
import { redactUrl } from '../redact-url';
|
||||
import { TrekPostgreSqlDriver } from './postgres-driver';
|
||||
import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap';
|
||||
import { sslOptionFor } from './ssl';
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */
|
||||
type PostgresSettings = Extract<DbEngineSettings, { engine: 'postgres' }>;
|
||||
|
||||
/** The advisory lock key every write transaction takes: "TREK" in ASCII. */
|
||||
const TREK_WRITE_LOCK = 0x5452454b;
|
||||
|
||||
/**
|
||||
* The Postgres engine: MikroORM's own `pg` driver on `TREK_DB_URL`, with a
|
||||
* pool of at most `TREK_DB_POOL_MAX` connections that keeps none open while
|
||||
* idle. The pool belongs to the ORM, which the Nest container closes; the
|
||||
* schema comes from the baseline plus the migrations after it
|
||||
* (`schema-bootstrap.ts`).
|
||||
*/
|
||||
export class PostgresEngine implements DatabaseEngine {
|
||||
readonly id = 'postgres';
|
||||
|
||||
constructor(
|
||||
private readonly settings: PostgresSettings,
|
||||
private readonly bootstrapDeps: (url: string) => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
|
||||
) {}
|
||||
|
||||
private onReopened: (() => Promise<void>) | null = null;
|
||||
|
||||
/**
|
||||
* Nothing of its own to open: the ORM connects the pool. Keeps what a reopen
|
||||
* runs. A reopen replaces the database's contents, which the pool's own
|
||||
* connections keep reaching, so the ORM is not pointed at anything new and
|
||||
* the pool is never closed (ending a pool waits for every connection it has
|
||||
* lent out, and one lent to a job during the restore would never come back).
|
||||
*/
|
||||
open(_orm: AnyOrm, onReopened: () => Promise<void>): void {
|
||||
this.onReopened = onReopened;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
// Nothing of its own to close: the pool goes with the ORM.
|
||||
}
|
||||
|
||||
/**
|
||||
* After `pg_restore` replaced the database's contents: the restored
|
||||
* database is migrated forward on the same pool (`onReopened`, which
|
||||
* `DatabaseLifecycle` hands to `open`).
|
||||
*/
|
||||
async reopen(): Promise<void> {
|
||||
if (!this.onReopened) throw new Error('The Postgres engine was reopened before it was opened.');
|
||||
await this.onReopened();
|
||||
}
|
||||
|
||||
/**
|
||||
* `pg_advisory_xact_lock` on one key for the whole database: held until the
|
||||
* transaction commits or rolls back, taken again by a savepoint in the same
|
||||
* session without waiting, and shared by every process on the database.
|
||||
*/
|
||||
async serializeWrites(em: EntityManager): Promise<void> {
|
||||
await em
|
||||
.getConnection()
|
||||
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
|
||||
}
|
||||
|
||||
/** `snapshot: false` (a restore, whose archive is the copy) migrates without a pg_dump first. */
|
||||
async schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
|
||||
const deps = this.bootstrapDeps(this.settings.url);
|
||||
await bootstrapPostgresSchema(orm, options.snapshot ? deps : { ...deps, safetyNet: NO_SAFETY_NET });
|
||||
await finishSchemaBootstrap(orm);
|
||||
}
|
||||
|
||||
describe(): string {
|
||||
return `Postgres ${redactUrl(this.settings.url)}`;
|
||||
}
|
||||
|
||||
ormOptions() {
|
||||
const ssl = sslOptionFor(this.settings.url);
|
||||
return {
|
||||
driver: TrekPostgreSqlDriver,
|
||||
clientUrl: this.settings.url,
|
||||
pool: { min: 0, max: this.settings.poolMax },
|
||||
driverOptions: ssl === undefined ? {} : { ssl },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* The SQLite schema's triggers as Postgres writes them. The entities describe
|
||||
* tables only, so the baseline (`scripts/generate-pg-baseline.ts`) adds these
|
||||
* after the tables, and `check:pg-baseline` fails while a trigger in the
|
||||
* migrated SQLite schema has no entry here. Each entry is the trigger's
|
||||
* function and the trigger that runs it, under the SQLite trigger's name.
|
||||
*/
|
||||
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
|
||||
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
|
||||
trg_release_stop_on_stay_delete: [
|
||||
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
|
||||
RETURN OLD;
|
||||
END
|
||||
$$`,
|
||||
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
|
||||
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
|
||||
],
|
||||
// A place that moved or was renamed drops the region it was resolved to (#2527).
|
||||
trg_place_regions_follow_place: [
|
||||
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
DELETE FROM place_regions WHERE place_id = NEW.id;
|
||||
RETURN NEW;
|
||||
END
|
||||
$$`,
|
||||
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
|
||||
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
|
||||
'EXECUTE FUNCTION trg_place_regions_follow_place()',
|
||||
],
|
||||
};
|
||||
@@ -0,0 +1,50 @@
|
||||
import { run } from './pg-process';
|
||||
|
||||
/**
|
||||
* The client-tool versions a restore accepts (CVE-2025-8714). A script from a
|
||||
* dump an admin uploaded is run by psql, which would also run a psql
|
||||
* meta-command (`\!` runs a shell command) smuggled into an object name. A
|
||||
* fixed pg_restore opens the script with `\restrict <random key>`, and a fixed
|
||||
* psql then refuses every meta-command until the matching `\unrestrict`.
|
||||
*
|
||||
* RESTRICT_SINCE holds, per major, the first minor that carries the fix. Every
|
||||
* major after the last one listed has it from its first release.
|
||||
*/
|
||||
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
|
||||
const LAST_MAJOR_WITHOUT_IT = Math.max(...Object.keys(RESTRICT_SINCE).map(Number));
|
||||
|
||||
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
|
||||
export function hasRestrictFix(version: string): boolean {
|
||||
const match = /\(PostgreSQL\) (\d+)\.(\d+)/.exec(version);
|
||||
if (!match) return false;
|
||||
const major = Number(match[1]);
|
||||
const minor = Number(match[2]);
|
||||
if (major > LAST_MAJOR_WITHOUT_IT) return true;
|
||||
const firstFixedMinor = RESTRICT_SINCE[major];
|
||||
return firstFixedMinor !== undefined && minor >= firstFixedMinor;
|
||||
}
|
||||
|
||||
/** The minimum versions as the error names them, newest major first: "17.6, 16.10, 15.14 or 14.19". */
|
||||
function minimumVersionsText(): string {
|
||||
const floors = Object.keys(RESTRICT_SINCE)
|
||||
.map(Number)
|
||||
.sort((a, b) => b - a)
|
||||
.map((major) => `${major}.${RESTRICT_SINCE[major]}`);
|
||||
const last = floors.pop() ?? '';
|
||||
return floors.length > 0 ? `${floors.join(', ')} or ${last}` : last;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses the restore when pg_restore or psql predates the fix. The tools'
|
||||
* own versions are checked, not the script, whose text the dump controls.
|
||||
*/
|
||||
export async function refuseToolsWithoutRestrict(binDir: string | undefined): Promise<void> {
|
||||
const tools = ['pg_restore', 'psql'] as const;
|
||||
const versions = await Promise.all(tools.map((tool) => run(tool, binDir, ['--version'], {})));
|
||||
const stale = tools.findIndex((_, i) => !hasRestrictFix(versions[i]));
|
||||
if (stale < 0) return;
|
||||
throw new Error(
|
||||
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least ${minimumVersionsText()} ` +
|
||||
'(the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* How the entities' schema is written in Postgres: the one definition the
|
||||
* Postgres baseline (`scripts/generate-pg-baseline.ts`), the schema parity
|
||||
* check and the probe (`scripts/pg-probe.ts`) share.
|
||||
*
|
||||
* The entities are generated from the migrated SQLite schema and a parity
|
||||
* test pins them to it (`npm run check:entities`), so they are the one
|
||||
* description of the tables both engines can read. MikroORM's schema
|
||||
* generator turns them into Postgres DDL after the adjustments below, which
|
||||
* keep the storage model the services and the dialect helpers are written
|
||||
* for (server/CLAUDE.md, "Storage model"):
|
||||
*
|
||||
* - timestamps stay TEXT;
|
||||
* - an INTEGER column is `bigint`: SQLite's INTEGER holds 64 bits, so a
|
||||
* column written with epoch milliseconds or a byte count past 2 GB would
|
||||
* overflow Postgres's 32-bit `integer` (the engine's driver reads
|
||||
* `bigint` back as a number, `postgres-driver.ts`);
|
||||
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`,
|
||||
* `CURRENT_TIMESTAMP` in a TEXT column) gets its Postgres equivalent, and
|
||||
* a column declared `COLLATE NOCASE` uses the `nocase` collation
|
||||
* `collateNoCase()` names on Postgres.
|
||||
*
|
||||
* The baseline keeps the foreign keys; the probe leaves them out, so a
|
||||
* statement it sends is never refused only because the sample row it writes
|
||||
* points at nothing. The triggers the SQLite schema has are in
|
||||
* `postgres-triggers.ts`.
|
||||
*/
|
||||
import { pgTimestampText, PG_UTC_NOW } from '../../dialect/platform';
|
||||
import { DbTimestampType } from '../../types';
|
||||
|
||||
/** SQLite's timestamp text on Postgres, for a column default. */
|
||||
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
|
||||
|
||||
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
|
||||
export const NOCASE_COLLATION_SQL =
|
||||
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
|
||||
|
||||
/** The Postgres spelling of a column default the entities carry in SQLite's. */
|
||||
export function portableDefault(raw: string): string {
|
||||
const text = raw
|
||||
.trim()
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
|
||||
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
|
||||
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** A column type SQLite stores as its 64-bit INTEGER. */
|
||||
const INTEGER_TYPE = /^(integer|int)$/i;
|
||||
|
||||
/**
|
||||
* The subset of MikroORM's property metadata the adjustments read and write.
|
||||
* An entity's `props` satisfy it as they are, so no cast is needed.
|
||||
*/
|
||||
export interface ColumnMeta {
|
||||
name: string;
|
||||
defaultRaw?: string;
|
||||
collation?: string;
|
||||
columnTypes?: string[];
|
||||
customType?: unknown;
|
||||
}
|
||||
|
||||
/** A timestamp column becomes text. Returns whether it changed. */
|
||||
function timestampAsText(column: ColumnMeta): boolean {
|
||||
if (!(column.customType instanceof DbTimestampType) || !column.columnTypes) return false;
|
||||
column.columnTypes = column.columnTypes.map(() => 'text');
|
||||
return true;
|
||||
}
|
||||
|
||||
/** An integer column becomes bigint. Returns whether it changed. */
|
||||
function integerAsBigint(column: ColumnMeta): boolean {
|
||||
if (!column.columnTypes?.some((type) => INTEGER_TYPE.test(type))) return false;
|
||||
column.columnTypes = column.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A SQLite-only default gets its Postgres spelling. Returns whether it changed. */
|
||||
function defaultAsPostgres(column: ColumnMeta): boolean {
|
||||
if (typeof column.defaultRaw !== 'string') return false;
|
||||
const next = portableDefault(column.defaultRaw);
|
||||
if (next === column.defaultRaw) return false;
|
||||
column.defaultRaw = next;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A NOCASE column uses the `nocase` collation. Returns whether it changed. */
|
||||
function nocaseAsCollation(column: ColumnMeta): boolean {
|
||||
if (column.collation?.toUpperCase() !== 'NOCASE') return false;
|
||||
column.collation = 'nocase';
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the adjustments above to an entity's properties, in place, and
|
||||
* returns the names of the properties it changed, for the log. A timestamp
|
||||
* column is text, so the integer adjustment never sees it.
|
||||
*/
|
||||
export function applyPostgresColumnAdjustments(properties: readonly ColumnMeta[]): string[] {
|
||||
const changed: string[] = [];
|
||||
for (const property of properties) {
|
||||
const typeChanged = timestampAsText(property) || integerAsBigint(property);
|
||||
const defaultChanged = defaultAsPostgres(property);
|
||||
const collationChanged = nocaseAsCollation(property);
|
||||
if (typeChanged || defaultChanged || collationChanged) changed.push(property.name);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
import { readEnv, type AppEnv } from '../../../app-config';
|
||||
import { resolveDataPaths } from '../../../app-config/data-paths';
|
||||
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
|
||||
import { knownMigrationNames } from '../../known-migrations';
|
||||
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
|
||||
import type { AnyOrm } from '../../orm';
|
||||
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
|
||||
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
|
||||
import { pgDump } from './pg-tools';
|
||||
import type { Connection } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const logger = new Logger('DB');
|
||||
|
||||
/** A Postgres copy is a pg_dump custom-format archive. */
|
||||
const DUMP_EXTENSION = '.pgdump';
|
||||
|
||||
/** What a Postgres bootstrap reads, passed in so a test can supply its own. */
|
||||
export interface PostgresBootstrapDeps {
|
||||
baseline: () => PostgresBaseline;
|
||||
known: Set<string>;
|
||||
safetyNet: MigrationSafetyNet;
|
||||
}
|
||||
|
||||
/**
|
||||
* The tables in the current schema. A database with no TREK migration history
|
||||
* must hold none: TREK creates its schema in an empty database and never
|
||||
* adopts tables something else made. Read before anything is written, so a
|
||||
* refused database is left exactly as it was.
|
||||
*/
|
||||
async function currentTables(connection: Connection): Promise<string[]> {
|
||||
const rows: Array<{ table_name: string }> = await connection.execute(CURRENT_SCHEMA_TABLES_SQL);
|
||||
return rows.map((row) => row.table_name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the schema in an empty Postgres database: the baseline's statements
|
||||
* and the record of every migration it stands for, in one transaction (DDL is
|
||||
* transactional on Postgres, so the database ends with all of it or none). A
|
||||
* database holding tables beside no TREK history is refused before anything is
|
||||
* written.
|
||||
*/
|
||||
async function createFromBaseline(
|
||||
connection: Connection,
|
||||
storage: ReturnType<Migrator['getStorage']>,
|
||||
tables: readonly string[],
|
||||
historyTable: string,
|
||||
baseline: () => PostgresBaseline,
|
||||
): Promise<void> {
|
||||
const foreign = tables.filter((table) => table !== historyTable);
|
||||
if (foreign.length > 0) {
|
||||
throw new Error(
|
||||
`${REFUSAL} the database holds ${foreign.length} table(s) (${foreign.slice(0, 5).join(', ')}) but no TREK ` +
|
||||
'migration history. Point TREK_DB_URL at an empty database.',
|
||||
);
|
||||
}
|
||||
await storage.ensureTable();
|
||||
const schema = baseline();
|
||||
logger.log(`Creating the schema in an empty Postgres database (through ${migrationLabel(schema.through)})`);
|
||||
await connection.transactional(async (trx) => {
|
||||
await connection.execute(schema.statements.join(';\n'), [], 'run', trx);
|
||||
await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx)));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a database whose recorded history names a migration this build does
|
||||
* not know: a newer TREK migrated it. The refusal names the copy taken before
|
||||
* that migration, when there is one.
|
||||
*/
|
||||
async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDeps): Promise<void> {
|
||||
const thisRelease = migrationLabel([...deps.known].sort().at(-1) ?? '');
|
||||
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings a Postgres database to the current schema: inspect what it holds,
|
||||
* then create the schema or verify the history it has, then migrate.
|
||||
*
|
||||
* - No migration recorded and no table: the baseline is created
|
||||
* (`createFromBaseline`).
|
||||
* - No migration recorded but tables present: refused, naming a few of them.
|
||||
* - Migrations recorded: refused when one is unknown to this build, else the
|
||||
* pending migrations run, after the safety net. A database the baseline just
|
||||
* created runs the migrations after the baseline's cut without it: there is
|
||||
* nothing yet to lose.
|
||||
*/
|
||||
export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstrapDeps): Promise<void> {
|
||||
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
|
||||
const connection = orm.em.getConnection();
|
||||
const historyTable = orm.config.get('migrations').tableName ?? 'mikro_orm_migrations';
|
||||
|
||||
const tables = await currentTables(connection);
|
||||
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
|
||||
|
||||
if (executed.length === 0) {
|
||||
await createFromBaseline(connection, migrator.getStorage(), tables, historyTable, deps.baseline);
|
||||
} else {
|
||||
await refuseNewerHistory(executed, deps);
|
||||
}
|
||||
|
||||
const pending = await migrator.getPending();
|
||||
if (pending.length === 0) return;
|
||||
// Only a database with history has a last migration to label its copy by; a
|
||||
// database the baseline just created has nothing yet to lose.
|
||||
const latest = [...executed].sort().at(-1);
|
||||
if (latest !== undefined) await deps.safetyNet.beforeMigrate(migrationLabel(latest));
|
||||
logger.log(`Applying ${pending.length} pending migration(s)`);
|
||||
await migrator.up();
|
||||
}
|
||||
|
||||
/** Dumps a killed boot left half-written: never a copy, only disk taken. */
|
||||
function removeOrphanedPartials(dir: string): void {
|
||||
for (const file of fs.readdirSync(dir)) {
|
||||
if (file.startsWith(PREFIX) && file.endsWith(`${DUMP_EXTENSION}.partial`)) {
|
||||
fs.rmSync(path.join(dir, file), { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres safety net before pending migrations, as the SQLite file's
|
||||
* (`db/pre-migrate-snapshot.ts`): a `pg_dump` of the database to
|
||||
* `<data>/pre-migrate-<schema>-<time>.pgdump`, the newest
|
||||
* `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` kept. A copy that cannot be written
|
||||
* (pg_dump missing, the data dir full) refuses the boot before anything is
|
||||
* migrated, unless `TREK_DB_PRE_MIGRATE_SNAPSHOT=false`. Off under
|
||||
* NODE_ENV=test unless a suite asks for it.
|
||||
*/
|
||||
export function postgresSafetyNet(
|
||||
url: string,
|
||||
env: () => AppEnv = readEnv,
|
||||
now: () => Date = () => new Date(),
|
||||
dataDir: () => string = () => resolveDataPaths(env()).dataDir,
|
||||
): MigrationSafetyNet {
|
||||
return {
|
||||
async beforeMigrate(from) {
|
||||
const { db, app } = env();
|
||||
if (db.preMigrateSnapshot === undefined && app.isTest) return;
|
||||
if (db.preMigrateSnapshot === false) {
|
||||
logger.warn('TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating the Postgres database without a copy');
|
||||
return;
|
||||
}
|
||||
const target = path.join(dataDir(), `${PREFIX}${from ?? 'unversioned'}-${stamp(now())}${DUMP_EXTENSION}`);
|
||||
try {
|
||||
fs.mkdirSync(dataDir(), { recursive: true });
|
||||
removeOrphanedPartials(dataDir());
|
||||
await pgDump(url, target, db.pgBinDir);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
|
||||
`(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Fix that, ` +
|
||||
'or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
logger.log(`Copy of the database before migrating: ${target}`);
|
||||
pruneSnapshots(dataDir(), db.preMigrateSnapshotKeep, DUMP_EXTENSION);
|
||||
},
|
||||
|
||||
async restoreHint(thisRelease) {
|
||||
const snapshots = listSnapshots(dataDir(), DUMP_EXTENSION);
|
||||
const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
|
||||
if (match) {
|
||||
return (
|
||||
`The copy taken right before the newer TREK migrated it is ${match}: stop TREK, empty the database ` +
|
||||
'(drop and recreate it), and restore it with pg_restore --no-owner --no-privileges --single-transaction.'
|
||||
);
|
||||
}
|
||||
if (snapshots.length > 0) {
|
||||
return `Copies taken before earlier migrations are in ${dataDir()} (${PREFIX}*${DUMP_EXTENSION}), none of them from this release.`;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
|
||||
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
|
||||
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Splits generated DDL into single statements, so one statement Postgres
|
||||
* refuses does not take the rest of the schema with it.
|
||||
*
|
||||
* Understood: `;` outside quotes, `'` and `"` quoted text, and the `$$` dollar
|
||||
* quote a function body is written in, which is the only dollar quote TREK's
|
||||
* generated DDL and baseline contain. A tagged dollar quote (`$tag$`) is not
|
||||
* understood, so it is refused rather than split inside.
|
||||
*/
|
||||
|
||||
/** A tagged dollar quote (`$tag$`), matched where the scan stands. */
|
||||
const TAGGED_DOLLAR_QUOTE = /\$[A-Za-z_]\w*\$/y;
|
||||
|
||||
function refuseTaggedDollarQuote(sql: string, at: number): void {
|
||||
TAGGED_DOLLAR_QUOTE.lastIndex = at;
|
||||
const tag = TAGGED_DOLLAR_QUOTE.exec(sql)?.[0];
|
||||
if (tag) throw new Error(`splitStatements understands only the $$ dollar quote, not ${tag} (at ${at})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* The statements in `sql`, trimmed and in order. A statement that sets a
|
||||
* session parameter (`SET ...`) is dropped: a dump's preamble, which a schema
|
||||
* the entities generate never needs.
|
||||
*/
|
||||
export function splitStatements(sql: string): string[] {
|
||||
const statements: string[] = [];
|
||||
let current = '';
|
||||
let quote: '"' | "'" | '$$' | null = null;
|
||||
for (let i = 0; i < sql.length; i++) {
|
||||
const char = sql[i];
|
||||
const dollar = sql.startsWith('$$', i);
|
||||
if (quote === '$$') {
|
||||
if (dollar) {
|
||||
quote = null;
|
||||
current += '$';
|
||||
i++;
|
||||
}
|
||||
current += char;
|
||||
} else if (quote) {
|
||||
if (char === quote) quote = null;
|
||||
current += char;
|
||||
} else if (dollar) {
|
||||
quote = '$$';
|
||||
current += '$$';
|
||||
i++;
|
||||
} else if (char === "'" || char === '"') {
|
||||
quote = char;
|
||||
current += char;
|
||||
} else if (char === ';') {
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
if (char === '$') refuseTaggedDollarQuote(sql, i);
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
return statements.filter((statement) => !/^set\s/i.test(statement));
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
/** The sslmode values that verify the server, read the way the `pg` driver reads them. */
|
||||
const VERIFYING_MODES = new Set(['require', 'verify-ca', 'verify-full']);
|
||||
|
||||
/** The `ssl` option of a `pg` pool, as `sslOptionFor` builds it. */
|
||||
type PgSslOption = false | { rejectUnauthorized: true; ca?: string };
|
||||
|
||||
/**
|
||||
* The `ssl` option for the pool, from the URL's `sslmode` and `sslrootcert`.
|
||||
*
|
||||
* MikroORM builds the pool from the URL's host, port, user, password and
|
||||
* database alone and drops the rest of the query, so an `sslmode` in
|
||||
* `TREK_DB_URL` would be ignored without this. `require`, `verify-ca` and
|
||||
* `verify-full` all verify the server's certificate, as the `pg` driver itself
|
||||
* treats them; `disable` turns SSL off, and no `sslmode` leaves the driver's
|
||||
* default (no SSL). A mode that would connect without verifying (`prefer`,
|
||||
* `allow`, `no-verify`) or that is misspelt refuses, rather than quietly
|
||||
* connecting with less than was asked. `sslrootcert` names the CA file a host
|
||||
* that signs its own certificates needs (Supabase's, say).
|
||||
*/
|
||||
export function sslOptionFor(url: string): PgSslOption | undefined {
|
||||
const params = new URL(url).searchParams;
|
||||
const mode = params.get('sslmode');
|
||||
const rootCert = params.get('sslrootcert');
|
||||
if (mode === null) {
|
||||
if (rootCert !== null) throw new Error('TREK_DB_URL names an sslrootcert without sslmode=verify-full.');
|
||||
return undefined;
|
||||
}
|
||||
if (mode === 'disable') return false;
|
||||
if (!VERIFYING_MODES.has(mode)) {
|
||||
throw new Error(
|
||||
`TREK_DB_URL has sslmode "${mode}"; use disable, require, verify-ca or verify-full (all but disable verify the server).`,
|
||||
);
|
||||
}
|
||||
if (rootCert === null) return { rejectUnauthorized: true };
|
||||
try {
|
||||
return { rejectUnauthorized: true, ca: readFileSync(rootCert, 'utf8') };
|
||||
} catch (error) {
|
||||
throw new Error(`TREK_DB_URL names sslrootcert ${rootCert}, which cannot be read.`, { cause: error });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* A connection URL fit for a log line: the password masked and the query
|
||||
* dropped, since libpq reads a password from the query too
|
||||
* (`?password=...`). A value that does not parse is not echoed at all.
|
||||
*/
|
||||
export function redactUrl(url: string): string {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return '<unparseable URL>';
|
||||
}
|
||||
if (parsed.password) parsed.password = '***';
|
||||
parsed.search = '';
|
||||
parsed.hash = '';
|
||||
return parsed.toString();
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { closeDb, getRawConnection, registerReinitializeHook, reinitialize } from '../../database';
|
||||
import { resolveDbPath } from '../../db-path';
|
||||
import { runSchemaBootstrap, type AnyOrm } from '../../orm';
|
||||
import { SharedSqliteDriver } from '../../orm-driver';
|
||||
import type { DatabaseEngine } from '../engine';
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* The SQLite file engine, TREK's default: one better-sqlite3 handle that
|
||||
* `db/database.ts` owns and the ORM's bound driver shares (`db/orm-driver.ts`),
|
||||
* on `TREK_DB_FILE` or `data/travel.db` (`:memory:` under test). A restore
|
||||
* swaps the file and reopens the handle; the schema comes from the migration
|
||||
* chain (`db/orm.ts`).
|
||||
*/
|
||||
export class SqliteEngine implements DatabaseEngine {
|
||||
readonly id = 'sqlite';
|
||||
|
||||
describe(): string {
|
||||
return `SQLite file ${resolveDbPath()}`;
|
||||
}
|
||||
|
||||
ormOptions() {
|
||||
return { driver: SharedSqliteDriver, dbName: resolveDbPath() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the file (normally a no-op: the ORM's first connect inside
|
||||
* NestFactory.create() already did, said here so the boot does not depend on
|
||||
* that), and registers the reopen. A reopen swaps the file under the same
|
||||
* handle, which Kysely caches, so the ORM's connection is closed and connected
|
||||
* again first: the driver then reaches the new handle through
|
||||
* `createKyselyDialect()`. The bound driver's `destroy()` is a no-op
|
||||
* (`orm-driver.ts`), so the close never reaches the handle `db/database.ts` owns.
|
||||
*/
|
||||
open(orm: AnyOrm, onReopened: () => Promise<void>): void {
|
||||
getRawConnection();
|
||||
registerReinitializeHook(async () => {
|
||||
const connection = orm.em.getConnection();
|
||||
await connection.close(true);
|
||||
await connection.connect();
|
||||
await onReopened();
|
||||
});
|
||||
}
|
||||
|
||||
close(): void {
|
||||
closeDb();
|
||||
}
|
||||
|
||||
reopen(): Promise<void> {
|
||||
return reinitialize();
|
||||
}
|
||||
|
||||
schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
|
||||
return runSchemaBootstrap(orm, options);
|
||||
}
|
||||
|
||||
/** A no-op: SQLite has one connection and one writer, so its transactions already run one at a time. */
|
||||
serializeWrites(_em: EntityManager): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { EntityManager } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* The SQLite engine's whole-file maintenance: the two statements a backup,
|
||||
* a restore and the demo reset need that are about the database file rather
|
||||
* than any table (`PRAGMA wal_checkpoint`, `VACUUM INTO`). Neither has a
|
||||
* Postgres spelling, so they live under the engine that owns the file instead
|
||||
* of in `MaintenanceRepository`, whose remaining statements run on every
|
||||
* engine (and which the Postgres probe calls method by method).
|
||||
*
|
||||
* Provided through Nest DI by `MaintenanceModule` (`nest/database/`), a
|
||||
* factory over the context-resolving global `EntityManager`, and injected by
|
||||
* the SQLite backup port (`SqliteDatabaseBackup`). Free of Nest decorators
|
||||
* like every other file under `db/`, so a test can build one over its own
|
||||
* `EntityManager`.
|
||||
*/
|
||||
export class SqliteMaintenance {
|
||||
constructor(private readonly em: EntityManager) {}
|
||||
|
||||
/**
|
||||
* MikroORM's own validating `getContext()`: throws outside a request
|
||||
* context, the fail-closed check `TrekRepository` applies to every
|
||||
* repository, since a raw `connection.execute()` has no such guard.
|
||||
*/
|
||||
private validateRequestContext(): void {
|
||||
this.em.getContext();
|
||||
}
|
||||
|
||||
/**
|
||||
* `PRAGMA wal_checkpoint(TRUNCATE)`: flushes the WAL into the main file
|
||||
* before a snapshot or a swap. Reached through the backup port's
|
||||
* `checkpoint()`; every caller treats it as best effort.
|
||||
*/
|
||||
async walCheckpoint(): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
await this.em.getConnection().execute('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/**
|
||||
* `` VACUUM INTO '<path>' ``: a consistent point-in-time copy of the file
|
||||
* under concurrent writers, reached through the backup port's
|
||||
* `snapshot()`. Single quotes in `path` are escaped by doubling.
|
||||
*/
|
||||
async vacuumInto(path: string): Promise<void> {
|
||||
this.validateRequestContext();
|
||||
const escaped = path.replaceAll("'", "''");
|
||||
await this.em.getConnection().execute(`VACUUM INTO '${escaped}'`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user