mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-22 03:50:30 +00:00
feat(db): back up, restore and copy a Postgres database before migrating it
On Postgres the backup port dumps with pg_dump and restores by turning the dump into a script with pg_restore and having psql clear the current schema and run it in one transaction. A dump only recreates what it holds, so a plain pg_restore --clean would leave a later migration's tables behind and break the migrations the reopen runs; a restore that fails leaves the database as it was. A restore refuses a pg_restore or psql older than 17.6/16.10/15.14/14.19, whose script would lack the \restrict line that keeps a psql meta-command smuggled into an uploaded dump from running (CVE-2025-8714). A restore refuses a SQLite archive, a dump without TREK's migration history or one a newer TREK wrote before anything is replaced, and keeps a dump of the database it replaces. RESTORE_FROM_BACKUP loads the archive's dump the same way into a database that holds no table yet, and is ignored with a warning once it holds one. Before pending migrations the boot now takes a pg_dump copy to data/pre-migrate-<last migration>-<time>.pgdump under the same TREK_DB_PRE_MIGRATE_SNAPSHOT(_KEEP) policy as the SQLite file, instead of refusing; a copy that cannot be written refuses the boot. The tools run by absolute path (TREK_DB_PG_BIN_DIR, or the usual directories), never through PATH, with the password in PGPASSWORD; an sslmode that asks for SSL verifies the certificate as the app pool does. Dumps are written beside their target and renamed into place, readable by their owner only, and a tool's stderr reaches errors without the lines that quote SQL or rows. The image installs postgresql-client-17 from Debian 13.
This commit is contained in:
+6
-1
@@ -70,9 +70,14 @@ COPY server/package.json ./server/
|
||||
# gypfile:false (npm/cli#9837). The only other install scripts among the runtime
|
||||
# deps are esbuild's binary check, redundant next to its optional platform
|
||||
# package, and a telemetry ping.
|
||||
#
|
||||
# postgresql-client-17 (Debian 13's own) brings pg_dump and pg_restore to
|
||||
# /usr/lib/postgresql/17/bin, which back up an install on Postgres
|
||||
# (TREK_DB_URL). pg_dump refuses a server newer than itself, so it stays at the
|
||||
# newest major TREK supports.
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends tzdata dumb-init wget ca-certificates \
|
||||
libkitinerary-bin libsqlite3-0 && \
|
||||
libkitinerary-bin libsqlite3-0 postgresql-client-17 && \
|
||||
npm ci --workspace=server --omit=dev --ignore-scripts && \
|
||||
{ ln -sf "$(find /usr/lib -name kitinerary-extractor -type f | head -1)" /usr/local/bin/kitinerary-extractor || true; } && \
|
||||
rm -rf /var/lib/apt/lists/* /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx && \
|
||||
|
||||
+2
-1
@@ -47,12 +47,13 @@ DEMO_MODE=false # Demo mode: resets data hourly
|
||||
# DEMO_ADMIN_EMAIL=admin@trek.app # Email of that seeded demo admin. (default: admin@trek.app)
|
||||
# DEMO_ADMIN_PASS=admin12345 # Initial password of that seeded demo admin. Change it on a public demo. (default: admin12345)
|
||||
|
||||
# RESTORE_FROM_BACKUP=/app/data/backup.zip # Restore this backup on the first start, before any setup. Ignored once a database exists.
|
||||
# RESTORE_FROM_BACKUP=/app/data/backup.zip # Restore this backup on the first start, before any setup. Ignored once a database exists (on Postgres: once the database holds a table).
|
||||
# FILE_UPLOAD_LIMIT_MB=50 # Max size (MB) of a file uploaded to a trip, a booking or a note. Videos keep their own 500 MB cap. Raise your reverse proxy's upload limit too.
|
||||
# BACKUP_MAX_DECOMPRESSED_MB=5120 # Max size (MB) a restore archive may unpack to, on both restore paths: the zip-bomb guard. Raise it if a large backup is refused with "Backup exceeds the maximum decompressed size". (default: 5120)
|
||||
# BACKUP_UPLOAD_LIMIT_MB=500 # Max size (MB) of a backup archive you can upload when restoring. Raise it if your backup exceeds 500 MB. If you sit behind a reverse proxy, raise its upload limit too (e.g. nginx client_max_body_size).
|
||||
|
||||
# TREK_DB_URL=postgres://trek:password@db:5432/trek # Run on a Postgres database instead of the SQLite file in data/. TREK creates its schema in an empty database and refuses one holding tables it did not create. sslmode=disable, require, verify-ca or verify-full (the last three always verify the certificate; sslrootcert= names the CA file). Cannot be combined with TREK_DB_FILE, TREK_DB_JOURNAL_MODE or TREK_DB_SYNCHRONOUS.
|
||||
# TREK_DB_PG_BIN_DIR=/usr/lib/postgresql/17/bin # The directory holding pg_dump, pg_restore and psql, which back up and restore a Postgres database. Unset, TREK looks in /usr/lib/postgresql/<17..14>/bin, /usr/local/bin and /usr/bin (the Docker image installs the version 17 tools, which cannot dump a newer server). Unused on SQLite.
|
||||
# TREK_DB_POOL_MAX=10 # Most connections TREK opens to the Postgres database (1 to 100). Unused on SQLite. (default: 10)
|
||||
# TREK_DB_JOURNAL_MODE=WAL # SQLite journal mode (DELETE, TRUNCATE, PERSIST, MEMORY, WAL, OFF). WAL is fastest and the right choice on local disk. Set DELETE if the data directory lives on network storage (Azure App Service, an SMB/NFS volume on a NAS) — WAL coordinates writers through a shared-memory file that those filesystems do not implement reliably. The mode is stored inside the database file, so it survives a restart; changing this applies on the next boot.
|
||||
# TREK_DB_PRE_MIGRATE_SNAPSHOT=true # Copy the database to data/pre-migrate-<schema>-<time>.db before a startup migrates it, so the previous image can run on the copy again. A copy that cannot be written stops the startup; false migrates without one.
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
@@ -346,6 +346,8 @@ export function deriveDb(raw: RawEnv) {
|
||||
url,
|
||||
/** The largest number of connections the Postgres pool opens. */
|
||||
poolMax: Math.min(positiveIntOr(raw.TREK_DB_POOL_MAX, 10), 100),
|
||||
/** Where pg_dump and pg_restore are; unset, the usual install directories are tried (never PATH). */
|
||||
pgBinDir: raw.TREK_DB_PG_BIN_DIR?.trim() || undefined,
|
||||
trekDbFile: raw.TREK_DB_FILE,
|
||||
/** Resolved journal_mode — WAL unless the operator asked for something else (network storage needs DELETE/TRUNCATE). */
|
||||
journalMode: durability.journalMode,
|
||||
|
||||
@@ -168,6 +168,7 @@ export const envSchema = z.object({
|
||||
// carries the password), so it is masked in the boot report.
|
||||
TREK_DB_URL: postgresUrl,
|
||||
TREK_DB_POOL_MAX: integer(1, 100),
|
||||
TREK_DB_PG_BIN_DIR: anyString,
|
||||
TREK_DB_FILE: anyString,
|
||||
// The two SQLite pragmas are deliberately unvalidated: a typo here must fall
|
||||
// back and warn rather than abort boot, because reset-admin.js — the way out
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** The Postgres client tools a backup runs. */
|
||||
type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
|
||||
|
||||
/**
|
||||
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
|
||||
* Debian's versioned directories newest first (the Docker image installs 17),
|
||||
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
|
||||
*/
|
||||
const DEFAULT_BIN_DIRS = [
|
||||
'/usr/lib/postgresql/17/bin',
|
||||
'/usr/lib/postgresql/16/bin',
|
||||
'/usr/lib/postgresql/15/bin',
|
||||
'/usr/lib/postgresql/14/bin',
|
||||
'/usr/local/bin',
|
||||
'/usr/bin',
|
||||
];
|
||||
|
||||
/** A dump or a restore of a large database takes minutes, not hours. */
|
||||
const TOOL_TIMEOUT_MS = 30 * 60 * 1000;
|
||||
const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
||||
/** An unreachable host fails the tool in seconds instead of holding a boot or a restore. */
|
||||
const CONNECT_TIMEOUT_SECONDS = '10';
|
||||
/**
|
||||
* A restore waiting on a lock another session holds gives up, and rolls back,
|
||||
* instead of hanging; the NOTICEs every CASCADE prints stay out of its stderr.
|
||||
*/
|
||||
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
|
||||
/** How much of a tool's stderr an error keeps. */
|
||||
const STDERR_LIMIT = 2000;
|
||||
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
|
||||
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
|
||||
|
||||
/**
|
||||
* Drops every table, view, sequence, routine, collation and standalone type in
|
||||
* the current schema, except an extension's, before a dump is loaded: a dump only recreates what it
|
||||
* holds, so whatever a later migration added would otherwise survive the
|
||||
* restore and break the migrations the reopen runs. It runs in the same
|
||||
* transaction as the load, so a restore that fails leaves the database as it
|
||||
* was.
|
||||
*/
|
||||
const CLEAR_SCHEMA_SQL = `DO $clear$
|
||||
DECLARE r record;
|
||||
BEGIN
|
||||
FOR r IN SELECT c.oid::regclass AS name, c.relkind FROM pg_class c
|
||||
WHERE c.relnamespace = current_schema()::regnamespace AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype IN ('e', 'a', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP %s IF EXISTS %s CASCADE',
|
||||
CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW' WHEN 'S' THEN 'SEQUENCE'
|
||||
WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END, r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT p.oid::regprocedure AS sig FROM pg_proc p
|
||||
WHERE p.pronamespace = current_schema()::regnamespace
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = p.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP ROUTINE IF EXISTS %s CASCADE', r.sig);
|
||||
END LOOP;
|
||||
FOR r IN SELECT format('%I.%I', n.nspname, c.collname) AS name FROM pg_collation c
|
||||
JOIN pg_namespace n ON n.oid = c.collnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = c.oid AND d.deptype = 'e')
|
||||
LOOP
|
||||
EXECUTE format('DROP COLLATION IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
FOR r IN SELECT t.oid::regtype AS name FROM pg_type t
|
||||
WHERE t.typnamespace = current_schema()::regnamespace AND t.typtype IN ('e', 'd', 'r')
|
||||
AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.objid = t.oid AND d.deptype IN ('e', 'i'))
|
||||
LOOP
|
||||
EXECUTE format('DROP TYPE IF EXISTS %s CASCADE', r.name);
|
||||
END LOOP;
|
||||
END
|
||||
$clear$;
|
||||
`;
|
||||
|
||||
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
|
||||
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
|
||||
const dirs = binDir ? [binDir] : DEFAULT_BIN_DIRS;
|
||||
const executable = (candidate: string): boolean => {
|
||||
try {
|
||||
fs.accessSync(candidate, fs.constants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false; // missing or not executable: the next directory may have it
|
||||
}
|
||||
};
|
||||
return dirs.map((dir) => path.join(dir, tool)).find(executable) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The libpq environment for `url`: the tools read the password from
|
||||
* PGPASSWORD, so it never appears in an argument list another process can
|
||||
* read. An sslmode that asks for SSL verifies the server's certificate, as the
|
||||
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
|
||||
* else the system's trusted CAs.
|
||||
*/
|
||||
export function pgConnectionEnv(url: string): Record<string, string> {
|
||||
const parsed = new URL(url);
|
||||
const env: Record<string, string> = {
|
||||
PGHOST: parsed.hostname.replace(/^\[(.*)\]$/, '$1'),
|
||||
PGPORT: parsed.port || '5432',
|
||||
PGDATABASE: decodeURIComponent(parsed.pathname.replace(/^\//, '')),
|
||||
PGCONNECT_TIMEOUT: CONNECT_TIMEOUT_SECONDS,
|
||||
};
|
||||
if (parsed.username) env.PGUSER = decodeURIComponent(parsed.username);
|
||||
if (parsed.password) env.PGPASSWORD = decodeURIComponent(parsed.password);
|
||||
const sslmode = parsed.searchParams.get('sslmode');
|
||||
if (sslmode === 'disable') env.PGSSLMODE = 'disable';
|
||||
else if (sslmode) {
|
||||
env.PGSSLMODE = 'verify-full';
|
||||
env.PGSSLROOTCERT = parsed.searchParams.get('sslrootcert') ?? 'system';
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
/** A client tool, refused with a clear message when it is not installed. */
|
||||
export class PgToolMissingError extends Error {
|
||||
constructor(tool: PgTool) {
|
||||
super(
|
||||
`${tool} was not found, so a Postgres database cannot be backed up or restored. ` +
|
||||
'Install the PostgreSQL client tools (the Docker image has them) or point TREK_DB_PG_BIN_DIR at them.',
|
||||
);
|
||||
this.name = 'PgToolMissingError';
|
||||
}
|
||||
}
|
||||
|
||||
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
|
||||
function summarizeStderr(stderr: string | undefined): string {
|
||||
const kept = (stderr ?? '')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim() !== '' && !DATA_LINE.test(line))
|
||||
.join('\n');
|
||||
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
|
||||
}
|
||||
|
||||
async function run(
|
||||
tool: PgTool,
|
||||
binDir: string | undefined,
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, args, {
|
||||
env,
|
||||
timeout: TOOL_TIMEOUT_MS,
|
||||
maxBuffer: TOOL_MAX_OUTPUT_BYTES,
|
||||
});
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
// stderr is the tool's own message; the password is in env, never in it.
|
||||
const message = summarizeStderr((err as { stderr?: string }).stderr);
|
||||
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
|
||||
function createPrivateFile(file: string): void {
|
||||
fs.writeFileSync(file, '', { mode: 0o600 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a custom-format dump of the database at `url` to `target`, readable
|
||||
* by its owner only. It is written beside the target and renamed into place,
|
||||
* so a dump that fails or is killed never leaves a file that passes for a copy.
|
||||
*/
|
||||
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
|
||||
const partial = `${target}.partial`;
|
||||
try {
|
||||
createPrivateFile(partial);
|
||||
await run(
|
||||
'pg_dump',
|
||||
binDir,
|
||||
['--format=custom', '--no-owner', '--no-privileges', `--file=${partial}`],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
fs.renameSync(partial, target);
|
||||
} catch (err) {
|
||||
fs.rmSync(partial, { force: true });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first minor of each major whose pg_restore and psql carry the fix for
|
||||
* CVE-2025-8714; every major after 17 has it from its first release.
|
||||
*/
|
||||
const RESTRICT_SINCE: Readonly<Record<number, number>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
|
||||
const LAST_MAJOR_WITHOUT_IT = 17;
|
||||
const OLDEST_MAJOR = 14;
|
||||
|
||||
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
|
||||
export function hasRestrictFix(version: string): boolean {
|
||||
const match = /\(PostgreSQL\) (\d+)\.(\d+)/.exec(version);
|
||||
if (!match) return false;
|
||||
const major = Number(match[1]);
|
||||
const minor = Number(match[2]);
|
||||
if (major > LAST_MAJOR_WITHOUT_IT) return true;
|
||||
return major >= OLDEST_MAJOR && minor >= RESTRICT_SINCE[major];
|
||||
}
|
||||
|
||||
/**
|
||||
* A script from a dump an admin uploaded is run by psql, which would also run
|
||||
* a psql meta-command (`\!` runs a shell command) smuggled into an object
|
||||
* name. A fixed pg_restore opens the script with `\restrict <random key>`, and
|
||||
* a fixed psql then refuses every meta-command until the matching
|
||||
* `\unrestrict` (CVE-2025-8714). The tools' own versions are checked, not the
|
||||
* script, whose text the dump controls.
|
||||
*/
|
||||
async function refuseToolsWithoutRestrict(binDir: string | undefined): Promise<void> {
|
||||
const tools = ['pg_restore', 'psql'] as const;
|
||||
const versions = await Promise.all(tools.map((tool) => run(tool, binDir, ['--version'], {})));
|
||||
const stale = tools.findIndex((_, i) => !hasRestrictFix(versions[i]));
|
||||
if (stale >= 0) {
|
||||
throw new Error(
|
||||
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least 17.6, 16.10, ` +
|
||||
'15.14 or 14.19 (the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the database at `url` with the dump at `source`: pg_restore turns
|
||||
* the dump into a script beside it, then psql clears the current schema and
|
||||
* runs the script in one transaction, so a restore that fails leaves the
|
||||
* database as it was.
|
||||
*/
|
||||
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
|
||||
await refuseToolsWithoutRestrict(binDir);
|
||||
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
|
||||
try {
|
||||
const clear = path.join(workDir, 'clear-schema.sql');
|
||||
const script = path.join(workDir, 'restore.sql');
|
||||
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
|
||||
createPrivateFile(script);
|
||||
await run('pg_restore', binDir, ['--no-owner', '--no-privileges', `--file=${script}`, '--', source], {});
|
||||
await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--quiet',
|
||||
'--single-transaction',
|
||||
'--set=ON_ERROR_STOP=1',
|
||||
'--output=/dev/null',
|
||||
`--file=${clear}`,
|
||||
`--file=${script}`,
|
||||
],
|
||||
{ ...pgConnectionEnv(url), PGOPTIONS: RESTORE_SESSION_OPTIONS },
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** The tables in the current schema of the database at `url`, read before TREK opens it. */
|
||||
export async function pgTableNames(url: string, binDir: string | undefined): Promise<string[]> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--no-align',
|
||||
'--tuples-only',
|
||||
'--command=select table_name from information_schema.tables where table_schema = current_schema()',
|
||||
],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
return out.split('\n').filter((line) => line !== '');
|
||||
}
|
||||
|
||||
/**
|
||||
* The migrations a dump records as executed, read from the dump itself (its
|
||||
* `mikro_orm_migrations` rows as COPY text), so a restore can refuse a newer
|
||||
* TREK's backup before anything is replaced. Throws on a file that is not a
|
||||
* dump; an empty list means the dump holds no TREK history.
|
||||
*/
|
||||
export async function pgDumpMigrations(source: string, binDir: string | undefined): Promise<string[]> {
|
||||
const script = await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--data-only', '--table=mikro_orm_migrations', '--file=-', '--', source],
|
||||
{},
|
||||
);
|
||||
const lines = script.split('\n');
|
||||
const copy = lines.findIndex((line) => /^COPY \S*mikro_orm_migrations \(/.test(line));
|
||||
if (copy < 0) return [];
|
||||
const columns =
|
||||
/\(([^)]*)\)/
|
||||
.exec(lines[copy])?.[1]
|
||||
.split(',')
|
||||
.map((column) => column.trim()) ?? [];
|
||||
const nameAt = columns.indexOf('name');
|
||||
if (nameAt < 0) return [];
|
||||
const names: string[] = [];
|
||||
for (const line of lines.slice(copy + 1)) {
|
||||
if (line === '\\.' || line === '') break;
|
||||
const name = line.split('\t')[nameAt];
|
||||
if (name) names.push(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { NO_SAFETY_NET } from '../../legacy-baseline';
|
||||
import { finishSchemaBootstrap, type AnyOrm } from '../../orm';
|
||||
import type { DatabaseEngine } from '../engine';
|
||||
import { redactUrl } from '../redact-url';
|
||||
@@ -69,19 +70,28 @@ export class PostgresEngine implements DatabaseEngine {
|
||||
|
||||
constructor(
|
||||
private readonly settings: PostgresSettings,
|
||||
private readonly bootstrapDeps: () => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
|
||||
private readonly bootstrapDeps: (url: string) => PostgresBootstrapDeps = defaultPostgresBootstrapDeps,
|
||||
) {}
|
||||
|
||||
open(): void {
|
||||
// Nothing of its own to open: the ORM connects the pool.
|
||||
private onReopened: (() => Promise<void>) | null = null;
|
||||
|
||||
/** Nothing of its own to open (the ORM connects the pool); keeps what a reopen runs. */
|
||||
open(onReopened: () => Promise<void>): void {
|
||||
this.onReopened = onReopened;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
// Nothing of its own to close: the pool goes with the ORM.
|
||||
}
|
||||
|
||||
reopen(): Promise<void> {
|
||||
return Promise.reject(new Error('Restoring a backup into a Postgres database is not available in this release.'));
|
||||
/**
|
||||
* After `pg_restore` replaced the database's contents: the ORM reconnects
|
||||
* and the restored database is migrated forward (`onReopened`, which
|
||||
* `DatabaseLifecycle` hands to `open`).
|
||||
*/
|
||||
async reopen(): Promise<void> {
|
||||
if (!this.onReopened) throw new Error('The Postgres engine was reopened before it was opened.');
|
||||
await this.onReopened();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -95,8 +105,10 @@ export class PostgresEngine implements DatabaseEngine {
|
||||
.execute('select pg_advisory_xact_lock(?)', [TREK_WRITE_LOCK], 'run', em.getTransactionContext());
|
||||
}
|
||||
|
||||
async schemaBootstrap(orm: AnyOrm): Promise<void> {
|
||||
await bootstrapPostgresSchema(orm, this.bootstrapDeps());
|
||||
/** `snapshot: false` (a restore, whose archive is the copy) migrates without a pg_dump first. */
|
||||
async schemaBootstrap(orm: AnyOrm, options: { snapshot: boolean }): Promise<void> {
|
||||
const deps = this.bootstrapDeps(this.settings.url);
|
||||
await bootstrapPostgresSchema(orm, options.snapshot ? deps : { ...deps, safetyNet: NO_SAFETY_NET });
|
||||
await finishSchemaBootstrap(orm);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,14 +1,23 @@
|
||||
import { readEnv, type AppEnv } from '../../../app-config';
|
||||
import { resolveDataPaths } from '../../../app-config/data-paths';
|
||||
import { knownMigrationNames } from '../../known-migrations';
|
||||
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
|
||||
import type { AnyOrm } from '../../orm';
|
||||
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
|
||||
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
|
||||
import { pgDump } from './pg-tools';
|
||||
import type { Connection } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const logger = new Logger('DB');
|
||||
|
||||
/** A Postgres copy is a pg_dump custom-format archive. */
|
||||
const DUMP_EXTENSION = '.pgdump';
|
||||
|
||||
/** What a Postgres bootstrap reads, passed in so a test can supply its own. */
|
||||
export interface PostgresBootstrapDeps {
|
||||
baseline: () => PostgresBaseline;
|
||||
@@ -67,7 +76,8 @@ export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstr
|
||||
await Promise.all(baseline.migrations.map((name) => storage.logMigration({ name }, trx)));
|
||||
});
|
||||
} else {
|
||||
refuseNewerDatabase(executed, deps.known);
|
||||
const thisRelease = migrationLabel([...deps.known].sort().at(-1) ?? '');
|
||||
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
|
||||
}
|
||||
|
||||
const pending = await migrator.getPending();
|
||||
@@ -78,32 +88,73 @@ export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstr
|
||||
await migrator.up();
|
||||
}
|
||||
|
||||
/** Dumps a killed boot left half-written: never a copy, only disk taken. */
|
||||
function removeOrphanedPartials(dir: string): void {
|
||||
for (const file of fs.readdirSync(dir)) {
|
||||
if (file.startsWith(PREFIX) && file.endsWith(`${DUMP_EXTENSION}.partial`)) {
|
||||
fs.rmSync(path.join(dir, file), { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres safety net before pending migrations. This release cannot copy
|
||||
* a Postgres database (the SQLite engine's `VACUUM INTO` has no Postgres
|
||||
* equivalent short of `pg_dump`), so it refuses unless the operator turned
|
||||
* the copy off: fail closed, rather than migrate with nothing to go back to.
|
||||
* Off under NODE_ENV=test unless a suite asks for it, as on SQLite.
|
||||
* The Postgres safety net before pending migrations, as the SQLite file's
|
||||
* (`db/pre-migrate-snapshot.ts`): a `pg_dump` of the database to
|
||||
* `<data>/pre-migrate-<schema>-<time>.pgdump`, the newest
|
||||
* `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` kept. A copy that cannot be written
|
||||
* (pg_dump missing, the data dir full) refuses the boot before anything is
|
||||
* migrated, unless `TREK_DB_PRE_MIGRATE_SNAPSHOT=false`. Off under
|
||||
* NODE_ENV=test unless a suite asks for it.
|
||||
*/
|
||||
export function postgresSafetyNet(env: () => AppEnv = readEnv): MigrationSafetyNet {
|
||||
export function postgresSafetyNet(
|
||||
url: string,
|
||||
env: () => AppEnv = readEnv,
|
||||
now: () => Date = () => new Date(),
|
||||
dataDir: () => string = () => resolveDataPaths(env()).dataDir,
|
||||
): MigrationSafetyNet {
|
||||
return {
|
||||
async beforeMigrate() {
|
||||
async beforeMigrate(from) {
|
||||
const { db, app } = env();
|
||||
if (db.preMigrateSnapshot === undefined && app.isTest) return;
|
||||
if (db.preMigrateSnapshot === false) {
|
||||
logger.warn('TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating the Postgres database without a copy');
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`${REFUSAL} migrations are pending and TREK cannot copy a Postgres database before migrating it in this ` +
|
||||
'release. Back it up with pg_dump, then start once with TREK_DB_PRE_MIGRATE_SNAPSHOT=false.',
|
||||
);
|
||||
const target = path.join(dataDir(), `${PREFIX}${from ?? 'unversioned'}-${stamp(now())}${DUMP_EXTENSION}`);
|
||||
try {
|
||||
fs.mkdirSync(dataDir(), { recursive: true });
|
||||
removeOrphanedPartials(dataDir());
|
||||
await pgDump(url, target, db.pgBinDir);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
|
||||
`(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Fix that, ` +
|
||||
'or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
logger.log(`Copy of the database before migrating: ${target}`);
|
||||
pruneSnapshots(dataDir(), db.preMigrateSnapshotKeep, DUMP_EXTENSION);
|
||||
},
|
||||
|
||||
async restoreHint(thisRelease) {
|
||||
const snapshots = listSnapshots(dataDir(), DUMP_EXTENSION);
|
||||
const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
|
||||
if (match) {
|
||||
return (
|
||||
`The copy taken right before the newer TREK migrated it is ${match}: stop TREK, empty the database ` +
|
||||
'(drop and recreate it), and restore it with pg_restore --no-owner --no-privileges --single-transaction.'
|
||||
);
|
||||
}
|
||||
if (snapshots.length > 0) {
|
||||
return `Copies taken before earlier migrations are in ${dataDir()} (${PREFIX}*${DUMP_EXTENSION}), none of them from this release.`;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
restoreHint: async () => null,
|
||||
};
|
||||
}
|
||||
|
||||
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
|
||||
export function defaultPostgresBootstrapDeps(): PostgresBootstrapDeps {
|
||||
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet() };
|
||||
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
|
||||
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
|
||||
}
|
||||
|
||||
@@ -25,11 +25,10 @@ import path from 'node:path';
|
||||
* migrated; `TREK_DB_PRE_MIGRATE_SNAPSHOT=false` is the way past that.
|
||||
*/
|
||||
|
||||
const PREFIX = 'pre-migrate-';
|
||||
const SNAPSHOT_FILE = /^pre-migrate-.+\.db$/;
|
||||
export const PREFIX = 'pre-migrate-';
|
||||
|
||||
/** `20261008T091500Z`: sortable, and free of the colons Windows refuses in a file name. */
|
||||
function stamp(now: Date): string {
|
||||
export function stamp(now: Date): string {
|
||||
return now
|
||||
.toISOString()
|
||||
.replace(/\.\d+Z$/, 'Z')
|
||||
@@ -51,11 +50,11 @@ async function holdsAnything(connection: Connection): Promise<boolean> {
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/** The snapshots in `dir`, newest first. */
|
||||
export function listSnapshots(dir: string): string[] {
|
||||
/** The snapshots in `dir`, newest first: the SQLite file's (`.db`) or a Postgres engine's (`.pgdump`). */
|
||||
export function listSnapshots(dir: string, extension = '.db'): string[] {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir).filter((entry) => SNAPSHOT_FILE.test(entry));
|
||||
entries = fs.readdirSync(dir).filter((entry) => entry.startsWith(PREFIX) && entry.endsWith(extension));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
@@ -66,8 +65,8 @@ export function listSnapshots(dir: string): string[] {
|
||||
}
|
||||
|
||||
/** Deletes all but the newest `keep`. A failure only warns: the new snapshot is already safe. */
|
||||
export function pruneSnapshots(dir: string, keep: number): void {
|
||||
for (const file of listSnapshots(dir).slice(keep)) {
|
||||
export function pruneSnapshots(dir: string, keep: number, extension = '.db'): void {
|
||||
for (const file of listSnapshots(dir, extension).slice(keep)) {
|
||||
try {
|
||||
fs.rmSync(file, { force: true });
|
||||
} catch (err) {
|
||||
|
||||
@@ -150,6 +150,7 @@ async function restoreBeforeTheDatabaseOpens() {
|
||||
dbFile: resolveDbPath(),
|
||||
dataDir: paths.dataDir,
|
||||
reclaimResetFile: paths.reclaimResetFile,
|
||||
postgres: env.db.url ? { url: env.db.url, binDir: env.db.pgBinDir } : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -474,7 +474,7 @@ export async function restoreFromZip({ storage, database }: BackupDeps, zipPath:
|
||||
const refused = await extractBackupArchive(zipPath, extractDir);
|
||||
if (refused) return { success: false, ...refused };
|
||||
|
||||
const unreadable = database.verify(extractDir);
|
||||
const unreadable = await database.verify(extractDir);
|
||||
if (unreadable) {
|
||||
fs.rmSync(extractDir, { recursive: true, force: true });
|
||||
return { success: false, ...unreadable };
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { pgRestore, pgTableNames } from '../../db/engine/postgres/pg-tools';
|
||||
import { stageExtractedPluginTrees } from '../plugins/plugin-backup';
|
||||
import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive';
|
||||
import { checkPostgresDump, PG_ARCHIVE_ENTRY } from './postgres-database-backup';
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
@@ -47,6 +49,11 @@ export interface BootRestorePlan {
|
||||
dataDir: string;
|
||||
/** `data/.trek-reclaim-reset` (data-paths.ts), left before the database is put in place. */
|
||||
reclaimResetFile: string;
|
||||
/**
|
||||
* Set when TREK runs on Postgres (TREK_DB_URL): the archive's dump is loaded
|
||||
* into the database, which must hold no table yet, instead of `dbFile`.
|
||||
*/
|
||||
postgres?: { url: string; binDir: string | undefined };
|
||||
}
|
||||
|
||||
export type BootRestoreOutcome =
|
||||
@@ -56,11 +63,38 @@ export type BootRestoreOutcome =
|
||||
|
||||
export class BootRestoreError extends Error {}
|
||||
|
||||
/** A database is there already: on a file, the file; on Postgres, any table in its schema. */
|
||||
async function existingDatabase(plan: BootRestorePlan): Promise<string | null> {
|
||||
if (!plan.postgres) return fs.existsSync(plan.dbFile) ? plan.dbFile : null;
|
||||
const tables = await pgTableNames(plan.postgres.url, plan.postgres.binDir);
|
||||
return tables.length > 0 ? `the Postgres database (${tables.length} table(s))` : null;
|
||||
}
|
||||
|
||||
/** Why the unpacked archive's database cannot be restored here, or null. */
|
||||
async function checkArchiveDatabase(plan: BootRestorePlan, staging: string): Promise<string | null> {
|
||||
if (plan.postgres) return checkPostgresDump(staging, plan.postgres.binDir);
|
||||
return checkBackupDatabase(staging)?.error ?? null;
|
||||
}
|
||||
|
||||
/** Puts the archive's database in place: renamed in beside the file, or loaded into Postgres in one transaction. */
|
||||
async function putDatabaseInPlace(plan: BootRestorePlan, staging: string): Promise<void> {
|
||||
if (plan.postgres) {
|
||||
await pgRestore(plan.postgres.url, path.join(staging, PG_ARCHIVE_ENTRY), plan.postgres.binDir);
|
||||
return;
|
||||
}
|
||||
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
|
||||
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
|
||||
const tmp = `${plan.dbFile}.restore-tmp`;
|
||||
fs.copyFileSync(path.join(staging, 'travel.db'), tmp);
|
||||
fs.renameSync(tmp, plan.dbFile);
|
||||
}
|
||||
|
||||
export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRestoreOutcome> {
|
||||
if (!plan.archive) return { restored: false, reason: 'unset' };
|
||||
if (fs.existsSync(plan.dbFile)) {
|
||||
const existing = await existingDatabase(plan);
|
||||
if (existing) {
|
||||
console.warn(
|
||||
`[restore] RESTORE_FROM_BACKUP is set, but ${plan.dbFile} already exists. The backup is only restored on a first start; remove the variable to silence this.`,
|
||||
`[restore] RESTORE_FROM_BACKUP is set, but ${existing} already exists. The backup is only restored on a first start; remove the variable to silence this.`,
|
||||
);
|
||||
return { restored: false, reason: 'database-exists' };
|
||||
}
|
||||
@@ -73,20 +107,16 @@ export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRes
|
||||
const refused = await extractBackupArchive(plan.archive, staging);
|
||||
if (refused) throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${refused.error}`);
|
||||
|
||||
const unreadable = checkBackupDatabase(staging);
|
||||
const unreadable = await checkArchiveDatabase(plan, staging);
|
||||
if (unreadable) {
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable.error}`);
|
||||
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable}`);
|
||||
}
|
||||
|
||||
// Storage reclaim drops what the restored database knew about its storage on
|
||||
// the start that follows, even when the backup is this installation's own.
|
||||
markReclaimResetPending(plan.reclaimResetFile);
|
||||
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
|
||||
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
|
||||
const tmp = `${plan.dbFile}.restore-tmp`;
|
||||
fs.copyFileSync(path.join(staging, 'travel.db'), tmp);
|
||||
fs.renameSync(tmp, plan.dbFile);
|
||||
await putDatabaseInPlace(plan, staging);
|
||||
|
||||
// The key the archive's secrets were encrypted with. An ENCRYPTION_KEY in the
|
||||
// environment still wins over the file, exactly as on any other start.
|
||||
|
||||
@@ -1,26 +1,90 @@
|
||||
import { readEnv } from '../../app-config';
|
||||
import { resolveEngine } from '../../db/engine';
|
||||
import { pgDump, pgDumpMigrations, pgRestore } from '../../db/engine/postgres/pg-tools';
|
||||
import { unknownMigrations } from '../../db/known-migrations';
|
||||
import { DataPathsService } from '../app-config/data-paths.service';
|
||||
import { logInfo, logWarn } from '../audit/audit-log.logger';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import type { DatabaseBackupStrategy } from '../database/database-backup.interface';
|
||||
import {
|
||||
DatabaseConnectionLostError,
|
||||
type DatabaseBackupRefusal,
|
||||
type DatabaseBackupStrategy,
|
||||
} from '../database/database-backup.interface';
|
||||
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/** Why a backup or a restore of a Postgres database is refused in this release. */
|
||||
export const POSTGRES_BACKUP_UNAVAILABLE =
|
||||
'Backups of a Postgres database are not available in this release: back it up with pg_dump.';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err));
|
||||
|
||||
/** The name a Postgres database takes inside a backup archive: a pg_dump custom-format file. */
|
||||
export const PG_ARCHIVE_ENTRY = 'travel.pgdump';
|
||||
|
||||
/**
|
||||
* The backup port on a Postgres database, before TREK can dump and load one.
|
||||
* Fails closed: a backup reports that it wrote nothing (rather than writing an
|
||||
* archive with the uploads and no database), a restore is refused before
|
||||
* anything is replaced, and the demo reset never gets a baseline to swap in.
|
||||
* Why the database in an unpacked archive cannot be restored into a Postgres
|
||||
* database, or null when it can: it must be a dump, carry TREK's migration
|
||||
* history, and none this build does not know. Both restore paths ask it, the
|
||||
* admin panel's and RESTORE_FROM_BACKUP's.
|
||||
*/
|
||||
export async function checkPostgresDump(extractDir: string, binDir: string | undefined): Promise<string | null> {
|
||||
const dump = path.join(extractDir, PG_ARCHIVE_ENTRY);
|
||||
if (!fs.existsSync(dump)) {
|
||||
return fs.existsSync(path.join(extractDir, 'travel.db'))
|
||||
? 'This backup holds a SQLite database, which cannot be restored into a Postgres database.'
|
||||
: `Invalid backup: ${PG_ARCHIVE_ENTRY} not found`;
|
||||
}
|
||||
let executed: string[];
|
||||
try {
|
||||
executed = await pgDumpMigrations(dump, binDir);
|
||||
} catch (err) {
|
||||
logWarn(`Restore: the archive's database could not be read (${describeError(err)})`);
|
||||
return 'Uploaded file is not a valid Postgres dump';
|
||||
}
|
||||
if (executed.length === 0) {
|
||||
return 'The uploaded dump holds no TREK migration history. This does not appear to be a TREK backup.';
|
||||
}
|
||||
if (unknownMigrations(executed).length > 0) {
|
||||
return 'This backup was made by a newer TREK version. Update TREK before restoring it.';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The backup port on a Postgres database: `pg_dump` and `pg_restore`
|
||||
* (`db/engine/postgres/pg-tools.ts`), run by their absolute paths with the
|
||||
* password in their environment.
|
||||
*
|
||||
* A snapshot is a custom-format dump, consistent on its own (pg_dump reads one
|
||||
* snapshot of the database). A restore checks the dump first (it must be one,
|
||||
* carry TREK's migration history, and none this build does not know), keeps a
|
||||
* dump of the database it replaces, then clears the schema and loads the dump
|
||||
* in one transaction (`pgRestore`), so a restore that fails leaves the database
|
||||
* as it was and one that succeeds holds nothing a later migration added, and
|
||||
* reopens through `DatabaseLifecycle`, which migrates the restored database
|
||||
* forward. A backup of a SQLite install cannot be restored here.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PostgresDatabaseBackup implements DatabaseBackupStrategy {
|
||||
readonly archiveEntry = 'database.dump';
|
||||
readonly archiveEntry = PG_ARCHIVE_ENTRY;
|
||||
|
||||
constructor(
|
||||
private readonly lifecycle: DatabaseLifecycle,
|
||||
private readonly paths: DataPathsService,
|
||||
) {}
|
||||
|
||||
/** The URL and the tools' directory, read when used: on SQLite this provider is built but never called. */
|
||||
private get settings(): { url: string; binDir: string | undefined } {
|
||||
const { url, pgBinDir } = readEnv().db;
|
||||
if (!url) throw new Error('The Postgres backup was used without TREK_DB_URL.');
|
||||
return { url, binDir: pgBinDir };
|
||||
}
|
||||
|
||||
location(): string {
|
||||
return resolveEngine().describe();
|
||||
}
|
||||
|
||||
/** A Postgres database is always there to copy; a missing pg_dump fails the snapshot, loudly. */
|
||||
canSnapshot(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -29,20 +93,53 @@ export class PostgresDatabaseBackup implements DatabaseBackupStrategy {
|
||||
// Postgres keeps nothing beside its store that a copy would need flushed.
|
||||
}
|
||||
|
||||
snapshot(): Promise<void> {
|
||||
return Promise.reject(new Error(POSTGRES_BACKUP_UNAVAILABLE));
|
||||
/** pg_dump writes beside `target` and renames into place: a failed dump leaves nothing there. */
|
||||
async snapshot(target: string): Promise<void> {
|
||||
const { url, binDir } = this.settings;
|
||||
await pgDump(url, target, binDir);
|
||||
}
|
||||
|
||||
/** Refuses before anything is replaced; the restore removes what it unpacked and answers 400. */
|
||||
verify(): never {
|
||||
throw new DomainError(400, POSTGRES_BACKUP_UNAVAILABLE);
|
||||
}
|
||||
|
||||
async keepCopyBeforeRestore(): Promise<string | null> {
|
||||
/** Refuses (a DomainError, 400) an archive this database cannot take, before anything is replaced. */
|
||||
async verify(extractDir: string): Promise<DatabaseBackupRefusal | null> {
|
||||
const refusal = await checkPostgresDump(extractDir, this.settings.binDir);
|
||||
if (refusal) throw new DomainError(400, refusal);
|
||||
return null;
|
||||
}
|
||||
|
||||
replace(): Promise<{ reopenError: unknown }> {
|
||||
return Promise.reject(new Error(POSTGRES_BACKUP_UNAVAILABLE));
|
||||
/** A dump of the database a restore is about to replace, in the data directory. */
|
||||
async keepCopyBeforeRestore(): Promise<string | null> {
|
||||
const target = path.join(this.paths.dataDir, `pre-restore-${Date.now()}.pgdump`);
|
||||
try {
|
||||
await this.snapshot(target);
|
||||
logInfo(`Restore: the replaced database was kept as ${target}`);
|
||||
return target;
|
||||
} catch (err) {
|
||||
logWarn(`Restore: could not keep a copy of the current database (${describeError(err)})`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async replace(source: string): Promise<{ reopenError: unknown }> {
|
||||
let swapError: unknown = null;
|
||||
try {
|
||||
const { url, binDir } = this.settings;
|
||||
await pgRestore(url, source, binDir);
|
||||
} catch (err) {
|
||||
swapError = err ?? new Error('pg_restore failed');
|
||||
}
|
||||
|
||||
// As on SQLite: the reopen always runs, so the ORM never keeps a stale pool,
|
||||
// and both errors go up together when the restore failed as well.
|
||||
let reopenError: unknown = null;
|
||||
try {
|
||||
await this.lifecycle.reopen();
|
||||
} catch (err) {
|
||||
reopenError = err;
|
||||
}
|
||||
if (swapError) {
|
||||
if (reopenError) throw new DatabaseConnectionLostError(swapError, reopenError);
|
||||
throw swapError;
|
||||
}
|
||||
return { reopenError };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ export class SqliteDatabaseBackup implements DatabaseBackupStrategy {
|
||||
}
|
||||
}
|
||||
|
||||
verify(extractDir: string): DatabaseBackupRefusal | null {
|
||||
async verify(extractDir: string): Promise<DatabaseBackupRefusal | null> {
|
||||
return checkBackupDatabase(extractDir);
|
||||
}
|
||||
|
||||
|
||||
@@ -42,10 +42,11 @@ export interface DatabaseBackupStrategy {
|
||||
snapshot(target: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Checks an unpacked archive's database before anything is replaced. Returns
|
||||
* null when it can be restored, or the refusal in the restore route's words.
|
||||
* Checks an unpacked archive's database before anything is replaced. Resolves
|
||||
* with null when it can be restored, or the refusal in the restore route's
|
||||
* words (an engine may throw a DomainError with it instead).
|
||||
*/
|
||||
verify(extractDir: string): DatabaseBackupRefusal | null;
|
||||
verify(extractDir: string): Promise<DatabaseBackupRefusal | null>;
|
||||
|
||||
/**
|
||||
* Keeps a copy of the database a restore is about to replace. Best effort: a
|
||||
|
||||
@@ -108,11 +108,14 @@ describe('buildOrmConfig', () => {
|
||||
expect(config.entities).toBe(buildOrmConfig(resolveEngine()).entities);
|
||||
});
|
||||
|
||||
it('ENGINE-032: a Postgres engine opens and closes nothing of its own, and refuses a reopen (no restore yet)', async () => {
|
||||
it('ENGINE-032: a Postgres engine opens and closes nothing of its own; a reopen (after pg_restore) rebinds and migrates', async () => {
|
||||
const engine = resolveEngine(deriveDb({ TREK_DB_URL: PG_URL }));
|
||||
expect(() => engine.open(() => undefined)).not.toThrow();
|
||||
await expect(engine.reopen()).rejects.toThrow(/reopened before it was opened/);
|
||||
const onReopened = vi.fn(async () => undefined);
|
||||
expect(() => engine.open(onReopened)).not.toThrow();
|
||||
expect(() => engine.close()).not.toThrow();
|
||||
await expect(engine.reopen()).rejects.toThrow(/not available in this release/);
|
||||
await engine.reopen();
|
||||
expect(onReopened).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('ENGINE-033: Postgres serializes write transactions with one advisory lock taken in the transaction, SQLite needs none', async () => {
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* The Postgres client tools (src/db/engine/postgres/pg-tools.ts), run against
|
||||
* stand-in pg_dump, pg_restore and psql scripts in a temp directory: the exact
|
||||
* arguments and environment each call hands the binary, the password kept out
|
||||
* of the arguments, the files a dump and a restore leave behind, and the
|
||||
* refusal when a tool is missing. The real tools against a real database are
|
||||
* exercised by the Postgres lane.
|
||||
*/
|
||||
import {
|
||||
hasRestrictFix,
|
||||
pgConnectionEnv,
|
||||
pgDump,
|
||||
pgDumpMigrations,
|
||||
pgRestore,
|
||||
PgToolMissingError,
|
||||
resolvePgTool,
|
||||
} from '../../../../../src/db/engine/postgres/pg-tools';
|
||||
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const URL_WITH_SECRET = 'postgresql://trek:s3cr%40t@db.example:6543/trek_db?sslmode=require';
|
||||
|
||||
let bin: string;
|
||||
let out: string;
|
||||
|
||||
/** pg_restore writing its --file (the third argument) as 17.6+ does: opened with a \\restrict line. */
|
||||
const WRITES_RESTRICTED_SCRIPT = "printf '%s\\n' '\\restrict k1' 'SELECT 1;' > \"${3#--file=}\"";
|
||||
|
||||
interface FakeOptions {
|
||||
stdout?: string;
|
||||
stderr?: string;
|
||||
exitCode?: number;
|
||||
/** Extra shell the tool runs before it exits. */
|
||||
body?: string;
|
||||
/** What `--version` answers. */
|
||||
version?: string;
|
||||
}
|
||||
|
||||
/** A stand-in tool that records its arguments and libpq environment, then prints `stdout`. */
|
||||
function fakeTool(
|
||||
name: string,
|
||||
{ stdout = '', stderr = '', exitCode = 0, body = '', version = '17.11' }: FakeOptions = {},
|
||||
): string {
|
||||
const log = path.join(bin, `${name}.log`);
|
||||
writeFileSync(
|
||||
path.join(bin, name),
|
||||
[
|
||||
'#!/bin/sh',
|
||||
`[ "$1" = --version ] && { echo '${name} (PostgreSQL) ${version} (Debian)'; exit 0; }`,
|
||||
`printf '%s\\n' "$@" > '${log}'`,
|
||||
`env | grep '^PG' | sort >> '${log}'`,
|
||||
body,
|
||||
`printf '%s' '${stdout}'`,
|
||||
`printf '%s' '${stderr}' >&2`,
|
||||
`exit ${exitCode}`,
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
chmodSync(path.join(bin, name), 0o755);
|
||||
return log;
|
||||
}
|
||||
|
||||
const argsOf = (log: string): string[] =>
|
||||
readFileSync(log, 'utf8')
|
||||
.split('\n')
|
||||
.filter((line) => !line.startsWith('PG'));
|
||||
|
||||
beforeEach(() => {
|
||||
bin = mkdtempSync(path.join(tmpdir(), 'trek-pg-bin-'));
|
||||
out = mkdtempSync(path.join(tmpdir(), 'trek-pg-out-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(bin, { recursive: true, force: true });
|
||||
rmSync(out, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('pg-tools', () => {
|
||||
it('PGTOOLS-001: finds a tool by absolute path in the configured directory, and nowhere else', () => {
|
||||
fakeTool('pg_dump');
|
||||
writeFileSync(path.join(bin, 'psql'), 'not executable');
|
||||
expect(resolvePgTool('pg_dump', bin)).toBe(path.join(bin, 'pg_dump'));
|
||||
expect(resolvePgTool('pg_restore', bin)).toBeNull();
|
||||
expect(resolvePgTool('psql', bin)).toBeNull();
|
||||
});
|
||||
|
||||
it('PGTOOLS-002: the connection goes in the libpq environment, the password decoded and never an argument', () => {
|
||||
expect(pgConnectionEnv(URL_WITH_SECRET)).toEqual({
|
||||
PGHOST: 'db.example',
|
||||
PGPORT: '6543',
|
||||
PGDATABASE: 'trek_db',
|
||||
PGUSER: 'trek',
|
||||
PGPASSWORD: 's3cr@t',
|
||||
PGCONNECT_TIMEOUT: '10',
|
||||
PGSSLMODE: 'verify-full',
|
||||
PGSSLROOTCERT: 'system',
|
||||
});
|
||||
expect(pgConnectionEnv('postgres://u@[::1]/d')).toEqual({
|
||||
PGHOST: '::1',
|
||||
PGPORT: '5432',
|
||||
PGDATABASE: 'd',
|
||||
PGUSER: 'u',
|
||||
PGCONNECT_TIMEOUT: '10',
|
||||
});
|
||||
});
|
||||
|
||||
it('PGTOOLS-003: SSL is verified as the app pool verifies it, against the URL root certificate when it names one', () => {
|
||||
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable')).toMatchObject({ PGSSLMODE: 'disable' });
|
||||
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable').PGSSLROOTCERT).toBeUndefined();
|
||||
expect(pgConnectionEnv('postgres://u@h/d?sslmode=verify-ca&sslrootcert=%2Fca%2Froot.crt')).toMatchObject({
|
||||
PGSSLMODE: 'verify-full',
|
||||
PGSSLROOTCERT: '/ca/root.crt',
|
||||
});
|
||||
});
|
||||
|
||||
it('PGTOOLS-004: pg_dump writes a private custom-format dump beside the target and renames it into place', async () => {
|
||||
const log = fakeTool('pg_dump', { body: 'printf dump >> "${4#--file=}"' });
|
||||
const target = path.join(out, 'x.pgdump');
|
||||
await pgDump(URL_WITH_SECRET, target, bin);
|
||||
expect(argsOf(log).slice(0, 4)).toEqual([
|
||||
'--format=custom',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
`--file=${target}.partial`,
|
||||
]);
|
||||
expect(readFileSync(log, 'utf8')).toContain('PGPASSWORD=s3cr@t');
|
||||
expect(argsOf(log).join(' ')).not.toContain('s3cr');
|
||||
expect(readFileSync(target, 'utf8')).toBe('dump');
|
||||
expect(statSync(target).mode & 0o777).toBe(0o600);
|
||||
expect(readdirSync(out)).toEqual(['x.pgdump']);
|
||||
});
|
||||
|
||||
it('PGTOOLS-005: a restore clears the schema and loads the script in one psql transaction, then removes its work files', async () => {
|
||||
const source = path.join(out, 'in.pgdump');
|
||||
const restoreLog = fakeTool('pg_restore', { body: WRITES_RESTRICTED_SCRIPT });
|
||||
const psqlLog = fakeTool('psql', { body: `cp "\${6#--file=}" '${path.join(bin, 'clear.sql')}'` });
|
||||
await pgRestore(URL_WITH_SECRET, source, bin);
|
||||
|
||||
const restoreArgs = argsOf(restoreLog);
|
||||
expect(restoreArgs.slice(0, 2)).toEqual(['--no-owner', '--no-privileges']);
|
||||
expect(restoreArgs[2]).toMatch(/^--file=.*\.pg-restore-[^/]+\/restore\.sql$/);
|
||||
expect(restoreArgs.slice(3, 5)).toEqual(['--', source]);
|
||||
expect(readFileSync(restoreLog, 'utf8')).not.toContain('PGPASSWORD');
|
||||
|
||||
const psqlArgs = argsOf(psqlLog);
|
||||
expect(psqlArgs.slice(0, 5)).toEqual([
|
||||
'--no-psqlrc',
|
||||
'--quiet',
|
||||
'--single-transaction',
|
||||
'--set=ON_ERROR_STOP=1',
|
||||
'--output=/dev/null',
|
||||
]);
|
||||
expect(psqlArgs[5]).toMatch(/clear-schema\.sql$/);
|
||||
expect(psqlArgs[6]).toBe(restoreArgs[2]);
|
||||
expect(readFileSync(psqlLog, 'utf8')).toContain('PGOPTIONS=-c lock_timeout=60s -c client_min_messages=warning');
|
||||
expect(readFileSync(path.join(bin, 'clear.sql'), 'utf8')).toMatch(/DROP %s IF EXISTS %s CASCADE/);
|
||||
expect(readdirSync(out)).toEqual([]);
|
||||
});
|
||||
|
||||
it("PGTOOLS-006: reads the dump's recorded migrations from its COPY block, and none from a dump without them", async () => {
|
||||
const copy = [
|
||||
'SET statement_timeout = 0;',
|
||||
'COPY public.mikro_orm_migrations (id, name, executed_at) FROM stdin;',
|
||||
'1\tMigration20200101000000_baseline_schema\t2026-01-01 00:00:00+00',
|
||||
'2\tMigration20200101000100_second\t2026-01-01 00:00:00+00',
|
||||
'\\.',
|
||||
'',
|
||||
].join('\n');
|
||||
const log = fakeTool('pg_restore', { stdout: copy });
|
||||
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([
|
||||
'Migration20200101000000_baseline_schema',
|
||||
'Migration20200101000100_second',
|
||||
]);
|
||||
expect(argsOf(log).slice(0, 5)).toEqual([
|
||||
'--data-only',
|
||||
'--table=mikro_orm_migrations',
|
||||
'--file=-',
|
||||
'--',
|
||||
'/data/in.pgdump',
|
||||
]);
|
||||
fakeTool('pg_restore', { stdout: 'SET statement_timeout = 0;\n' });
|
||||
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it('PGTOOLS-007: a failing tool reports its own message without the lines that quote SQL or rows, and leaves no file', async () => {
|
||||
fakeTool('pg_dump', {
|
||||
exitCode: 1,
|
||||
stderr: 'pg_dump: error: query failed\nCommand was: SELECT secret_row\nDETAIL: Key (email)=(a@b.c)\n',
|
||||
});
|
||||
const target = path.join(out, 'x.pgdump');
|
||||
await expect(pgDump(URL_WITH_SECRET, target, bin)).rejects.toThrow(
|
||||
/^pg_dump failed: pg_dump: error: query failed$/,
|
||||
);
|
||||
expect(readdirSync(out)).toEqual([]);
|
||||
|
||||
fakeTool('pg_dump', { exitCode: 1, stderr: 'x'.repeat(5000) });
|
||||
await expect(pgDump(URL_WITH_SECRET, target, bin)).rejects.toThrow(/^pg_dump failed: x{2000}…$/);
|
||||
});
|
||||
|
||||
it('PGTOOLS-008: a missing tool says how to install it, and a restore that cannot run leaves no work files', async () => {
|
||||
const source = path.join(out, 'in.pgdump');
|
||||
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(PgToolMissingError);
|
||||
await expect(pgDump(URL_WITH_SECRET, path.join(out, 'x.pgdump'), bin)).rejects.toThrow(/TREK_DB_PG_BIN_DIR/);
|
||||
fakeTool('pg_restore', { body: WRITES_RESTRICTED_SCRIPT });
|
||||
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(/psql was not found/);
|
||||
expect(existsSync(out) && readdirSync(out)).toEqual([]);
|
||||
});
|
||||
|
||||
it('PGTOOLS-009: client tools without the \\restrict fix (CVE-2025-8714) never turn a dump into a script', async () => {
|
||||
fakeTool('pg_restore', { version: '16.9', body: `touch '${path.join(bin, 'ran')}'` });
|
||||
fakeTool('psql', { body: `touch '${path.join(bin, 'ran')}'` });
|
||||
await expect(pgRestore(URL_WITH_SECRET, path.join(out, 'in.pgdump'), bin)).rejects.toThrow(
|
||||
/pg_restore is too old to restore safely \(pg_restore \(PostgreSQL\) 16\.9 \(Debian\)\): it must be at least 17\.6/,
|
||||
);
|
||||
fakeTool('pg_restore', { body: `touch '${path.join(bin, 'ran')}'` });
|
||||
fakeTool('psql', { version: '17.5', body: `touch '${path.join(bin, 'ran')}'` });
|
||||
await expect(pgRestore(URL_WITH_SECRET, path.join(out, 'in.pgdump'), bin)).rejects.toThrow(/psql is too old/);
|
||||
expect(existsSync(path.join(bin, 'ran'))).toBe(false);
|
||||
expect(readdirSync(out)).toEqual([]);
|
||||
});
|
||||
|
||||
it('PGTOOLS-010: knows which client versions carry the \\restrict fix', () => {
|
||||
const fixed = ['14.19', '15.14', '16.10', '17.6', '17.11', '18.0', '19.1'];
|
||||
const unfixed = ['13.30', '14.18', '15.13', '16.9', '17.5'];
|
||||
for (const v of fixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(true);
|
||||
for (const v of unfixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(false);
|
||||
expect(hasRestrictFix('not a version')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,10 @@ import type { MigrationSafetyNet } from '../../../../../src/db/legacy-baseline';
|
||||
import type { AnyOrm } from '../../../../../src/db/orm';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { chmodSync, mkdtempSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const M1 = 'Migration20200101000000_baseline_schema';
|
||||
const M2 = 'Migration20200101000100_second';
|
||||
@@ -175,27 +178,89 @@ describe('bootstrapPostgresSchema', () => {
|
||||
});
|
||||
|
||||
describe('postgresSafetyNet', () => {
|
||||
const envWith = (preMigrateSnapshot: boolean | undefined, isTest: boolean) => () =>
|
||||
({ db: { preMigrateSnapshot }, app: { isTest } }) as unknown as AppEnv;
|
||||
const URL = 'postgresql://trek:pw@db/trek';
|
||||
const NOW = new Date('2026-11-02T13:05:09Z');
|
||||
let dir: string;
|
||||
let bin: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(path.join(tmpdir(), 'trek-pg-net-'));
|
||||
bin = mkdtempSync(path.join(tmpdir(), 'trek-pg-net-bin-'));
|
||||
});
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
rmSync(bin, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** A stand-in pg_dump that writes its --file, or fails like one that cannot connect. */
|
||||
const fakePgDump = (ok: boolean) => {
|
||||
const body = ok
|
||||
? 'for a in "$@"; do case "$a" in --file=*) printf dump > "${a#--file=}";; esac; done'
|
||||
: 'echo "could not connect" >&2; exit 1';
|
||||
writeFileSync(path.join(bin, 'pg_dump'), `#!/bin/sh\n${body}\n`);
|
||||
chmodSync(path.join(bin, 'pg_dump'), 0o755);
|
||||
};
|
||||
const envWith =
|
||||
(preMigrateSnapshot: boolean | undefined, isTest: boolean, keep = 3) =>
|
||||
() =>
|
||||
({
|
||||
db: { preMigrateSnapshot, preMigrateSnapshotKeep: keep, pgBinDir: bin },
|
||||
app: { isTest },
|
||||
}) as unknown as AppEnv;
|
||||
const net = (env: () => AppEnv) =>
|
||||
postgresSafetyNet(
|
||||
URL,
|
||||
env,
|
||||
() => NOW,
|
||||
() => dir,
|
||||
);
|
||||
|
||||
it.each([undefined, true])(
|
||||
'PGBOOT-010: refuses by default (setting %s), naming pg_dump and the switch, since this release cannot copy a Postgres database',
|
||||
'PGBOOT-010: dumps the database before migrating it (setting %s), keeping the newest copies and no half-written one',
|
||||
async (setting) => {
|
||||
await expect(postgresSafetyNet(envWith(setting, false)).beforeMigrate('1')).rejects.toThrow(
|
||||
/cannot copy a Postgres database.*pg_dump.*TREK_DB_PRE_MIGRATE_SNAPSHOT=false/,
|
||||
);
|
||||
vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
|
||||
fakePgDump(true);
|
||||
for (const old of ['20200101000001', '20200101000002']) {
|
||||
writeFileSync(path.join(dir, `pre-migrate-${old}-20250101T000000Z.pgdump`), 'old');
|
||||
utimesSync(path.join(dir, `pre-migrate-${old}-20250101T000000Z.pgdump`), 1, 1);
|
||||
}
|
||||
writeFileSync(path.join(dir, 'pre-migrate-20200101000003-20250101T000000Z.pgdump.partial'), 'killed');
|
||||
await net(envWith(setting, false, 2)).beforeMigrate('20200101000100');
|
||||
const kept = readdirSync(dir).sort();
|
||||
expect(kept).toHaveLength(2);
|
||||
expect(kept).toContain('pre-migrate-20200101000100-20261102T130509Z.pgdump');
|
||||
expect(kept.some((file) => file.endsWith('.partial'))).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it('PGBOOT-011: migrates without a copy when the operator turned it off, and says so', async () => {
|
||||
const warn = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
await expect(postgresSafetyNet(envWith(false, false)).beforeMigrate('1')).resolves.toBeUndefined();
|
||||
await expect(net(envWith(false, false)).beforeMigrate('1')).resolves.toBeUndefined();
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('without a copy'));
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
});
|
||||
|
||||
it('PGBOOT-012: stays out of the way under test unless a suite asks for it, and has no restore hint', async () => {
|
||||
const net = postgresSafetyNet(envWith(undefined, true));
|
||||
await expect(net.beforeMigrate('1')).resolves.toBeUndefined();
|
||||
await expect(net.restoreHint('1')).resolves.toBeNull();
|
||||
it('PGBOOT-012: stays out of the way under test unless a suite asks for it', async () => {
|
||||
await expect(net(envWith(undefined, true)).beforeMigrate('1')).resolves.toBeUndefined();
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
});
|
||||
|
||||
it('PGBOOT-013: a copy that cannot be written refuses the boot, names the switch, and leaves no partial file', async () => {
|
||||
fakePgDump(false);
|
||||
await expect(net(envWith(true, false)).beforeMigrate('1')).rejects.toThrow(
|
||||
/Refusing to boot: could not write a copy.*pg_dump failed: could not connect.*TREK_DB_PRE_MIGRATE_SNAPSHOT=false/s,
|
||||
);
|
||||
expect(readdirSync(dir)).toEqual([]);
|
||||
});
|
||||
|
||||
it('PGBOOT-014: the newer-database refusal points at the copy taken from this release, or at the others', async () => {
|
||||
const hint = net(envWith(true, false));
|
||||
await expect(hint.restoreHint('20200101000100')).resolves.toBeNull();
|
||||
writeFileSync(path.join(dir, 'pre-migrate-20200101000050-20250101T000000Z.pgdump'), 'old');
|
||||
await expect(hint.restoreHint('20200101000100')).resolves.toMatch(/none of them from this release/);
|
||||
writeFileSync(path.join(dir, 'pre-migrate-20200101000100-20250102T000000Z.pgdump'), 'mine');
|
||||
await expect(hint.restoreHint('20200101000100')).resolves.toMatch(
|
||||
/pre-migrate-20200101000100-20250102T000000Z\.pgdump: stop TREK, empty the database .*pg_restore --no-owner/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
* calls this file makes. The plugin staging is the one seam stubbed, because
|
||||
* it moves trees under the plugin roots of the machine running the tests.
|
||||
*/
|
||||
import { knownMigrationNames } from '../../../src/db/known-migrations';
|
||||
import { BootRestoreError, restoreOnFirstBoot } from '../../../src/nest/backup/boot-restore';
|
||||
|
||||
import archiver from 'archiver';
|
||||
@@ -125,6 +126,77 @@ describe('restoreOnFirstBoot', () => {
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
});
|
||||
|
||||
describe('on Postgres', () => {
|
||||
/**
|
||||
* Stand-in client tools: psql lists the tables written to `tables` and
|
||||
* marks `loaded` when it runs a restore; pg_restore lists a known migration
|
||||
* as the dump's history, or writes a restricted script. Their arguments are
|
||||
* pinned in pg-tools.test.ts.
|
||||
*/
|
||||
const postgresPlan = (archivePath: string) => {
|
||||
const bin = path.join(root, 'bin');
|
||||
fs.mkdirSync(bin, { recursive: true });
|
||||
const write = (name: string, body: string) => {
|
||||
const version = `[ "$1" = --version ] && { echo '${name} (PostgreSQL) 17.11'; exit 0; }`;
|
||||
fs.writeFileSync(path.join(bin, name), `#!/bin/sh\n${version}\n${body}\n`);
|
||||
fs.chmodSync(path.join(bin, name), 0o755);
|
||||
};
|
||||
write(
|
||||
'psql',
|
||||
`case "$*" in *--command=*) [ -f '${root}/tables' ] && cat '${root}/tables'; exit 0;; *) touch '${root}/loaded';; esac`,
|
||||
);
|
||||
const migration = [...knownMigrationNames()][0];
|
||||
write(
|
||||
'pg_restore',
|
||||
[
|
||||
'case "$*" in',
|
||||
` *--data-only*) printf 'COPY public.mikro_orm_migrations (id, name) FROM stdin;\\n1\\t${migration}\\n\\\\.\\n';;`,
|
||||
` *) for a in "$@"; do case "$a" in --file=*) printf '%s\\n' '\\restrict k1' > "\${a#--file=}";; esac; done;;`,
|
||||
'esac',
|
||||
].join('\n'),
|
||||
);
|
||||
return {
|
||||
archive: archivePath,
|
||||
dbFile,
|
||||
dataDir,
|
||||
reclaimResetFile,
|
||||
postgres: { url: 'postgres://trek:pw@db/trek', binDir: bin },
|
||||
};
|
||||
};
|
||||
const loaded = () => fs.existsSync(path.join(root, 'loaded'));
|
||||
|
||||
it('BOOT-RESTORE-004b: a database holding tables is left alone, so a restart never rolls back', async () => {
|
||||
const plan = postgresPlan(await archive({ 'travel.pgdump': 'dump' }));
|
||||
fs.writeFileSync(path.join(root, 'tables'), 'users\ntrips\n');
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
expect(await restoreOnFirstBoot(plan)).toEqual({ restored: false, reason: 'database-exists' });
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('the Postgres database (2 table(s))'));
|
||||
expect(loaded()).toBe(false);
|
||||
});
|
||||
|
||||
it("BOOT-RESTORE-004c: an empty database is loaded from the archive's dump, with its key and uploads", async () => {
|
||||
const plan = postgresPlan(
|
||||
await archive({ 'travel.pgdump': 'dump', '.encryption_key': 'b'.repeat(64), 'uploads/a.jpg': 'jpeg' }),
|
||||
);
|
||||
const out = await restoreOnFirstBoot(plan);
|
||||
expect(out.restored).toBe(true);
|
||||
expect(loaded()).toBe(true);
|
||||
expect(fs.readFileSync(path.join(dataDir, '.encryption_key'), 'utf8')).toBe('b'.repeat(64));
|
||||
expect(fs.existsSync(reclaimResetFile)).toBe(true);
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-004d: a SQLite backup is refused before anything is loaded, leaving no scratch', async () => {
|
||||
const plan = postgresPlan(await goodArchive());
|
||||
await expect(restoreOnFirstBoot(plan)).rejects.toThrow(
|
||||
'RESTORE_FROM_BACKUP: This backup holds a SQLite database, which cannot be restored into a Postgres database.',
|
||||
);
|
||||
expect(loaded()).toBe(false);
|
||||
expect(fs.existsSync(reclaimResetFile)).toBe(false);
|
||||
expect(fs.readdirSync(dataDir).filter((f) => f.startsWith('restore-boot-'))).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-005: an archive without a database is refused and leaves no scratch behind', async () => {
|
||||
const zipPath = await archive({ 'notes.txt': 'hello' });
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
|
||||
|
||||
@@ -1,40 +1,142 @@
|
||||
/**
|
||||
* PostgresDatabaseBackup: the backup port on a Postgres database before TREK
|
||||
* can dump one. Every path fails closed, before anything is written or swapped.
|
||||
* PostgresDatabaseBackup: the backup port on a Postgres database, over
|
||||
* stand-in pg_dump, pg_restore and psql scripts (TREK_DB_PG_BIN_DIR) and a
|
||||
* stand-in DatabaseLifecycle. The tools' own arguments are pinned in
|
||||
* tests/unit/db/engine/postgres/pg-tools.test.ts.
|
||||
*/
|
||||
import { resolveEngine } from '../../../src/db/engine';
|
||||
import { POSTGRES_BACKUP_UNAVAILABLE, PostgresDatabaseBackup } from '../../../src/nest/backup/postgres-database-backup';
|
||||
import { knownMigrationNames } from '../../../src/db/known-migrations';
|
||||
import type { DataPathsService } from '../../../src/nest/app-config/data-paths.service';
|
||||
import { PostgresDatabaseBackup } from '../../../src/nest/backup/postgres-database-backup';
|
||||
import { DomainError } from '../../../src/nest/common/domain-error';
|
||||
import { DatabaseConnectionLostError } from '../../../src/nest/database/database-backup.interface';
|
||||
import type { DatabaseLifecycle } from '../../../src/nest/database/database-lifecycle.service';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const KNOWN = [...knownMigrationNames()].sort();
|
||||
let root: string;
|
||||
let bin: string;
|
||||
let extract: string;
|
||||
let reopen: ReturnType<typeof vi.fn>;
|
||||
let backup: PostgresDatabaseBackup;
|
||||
|
||||
/** A stand-in tool: `body` is the shell it runs. */
|
||||
function tool(name: 'pg_dump' | 'pg_restore' | 'psql', body: string): void {
|
||||
const version = `[ "$1" = --version ] && { echo '${name} (PostgreSQL) 17.11'; exit 0; }`;
|
||||
writeFileSync(path.join(bin, name), `#!/bin/sh\n${version}\n${body}\n`);
|
||||
chmodSync(path.join(bin, name), 0o755);
|
||||
}
|
||||
const WRITES_ITS_FILE = 'for a in "$@"; do case "$a" in --file=*) printf dump > "${a#--file=}";; esac; done';
|
||||
const FAILS = 'echo "connection refused" >&2; exit 1';
|
||||
/** A pg_restore whose --data-only listing shows `names` as the dump's migrations. */
|
||||
const LISTS = (names: string[]) =>
|
||||
[
|
||||
'case "$*" in *--data-only*)',
|
||||
" printf 'COPY public.mikro_orm_migrations (id, name, executed_at) FROM stdin;\\n'",
|
||||
...names.map((name, i) => ` printf '${i + 1}\\t${name}\\tx\\n'`),
|
||||
" printf '\\\\.\\n';;",
|
||||
'esac',
|
||||
].join('\n');
|
||||
|
||||
beforeEach(() => {
|
||||
root = mkdtempSync(path.join(tmpdir(), 'trek-pg-backup-'));
|
||||
bin = path.join(root, 'bin');
|
||||
extract = path.join(root, 'extract');
|
||||
mkdirSync(bin);
|
||||
mkdirSync(extract);
|
||||
vi.stubEnv('TREK_DB_URL', 'postgresql://trek:pw@db/trek');
|
||||
vi.stubEnv('TREK_DB_PG_BIN_DIR', bin);
|
||||
reopen = vi.fn(async () => undefined);
|
||||
backup = new PostgresDatabaseBackup(
|
||||
{ reopen } as unknown as DatabaseLifecycle,
|
||||
{ dataDir: root } as unknown as DataPathsService,
|
||||
);
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
vi.restoreAllMocks();
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function refusal(promise: Promise<unknown>): Promise<{ status: number; message: string }> {
|
||||
const error = await promise.then(
|
||||
() => null,
|
||||
(err: unknown) => err,
|
||||
);
|
||||
expect(error).toBeInstanceOf(DomainError);
|
||||
return { status: (error as DomainError).getStatus(), message: (error as DomainError).publicMessage };
|
||||
}
|
||||
|
||||
describe('PostgresDatabaseBackup', () => {
|
||||
const backup = new PostgresDatabaseBackup();
|
||||
|
||||
it('PGBACKUP-001: a backup attempt reports that it wrote nothing, rather than an archive without the database', async () => {
|
||||
it('PGBACKUP-001: a snapshot is a pg_dump; a failed one leaves nothing that could pass for a copy', async () => {
|
||||
expect(backup.archiveEntry).toBe('travel.pgdump');
|
||||
expect(backup.canSnapshot()).toBe(true);
|
||||
await expect(backup.checkpoint()).resolves.toBeUndefined();
|
||||
await expect(backup.snapshot()).rejects.toThrow(POSTGRES_BACKUP_UNAVAILABLE);
|
||||
tool('pg_dump', WRITES_ITS_FILE);
|
||||
await backup.snapshot(path.join(root, 'a.pgdump'));
|
||||
expect(existsSync(path.join(root, 'a.pgdump'))).toBe(true);
|
||||
tool('pg_dump', `printf partial > "$(echo "$4" | sed 's/--file=//')"; ${FAILS}`);
|
||||
await expect(backup.snapshot(path.join(root, 'b.pgdump'))).rejects.toThrow('pg_dump failed: connection refused');
|
||||
expect(existsSync(path.join(root, 'b.pgdump'))).toBe(false);
|
||||
expect(existsSync(path.join(root, 'b.pgdump.partial'))).toBe(false);
|
||||
});
|
||||
|
||||
it('PGBACKUP-002: a restore is refused at the verify step with a 400, and nothing is swapped', async () => {
|
||||
const refusal = (() => {
|
||||
try {
|
||||
return backup.verify();
|
||||
} catch (error) {
|
||||
return error;
|
||||
}
|
||||
})();
|
||||
expect(refusal).toBeInstanceOf(DomainError);
|
||||
expect((refusal as DomainError).getStatus()).toBe(400);
|
||||
expect((refusal as DomainError).publicMessage).toBe(POSTGRES_BACKUP_UNAVAILABLE);
|
||||
it('PGBACKUP-002: verify takes a TREK dump this build knows, and refuses a missing, SQLite, foreign or newer one', async () => {
|
||||
expect(await refusal(backup.verify(extract))).toEqual({
|
||||
status: 400,
|
||||
message: 'Invalid backup: travel.pgdump not found',
|
||||
});
|
||||
writeFileSync(path.join(extract, 'travel.db'), 'sqlite');
|
||||
expect((await refusal(backup.verify(extract))).message).toMatch(/holds a SQLite database/);
|
||||
writeFileSync(path.join(extract, 'travel.pgdump'), 'dump');
|
||||
tool('pg_restore', LISTS(KNOWN.slice(0, 3)));
|
||||
await expect(backup.verify(extract)).resolves.toBeNull();
|
||||
tool('pg_restore', LISTS([...KNOWN, 'Migration29990101000000_from_the_future']));
|
||||
expect((await refusal(backup.verify(extract))).message).toMatch(/made by a newer TREK version/);
|
||||
tool('pg_restore', LISTS([]));
|
||||
expect((await refusal(backup.verify(extract))).message).toMatch(/holds no TREK migration history/);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
tool('pg_restore', FAILS);
|
||||
expect((await refusal(backup.verify(extract))).message).toBe('Uploaded file is not a valid Postgres dump');
|
||||
});
|
||||
|
||||
it('PGBACKUP-003: keeps a dump of the database a restore replaces, or says it could not', async () => {
|
||||
vi.spyOn(console, 'log').mockImplementation(() => undefined);
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
tool('pg_dump', WRITES_ITS_FILE);
|
||||
const kept = await backup.keepCopyBeforeRestore();
|
||||
expect(kept).toMatch(/pre-restore-\d+\.pgdump$/);
|
||||
expect(existsSync(kept as string)).toBe(true);
|
||||
tool('pg_dump', FAILS);
|
||||
await expect(backup.keepCopyBeforeRestore()).resolves.toBeNull();
|
||||
await expect(backup.replace()).rejects.toThrow(POSTGRES_BACKUP_UNAVAILABLE);
|
||||
});
|
||||
|
||||
it('PGBACKUP-003: names the database the way the engine describes it (no password) and points at pg_dump', () => {
|
||||
expect(backup.location()).toBe(resolveEngine().describe());
|
||||
expect(backup.archiveEntry).toBe('database.dump');
|
||||
expect(POSTGRES_BACKUP_UNAVAILABLE).toContain('pg_dump');
|
||||
it('PGBACKUP-004: replace restores, then reopens; a reopen failure is reported, not thrown', async () => {
|
||||
tool('pg_restore', "printf '%s\\n' '\\restrict k1' > \"${3#--file=}\"");
|
||||
tool('psql', 'exit 0');
|
||||
await expect(backup.replace(path.join(extract, 'travel.pgdump'))).resolves.toEqual({ reopenError: null });
|
||||
expect(reopen).toHaveBeenCalledTimes(1);
|
||||
const lost = new Error('reconnect failed');
|
||||
reopen.mockRejectedValueOnce(lost);
|
||||
await expect(backup.replace(path.join(extract, 'travel.pgdump'))).resolves.toEqual({ reopenError: lost });
|
||||
});
|
||||
|
||||
it('PGBACKUP-005: a failed restore still reopens, and both failures surface when the reopen fails too', async () => {
|
||||
tool('pg_restore', FAILS);
|
||||
tool('psql', 'exit 0');
|
||||
const source = path.join(extract, 'travel.pgdump');
|
||||
await expect(backup.replace(source)).rejects.toThrow('pg_restore failed: connection refused');
|
||||
expect(reopen).toHaveBeenCalledTimes(1);
|
||||
reopen.mockRejectedValueOnce(new Error('reconnect failed'));
|
||||
await expect(backup.replace(source)).rejects.toBeInstanceOf(DatabaseConnectionLostError);
|
||||
});
|
||||
|
||||
it('PGBACKUP-006: names the database without its password, and refuses to run without TREK_DB_URL', async () => {
|
||||
expect(backup.location()).toBe('Postgres postgresql://trek:***@db/trek');
|
||||
vi.stubEnv('TREK_DB_URL', '');
|
||||
await expect(backup.snapshot(path.join(root, 'c.pgdump'))).rejects.toThrow(/without TREK_DB_URL/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -146,8 +146,8 @@ describe('SqliteDatabaseBackup', () => {
|
||||
expect(fs.existsSync(target)).toBe(false);
|
||||
});
|
||||
|
||||
it('SQLBK-007: verify refuses an unpacked archive without a database', () => {
|
||||
expect(portWith(maintenanceStub()).verify(dir)).toEqual({
|
||||
it('SQLBK-007: verify refuses an unpacked archive without a database', async () => {
|
||||
await expect(portWith(maintenanceStub()).verify(dir)).resolves.toEqual({
|
||||
error: 'Invalid backup: travel.db not found',
|
||||
status: 400,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user