mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fc9e634d86
On Postgres the backup port dumps with pg_dump and restores by turning the dump into a script with pg_restore and having psql clear the current schema and run it in one transaction. A dump only recreates what it holds, so a plain pg_restore --clean would leave a later migration's tables behind and break the migrations the reopen runs; a restore that fails leaves the database as it was. A restore refuses a pg_restore or psql older than 17.6/16.10/15.14/14.19, whose script would lack the \restrict line that keeps a psql meta-command smuggled into an uploaded dump from running (CVE-2025-8714). A restore refuses a SQLite archive, a dump without TREK's migration history or one a newer TREK wrote before anything is replaced, and keeps a dump of the database it replaces. RESTORE_FROM_BACKUP loads the archive's dump the same way into a database that holds no table yet, and is ignored with a warning once it holds one. Before pending migrations the boot now takes a pg_dump copy to data/pre-migrate-<last migration>-<time>.pgdump under the same TREK_DB_PRE_MIGRATE_SNAPSHOT(_KEEP) policy as the SQLite file, instead of refusing; a copy that cannot be written refuses the boot. The tools run by absolute path (TREK_DB_PG_BIN_DIR, or the usual directories), never through PATH, with the password in PGPASSWORD; an sslmode that asks for SSL verifies the certificate as the app pool does. Dumps are written beside their target and renamed into place, readable by their owner only, and a tool's stderr reaches errors without the lines that quote SQL or rows. The image installs postgresql-client-17 from Debian 13.
241 lines
9.7 KiB
TypeScript
241 lines
9.7 KiB
TypeScript
/**
|
|
* The Postgres client tools (src/db/engine/postgres/pg-tools.ts), run against
|
|
* stand-in pg_dump, pg_restore and psql scripts in a temp directory: the exact
|
|
* arguments and environment each call hands the binary, the password kept out
|
|
* of the arguments, the files a dump and a restore leave behind, and the
|
|
* refusal when a tool is missing. The real tools against a real database are
|
|
* exercised by the Postgres lane.
|
|
*/
|
|
import {
|
|
hasRestrictFix,
|
|
pgConnectionEnv,
|
|
pgDump,
|
|
pgDumpMigrations,
|
|
pgRestore,
|
|
PgToolMissingError,
|
|
resolvePgTool,
|
|
} from '../../../../../src/db/engine/postgres/pg-tools';
|
|
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
rmSync,
|
|
statSync,
|
|
writeFileSync,
|
|
} from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
|
|
const URL_WITH_SECRET = 'postgresql://trek:s3cr%40t@db.example:6543/trek_db?sslmode=require';
|
|
|
|
let bin: string;
|
|
let out: string;
|
|
|
|
/** pg_restore writing its --file (the third argument) as 17.6+ does: opened with a \\restrict line. */
|
|
const WRITES_RESTRICTED_SCRIPT = "printf '%s\\n' '\\restrict k1' 'SELECT 1;' > \"${3#--file=}\"";
|
|
|
|
interface FakeOptions {
|
|
stdout?: string;
|
|
stderr?: string;
|
|
exitCode?: number;
|
|
/** Extra shell the tool runs before it exits. */
|
|
body?: string;
|
|
/** What `--version` answers. */
|
|
version?: string;
|
|
}
|
|
|
|
/** A stand-in tool that records its arguments and libpq environment, then prints `stdout`. */
|
|
function fakeTool(
|
|
name: string,
|
|
{ stdout = '', stderr = '', exitCode = 0, body = '', version = '17.11' }: FakeOptions = {},
|
|
): string {
|
|
const log = path.join(bin, `${name}.log`);
|
|
writeFileSync(
|
|
path.join(bin, name),
|
|
[
|
|
'#!/bin/sh',
|
|
`[ "$1" = --version ] && { echo '${name} (PostgreSQL) ${version} (Debian)'; exit 0; }`,
|
|
`printf '%s\\n' "$@" > '${log}'`,
|
|
`env | grep '^PG' | sort >> '${log}'`,
|
|
body,
|
|
`printf '%s' '${stdout}'`,
|
|
`printf '%s' '${stderr}' >&2`,
|
|
`exit ${exitCode}`,
|
|
'',
|
|
].join('\n'),
|
|
);
|
|
chmodSync(path.join(bin, name), 0o755);
|
|
return log;
|
|
}
|
|
|
|
const argsOf = (log: string): string[] =>
|
|
readFileSync(log, 'utf8')
|
|
.split('\n')
|
|
.filter((line) => !line.startsWith('PG'));
|
|
|
|
beforeEach(() => {
|
|
bin = mkdtempSync(path.join(tmpdir(), 'trek-pg-bin-'));
|
|
out = mkdtempSync(path.join(tmpdir(), 'trek-pg-out-'));
|
|
});
|
|
afterEach(() => {
|
|
rmSync(bin, { recursive: true, force: true });
|
|
rmSync(out, { recursive: true, force: true });
|
|
});
|
|
|
|
describe('pg-tools', () => {
|
|
it('PGTOOLS-001: finds a tool by absolute path in the configured directory, and nowhere else', () => {
|
|
fakeTool('pg_dump');
|
|
writeFileSync(path.join(bin, 'psql'), 'not executable');
|
|
expect(resolvePgTool('pg_dump', bin)).toBe(path.join(bin, 'pg_dump'));
|
|
expect(resolvePgTool('pg_restore', bin)).toBeNull();
|
|
expect(resolvePgTool('psql', bin)).toBeNull();
|
|
});
|
|
|
|
it('PGTOOLS-002: the connection goes in the libpq environment, the password decoded and never an argument', () => {
|
|
expect(pgConnectionEnv(URL_WITH_SECRET)).toEqual({
|
|
PGHOST: 'db.example',
|
|
PGPORT: '6543',
|
|
PGDATABASE: 'trek_db',
|
|
PGUSER: 'trek',
|
|
PGPASSWORD: 's3cr@t',
|
|
PGCONNECT_TIMEOUT: '10',
|
|
PGSSLMODE: 'verify-full',
|
|
PGSSLROOTCERT: 'system',
|
|
});
|
|
expect(pgConnectionEnv('postgres://u@[::1]/d')).toEqual({
|
|
PGHOST: '::1',
|
|
PGPORT: '5432',
|
|
PGDATABASE: 'd',
|
|
PGUSER: 'u',
|
|
PGCONNECT_TIMEOUT: '10',
|
|
});
|
|
});
|
|
|
|
it('PGTOOLS-003: SSL is verified as the app pool verifies it, against the URL root certificate when it names one', () => {
|
|
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable')).toMatchObject({ PGSSLMODE: 'disable' });
|
|
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable').PGSSLROOTCERT).toBeUndefined();
|
|
expect(pgConnectionEnv('postgres://u@h/d?sslmode=verify-ca&sslrootcert=%2Fca%2Froot.crt')).toMatchObject({
|
|
PGSSLMODE: 'verify-full',
|
|
PGSSLROOTCERT: '/ca/root.crt',
|
|
});
|
|
});
|
|
|
|
it('PGTOOLS-004: pg_dump writes a private custom-format dump beside the target and renames it into place', async () => {
|
|
const log = fakeTool('pg_dump', { body: 'printf dump >> "${4#--file=}"' });
|
|
const target = path.join(out, 'x.pgdump');
|
|
await pgDump(URL_WITH_SECRET, target, bin);
|
|
expect(argsOf(log).slice(0, 4)).toEqual([
|
|
'--format=custom',
|
|
'--no-owner',
|
|
'--no-privileges',
|
|
`--file=${target}.partial`,
|
|
]);
|
|
expect(readFileSync(log, 'utf8')).toContain('PGPASSWORD=s3cr@t');
|
|
expect(argsOf(log).join(' ')).not.toContain('s3cr');
|
|
expect(readFileSync(target, 'utf8')).toBe('dump');
|
|
expect(statSync(target).mode & 0o777).toBe(0o600);
|
|
expect(readdirSync(out)).toEqual(['x.pgdump']);
|
|
});
|
|
|
|
it('PGTOOLS-005: a restore clears the schema and loads the script in one psql transaction, then removes its work files', async () => {
|
|
const source = path.join(out, 'in.pgdump');
|
|
const restoreLog = fakeTool('pg_restore', { body: WRITES_RESTRICTED_SCRIPT });
|
|
const psqlLog = fakeTool('psql', { body: `cp "\${6#--file=}" '${path.join(bin, 'clear.sql')}'` });
|
|
await pgRestore(URL_WITH_SECRET, source, bin);
|
|
|
|
const restoreArgs = argsOf(restoreLog);
|
|
expect(restoreArgs.slice(0, 2)).toEqual(['--no-owner', '--no-privileges']);
|
|
expect(restoreArgs[2]).toMatch(/^--file=.*\.pg-restore-[^/]+\/restore\.sql$/);
|
|
expect(restoreArgs.slice(3, 5)).toEqual(['--', source]);
|
|
expect(readFileSync(restoreLog, 'utf8')).not.toContain('PGPASSWORD');
|
|
|
|
const psqlArgs = argsOf(psqlLog);
|
|
expect(psqlArgs.slice(0, 5)).toEqual([
|
|
'--no-psqlrc',
|
|
'--quiet',
|
|
'--single-transaction',
|
|
'--set=ON_ERROR_STOP=1',
|
|
'--output=/dev/null',
|
|
]);
|
|
expect(psqlArgs[5]).toMatch(/clear-schema\.sql$/);
|
|
expect(psqlArgs[6]).toBe(restoreArgs[2]);
|
|
expect(readFileSync(psqlLog, 'utf8')).toContain('PGOPTIONS=-c lock_timeout=60s -c client_min_messages=warning');
|
|
expect(readFileSync(path.join(bin, 'clear.sql'), 'utf8')).toMatch(/DROP %s IF EXISTS %s CASCADE/);
|
|
expect(readdirSync(out)).toEqual([]);
|
|
});
|
|
|
|
it("PGTOOLS-006: reads the dump's recorded migrations from its COPY block, and none from a dump without them", async () => {
|
|
const copy = [
|
|
'SET statement_timeout = 0;',
|
|
'COPY public.mikro_orm_migrations (id, name, executed_at) FROM stdin;',
|
|
'1\tMigration20200101000000_baseline_schema\t2026-01-01 00:00:00+00',
|
|
'2\tMigration20200101000100_second\t2026-01-01 00:00:00+00',
|
|
'\\.',
|
|
'',
|
|
].join('\n');
|
|
const log = fakeTool('pg_restore', { stdout: copy });
|
|
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([
|
|
'Migration20200101000000_baseline_schema',
|
|
'Migration20200101000100_second',
|
|
]);
|
|
expect(argsOf(log).slice(0, 5)).toEqual([
|
|
'--data-only',
|
|
'--table=mikro_orm_migrations',
|
|
'--file=-',
|
|
'--',
|
|
'/data/in.pgdump',
|
|
]);
|
|
fakeTool('pg_restore', { stdout: 'SET statement_timeout = 0;\n' });
|
|
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([]);
|
|
});
|
|
|
|
it('PGTOOLS-007: a failing tool reports its own message without the lines that quote SQL or rows, and leaves no file', async () => {
|
|
fakeTool('pg_dump', {
|
|
exitCode: 1,
|
|
stderr: 'pg_dump: error: query failed\nCommand was: SELECT secret_row\nDETAIL: Key (email)=(a@b.c)\n',
|
|
});
|
|
const target = path.join(out, 'x.pgdump');
|
|
await expect(pgDump(URL_WITH_SECRET, target, bin)).rejects.toThrow(
|
|
/^pg_dump failed: pg_dump: error: query failed$/,
|
|
);
|
|
expect(readdirSync(out)).toEqual([]);
|
|
|
|
fakeTool('pg_dump', { exitCode: 1, stderr: 'x'.repeat(5000) });
|
|
await expect(pgDump(URL_WITH_SECRET, target, bin)).rejects.toThrow(/^pg_dump failed: x{2000}…$/);
|
|
});
|
|
|
|
it('PGTOOLS-008: a missing tool says how to install it, and a restore that cannot run leaves no work files', async () => {
|
|
const source = path.join(out, 'in.pgdump');
|
|
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(PgToolMissingError);
|
|
await expect(pgDump(URL_WITH_SECRET, path.join(out, 'x.pgdump'), bin)).rejects.toThrow(/TREK_DB_PG_BIN_DIR/);
|
|
fakeTool('pg_restore', { body: WRITES_RESTRICTED_SCRIPT });
|
|
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(/psql was not found/);
|
|
expect(existsSync(out) && readdirSync(out)).toEqual([]);
|
|
});
|
|
|
|
it('PGTOOLS-009: client tools without the \\restrict fix (CVE-2025-8714) never turn a dump into a script', async () => {
|
|
fakeTool('pg_restore', { version: '16.9', body: `touch '${path.join(bin, 'ran')}'` });
|
|
fakeTool('psql', { body: `touch '${path.join(bin, 'ran')}'` });
|
|
await expect(pgRestore(URL_WITH_SECRET, path.join(out, 'in.pgdump'), bin)).rejects.toThrow(
|
|
/pg_restore is too old to restore safely \(pg_restore \(PostgreSQL\) 16\.9 \(Debian\)\): it must be at least 17\.6/,
|
|
);
|
|
fakeTool('pg_restore', { body: `touch '${path.join(bin, 'ran')}'` });
|
|
fakeTool('psql', { version: '17.5', body: `touch '${path.join(bin, 'ran')}'` });
|
|
await expect(pgRestore(URL_WITH_SECRET, path.join(out, 'in.pgdump'), bin)).rejects.toThrow(/psql is too old/);
|
|
expect(existsSync(path.join(bin, 'ran'))).toBe(false);
|
|
expect(readdirSync(out)).toEqual([]);
|
|
});
|
|
|
|
it('PGTOOLS-010: knows which client versions carry the \\restrict fix', () => {
|
|
const fixed = ['14.19', '15.14', '16.10', '17.6', '17.11', '18.0', '19.1'];
|
|
const unfixed = ['13.30', '14.18', '15.13', '16.9', '17.5'];
|
|
for (const v of fixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(true);
|
|
for (const v of unfixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(false);
|
|
expect(hasRestrictFix('not a version')).toBe(false);
|
|
});
|
|
});
|