mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
fix(db): check the Postgres server and tools before a dump or a restore
- The client tools are looked for newest first, from 18 down: a client handles every server up to its own major and refuses a newer one. A dump or a restore now refuses a client older than the server up front, naming both versions, and the boot refuses a server outside PostgreSQL 14 to 18. - sslrootcert=system is used only by a client from libpq 16 on; an older client is refused with a message that names sslrootcert and the bin dir. - The restore's CVE-2025-8714 table covers 18. - A restore refuses a role that is a superuser or may run programs or write files on the database host, since the archive's SQL runs as that role; the boot warns about such a role. - pg_restore's script is streamed to its work file and stopped once it passes BACKUP_MAX_DECOMPRESSED_MB, so a small dump can no longer expand without limit on the data volume. - A failed tool's error keeps only the summarized stderr; its cause names the exit code and signal instead of carrying the process error, whose message and stderr quoted row data into the log. - The migration history is looked for in the current schema, where the tables and the backups are, so a role whose search_path is not public boots a second time.
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { execFile, spawn } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Transform } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -10,10 +12,13 @@ export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
|
||||
|
||||
/**
|
||||
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
|
||||
* Debian's versioned directories newest first (the Docker image installs 17),
|
||||
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
|
||||
* Debian's versioned directories newest first (the Docker image installs 18),
|
||||
* then the usual prefixes. Newest first because a client dumps and restores
|
||||
* every server up to its own major version and refuses a newer one. Never
|
||||
* PATH: a binary is run by its absolute path.
|
||||
*/
|
||||
const DEFAULT_BIN_DIRS = [
|
||||
'/usr/lib/postgresql/18/bin',
|
||||
'/usr/lib/postgresql/17/bin',
|
||||
'/usr/lib/postgresql/16/bin',
|
||||
'/usr/lib/postgresql/15/bin',
|
||||
@@ -29,8 +34,12 @@ const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
||||
const CONNECT_TIMEOUT_SECONDS = '10';
|
||||
/** How much of a tool's stderr an error keeps. */
|
||||
const STDERR_LIMIT = 2000;
|
||||
/** How much of a streaming tool's stderr is held while it runs, before it is summarized. */
|
||||
const STDERR_CAPTURE_LIMIT = 64 * 1024;
|
||||
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
|
||||
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
|
||||
/** libpq reads `sslrootcert=system` as the system's trusted CAs from this major on; older clients read it as a file name. */
|
||||
const SYSTEM_ROOT_CERT_SINCE = 16;
|
||||
|
||||
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
|
||||
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
|
||||
@@ -51,7 +60,8 @@ export function resolvePgTool(tool: PgTool, binDir: string | undefined): string
|
||||
* PGPASSWORD, so it never appears in an argument list another process can
|
||||
* read. An sslmode that asks for SSL verifies the server's certificate, as the
|
||||
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
|
||||
* else the system's trusted CAs.
|
||||
* else the system's trusted CAs (`system`, which `run` accepts only from a
|
||||
* client that understands it).
|
||||
*/
|
||||
export function pgConnectionEnv(url: string): Record<string, string> {
|
||||
const parsed = new URL(url);
|
||||
@@ -83,6 +93,17 @@ export class PgToolMissingError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** A tool that wrote more than the caller allowed; it was stopped and its output is incomplete. */
|
||||
export class PgOutputLimitError extends Error {
|
||||
constructor(
|
||||
tool: PgTool,
|
||||
readonly limitBytes: number,
|
||||
) {
|
||||
super(`${tool} wrote more than ${limitBytes} bytes and was stopped`);
|
||||
this.name = 'PgOutputLimitError';
|
||||
}
|
||||
}
|
||||
|
||||
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
|
||||
function summarizeStderr(stderr: string | undefined): string {
|
||||
const kept = (stderr ?? '')
|
||||
@@ -92,10 +113,104 @@ function summarizeStderr(stderr: string | undefined): string {
|
||||
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* The error a failed tool run throws: `<tool> failed: <its stderr, summarized>`.
|
||||
* Its cause names only how the process ended (exit code, signal, whether the
|
||||
* timeout killed it), never the process error itself, whose message and
|
||||
* `stderr` hold the tool's whole stderr, row data included, and would reach
|
||||
* the log through the cause chain.
|
||||
*/
|
||||
function toolFailure(tool: PgTool, failure: unknown): Error {
|
||||
const { stderr, code, signal, killed } = (failure ?? {}) as {
|
||||
stderr?: unknown;
|
||||
code?: unknown;
|
||||
signal?: unknown;
|
||||
killed?: unknown;
|
||||
};
|
||||
const message = summarizeStderr(typeof stderr === 'string' ? stderr : undefined);
|
||||
const ending = signal ? `signal ${String(signal)}` : `code ${String(code)}`;
|
||||
const cause = Object.assign(new Error(`${tool} ended with ${ending}${killed === true ? ' (timed out)' : ''}`), {
|
||||
code,
|
||||
signal,
|
||||
killed,
|
||||
});
|
||||
return new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause });
|
||||
}
|
||||
|
||||
/** The major version in a tool's `--version` line (`pg_dump (PostgreSQL) 18.1 (Debian …)`), or null. */
|
||||
function toolMajor(version: string): number | null {
|
||||
const match = /\(PostgreSQL\) (\d+)/.exec(version);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
/** `binary --version`, read directly: the check before a run must not go through `run` itself. */
|
||||
async function binaryVersion(tool: PgTool, binary: string): Promise<string> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, ['--version'], { env: {}, timeout: TOOL_TIMEOUT_MS });
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
throw toolFailure(tool, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** The absolute path of `tool`, or the refusal that says how to install it. */
|
||||
function locate(tool: PgTool, binDir: string | undefined): string {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
return binary;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses to run a client older than libpq 16 with `sslrootcert=system`: it
|
||||
* would read `system` as a file name and fail on every connection. Such a
|
||||
* client needs the CA file named in the URL.
|
||||
*/
|
||||
async function refuseSystemRootCertOnOldClient(
|
||||
tool: PgTool,
|
||||
binary: string,
|
||||
env: Record<string, string>,
|
||||
): Promise<void> {
|
||||
if (env.PGSSLROOTCERT !== 'system') return;
|
||||
const version = await binaryVersion(tool, binary);
|
||||
const major = toolMajor(version);
|
||||
if (major !== null && major >= SYSTEM_ROOT_CERT_SINCE) return;
|
||||
throw new Error(
|
||||
`${tool} (${version.trim()}) cannot verify the server against the system's trusted certificate authorities, ` +
|
||||
`which needs a client of version ${SYSTEM_ROOT_CERT_SINCE} or later. Add sslrootcert=<CA file> to TREK_DB_URL, ` +
|
||||
'or point TREK_DB_PG_BIN_DIR at newer client tools (the Docker image has them).',
|
||||
);
|
||||
}
|
||||
|
||||
/** The major version of `tool` as TREK would run it. */
|
||||
async function pgToolMajor(tool: PgTool, binDir: string | undefined): Promise<number | null> {
|
||||
return toolMajor(await binaryVersion(tool, locate(tool, binDir)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses `tools` when one is older than the server it is about to talk to:
|
||||
* pg_dump refuses a newer server outright, and what an older pg_restore or
|
||||
* psql makes of a newer server's dump is not something a restore should find
|
||||
* out halfway. A newer client is fine; it handles every older server.
|
||||
*/
|
||||
export async function refuseClientOlderThanServer(
|
||||
tools: readonly PgTool[],
|
||||
binDir: string | undefined,
|
||||
serverMajor: number,
|
||||
): Promise<void> {
|
||||
for (const tool of tools) {
|
||||
const major = await pgToolMajor(tool, binDir);
|
||||
if (major !== null && major >= serverMajor) continue;
|
||||
throw new Error(
|
||||
`${tool} is version ${major ?? 'unknown'}, older than the PostgreSQL ${serverMajor} server, which it cannot ` +
|
||||
`back up or restore. Point TREK_DB_PG_BIN_DIR at client tools of version ${serverMajor} or later ` +
|
||||
'(the Docker image has the newest supported ones).',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `tool` by its absolute path and resolves with its stdout. A failure
|
||||
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
|
||||
* the cause.
|
||||
* throws `<tool> failed: <its stderr, summarized>`.
|
||||
*/
|
||||
export async function run(
|
||||
tool: PgTool,
|
||||
@@ -103,8 +218,8 @@ export async function run(
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
const binary = locate(tool, binDir);
|
||||
await refuseSystemRootCertOnOldClient(tool, binary, env);
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, args, {
|
||||
env,
|
||||
@@ -113,7 +228,55 @@ export async function run(
|
||||
});
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const message = summarizeStderr((err as { stderr?: string }).stderr);
|
||||
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
|
||||
throw toolFailure(tool, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** How a spawned tool ended: its exit, or the error that kept it from starting. */
|
||||
type Ending = { code: number | null; signal: NodeJS.Signals | null } | { error: Error };
|
||||
|
||||
/**
|
||||
* Runs `tool` and streams its stdout into `target` (which must exist; its mode
|
||||
* is kept), stopping the tool with {@link PgOutputLimitError} once more than
|
||||
* `maxBytes` have come out. The caller removes `target` on failure. A failure
|
||||
* of the tool itself throws as `run` does.
|
||||
*/
|
||||
export async function runToFile(
|
||||
tool: PgTool,
|
||||
binDir: string | undefined,
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
target: string,
|
||||
maxBytes: number,
|
||||
): Promise<void> {
|
||||
const binary = locate(tool, binDir);
|
||||
await refuseSystemRootCertOnOldClient(tool, binary, env);
|
||||
const child = spawn(binary, args, { env, stdio: ['ignore', 'pipe', 'pipe'], timeout: TOOL_TIMEOUT_MS });
|
||||
const ended = new Promise<Ending>((resolve) => {
|
||||
child.once('error', (error) => resolve({ error }));
|
||||
child.once('close', (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
let stderr = '';
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
if (stderr.length < STDERR_CAPTURE_LIMIT) stderr += chunk;
|
||||
});
|
||||
let written = 0;
|
||||
const limit = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
written += chunk.length;
|
||||
if (written > maxBytes) callback(new PgOutputLimitError(tool, maxBytes));
|
||||
else callback(null, chunk);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await pipeline(child.stdout, limit, fs.createWriteStream(target, { flags: 'w' }));
|
||||
} catch (err) {
|
||||
child.kill('SIGKILL');
|
||||
await ended;
|
||||
throw err instanceof PgOutputLimitError ? err : toolFailure(tool, { stderr, code: 'stream' });
|
||||
}
|
||||
const ending = await ended;
|
||||
if ('error' in ending) throw toolFailure(tool, { stderr, code: (ending.error as { code?: unknown }).code });
|
||||
if (ending.code !== 0) throw toolFailure(tool, { stderr, code: ending.code, signal: ending.signal });
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { CURRENT_SCHEMA_TABLES_SQL } from '../../dialect/catalog';
|
||||
import { CLEAR_SCHEMA_SQL } from './clear-schema';
|
||||
import { pgConnectionEnv, run } from './pg-process';
|
||||
import { pgConnectionEnv, PgOutputLimitError, refuseClientOlderThanServer, run, runToFile } from './pg-process';
|
||||
import { refuseToolsWithoutRestrict } from './restore-safety';
|
||||
import { PRIVILEGED_ROLE_SQL, serverMajor } from './server-checks';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
@@ -15,6 +16,13 @@ import readline from 'node:readline';
|
||||
*/
|
||||
const RESTORE_SESSION_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
|
||||
|
||||
/** Why a restore refuses a role that could reach the database host (`PRIVILEGED_ROLE_SQL`). */
|
||||
const PRIVILEGED_RESTORE_REFUSAL =
|
||||
'TREK_DB_URL connects as a superuser, or as a role that may run programs or write files on the database server. ' +
|
||||
'A backup archive is a SQL script that runs as this role, so a crafted one could take over the database host; ' +
|
||||
'nothing was replaced. Connect TREK as a role that owns its database and is not a superuser ' +
|
||||
'(the wiki page "Database: Postgres" shows how).';
|
||||
|
||||
/** An empty file only its owner can read: the tool writing into it keeps the mode. */
|
||||
function createPrivateFile(file: string): void {
|
||||
fs.writeFileSync(file, '', { mode: 0o600 });
|
||||
@@ -26,42 +34,53 @@ interface PgSchema {
|
||||
quoted: string;
|
||||
}
|
||||
|
||||
/** What a dump or a restore needs to know about the server before it runs. */
|
||||
interface PgServer {
|
||||
schema: PgSchema;
|
||||
/** The server's major version, which the client tools must reach. */
|
||||
major: number;
|
||||
/** Whether TREK's role could run programs or write files on the database host (`PRIVILEGED_ROLE_SQL`). */
|
||||
privileged: boolean;
|
||||
}
|
||||
|
||||
/** One line, tab-separated: the current schema, its quoted name, the server version, whether the role is privileged. */
|
||||
const SERVER_SQL =
|
||||
"select concat_ws(E'\\t', coalesce(current_schema(), ''), coalesce(quote_ident(current_schema()), ''), " +
|
||||
`current_setting('server_version_num'), (${PRIVILEGED_ROLE_SQL})::text)`;
|
||||
|
||||
/**
|
||||
* The current schema of the database at `url`, the one TREK creates its
|
||||
* tables in. A backup and a restore stay inside it: a hosted database
|
||||
* (Supabase) keeps the platform's own schemas, `auth` among them, beside it,
|
||||
* and they are neither TREK's to copy nor to replace.
|
||||
* tables in, with the server's version and what TREK's role may do there. A
|
||||
* backup and a restore stay inside that schema: a hosted database (Supabase)
|
||||
* keeps the platform's own schemas, `auth` among them, beside it, and they are
|
||||
* neither TREK's to copy nor to replace.
|
||||
*/
|
||||
async function pgCurrentSchema(url: string, binDir: string | undefined): Promise<PgSchema> {
|
||||
async function pgServer(url: string, binDir: string | undefined): Promise<PgServer> {
|
||||
const out = await run(
|
||||
'psql',
|
||||
binDir,
|
||||
[
|
||||
'--no-psqlrc',
|
||||
'--no-align',
|
||||
'--tuples-only',
|
||||
"--command=select concat_ws(E'\\t', current_schema(), quote_ident(current_schema()))",
|
||||
],
|
||||
['--no-psqlrc', '--no-align', '--tuples-only', `--command=${SERVER_SQL}`],
|
||||
pgConnectionEnv(url),
|
||||
);
|
||||
const [name, quoted] = out.replace(/\n$/, '').split('\t');
|
||||
const [name, quoted, version, privileged] = out.replace(/\n$/, '').split('\t');
|
||||
if (!name || !quoted) {
|
||||
throw new Error('The database has no current schema: set the search_path of the role in TREK_DB_URL.');
|
||||
}
|
||||
return { name, quoted };
|
||||
return { schema: { name, quoted }, major: serverMajor(version), privileged: privileged === 'true' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a custom-format dump of the current schema of the database at `url`
|
||||
* to `target`, readable by its owner only. It is written beside the target and
|
||||
* renamed into place, so a dump that fails or is killed never leaves a file
|
||||
* that passes for a copy.
|
||||
* that passes for a copy. A pg_dump older than the server is refused first.
|
||||
*/
|
||||
export async function pgDump(url: string, target: string, binDir: string | undefined): Promise<void> {
|
||||
const partial = `${target}.partial`;
|
||||
try {
|
||||
createPrivateFile(partial);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
const { schema, major } = await pgServer(url, binDir);
|
||||
await refuseClientOlderThanServer(['pg_dump'], binDir, major);
|
||||
await run(
|
||||
'pg_dump',
|
||||
binDir,
|
||||
@@ -108,29 +127,61 @@ async function scriptHoldsTrekSchema(script: string, schema: PgSchema): Promise<
|
||||
}
|
||||
}
|
||||
|
||||
/** pg_restore's script for the dump at `source`, written to `script` and held to `maxBytes`. */
|
||||
async function writeRestoreScript(
|
||||
source: string,
|
||||
script: string,
|
||||
schema: PgSchema,
|
||||
binDir: string | undefined,
|
||||
maxBytes: number,
|
||||
): Promise<void> {
|
||||
const args = ['--no-owner', '--no-privileges', `--schema=${schema.name}`, '--file=-', '--', source];
|
||||
try {
|
||||
await runToFile('pg_restore', binDir, args, {}, script, maxBytes);
|
||||
} catch (err) {
|
||||
if (!(err instanceof PgOutputLimitError)) throw err;
|
||||
throw new Error(
|
||||
`The backup's database expands to more than ${Number((maxBytes / 1024 / 1024).toFixed(2))} MB of SQL, the limit ` +
|
||||
'BACKUP_MAX_DECOMPRESSED_MB sets; nothing was replaced. Raise the limit if the backup is genuine.',
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the current schema of the database at `url` with the dump at
|
||||
* `source`: pg_restore turns the dump's objects in that schema into a script
|
||||
* beside it (anything else the dump holds is left out), then psql clears the
|
||||
* schema and runs the script in one transaction, so a restore that fails
|
||||
* leaves the database as it was. A dump without TREK's tables in that schema
|
||||
* is refused before anything is cleared.
|
||||
* leaves the database as it was. Refused before anything is cleared:
|
||||
*
|
||||
* - client tools without the CVE-2025-8714 fix, or older than the server;
|
||||
* - a role that could run programs or write files on the database host,
|
||||
* since the script comes from an archive and runs as that role;
|
||||
* - a script past `maxScriptBytes` (BACKUP_MAX_DECOMPRESSED_MB): the dump is
|
||||
* compressed once more inside the archive, so its script is held to the
|
||||
* limit while pg_restore writes it, not only the archive's entry;
|
||||
* - a dump without TREK's tables in that schema.
|
||||
*
|
||||
* The work files are removed whatever happens.
|
||||
*/
|
||||
export async function pgRestore(url: string, source: string, binDir: string | undefined): Promise<void> {
|
||||
export async function pgRestore(
|
||||
url: string,
|
||||
source: string,
|
||||
binDir: string | undefined,
|
||||
maxScriptBytes: number,
|
||||
): Promise<void> {
|
||||
await refuseToolsWithoutRestrict(binDir);
|
||||
const schema = await pgCurrentSchema(url, binDir);
|
||||
const { schema, major, privileged } = await pgServer(url, binDir);
|
||||
if (privileged) throw new Error(PRIVILEGED_RESTORE_REFUSAL);
|
||||
await refuseClientOlderThanServer(['pg_restore', 'psql'], binDir, major);
|
||||
const workDir = fs.mkdtempSync(path.join(path.dirname(source), '.pg-restore-'));
|
||||
try {
|
||||
const clear = path.join(workDir, 'clear-schema.sql');
|
||||
const script = path.join(workDir, 'restore.sql');
|
||||
fs.writeFileSync(clear, CLEAR_SCHEMA_SQL, { mode: 0o600 });
|
||||
createPrivateFile(script);
|
||||
await run(
|
||||
'pg_restore',
|
||||
binDir,
|
||||
['--no-owner', '--no-privileges', `--schema=${schema.name}`, `--file=${script}`, '--', source],
|
||||
{},
|
||||
);
|
||||
await writeRestoreScript(source, script, schema, binDir, maxScriptBytes);
|
||||
if (!(await scriptHoldsTrekSchema(script, schema))) {
|
||||
throw new Error(`The backup holds no TREK tables in the schema "${schema.name}"; nothing was replaced.`);
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ import { run } from './pg-process';
|
||||
* fixed pg_restore opens the script with `\restrict <random key>`, and a fixed
|
||||
* psql then refuses every meta-command until the matching `\unrestrict`.
|
||||
*
|
||||
* RESTRICT_SINCE holds, per major, the first minor that carries the fix. Every
|
||||
* major after the last one listed has it from its first release.
|
||||
* RESTRICT_SINCE holds, per major, the first minor that carries the fix, for
|
||||
* every major TREK supports (14 to 18; 18 has it from 18.0). Every major after
|
||||
* the last one listed has it from its first release.
|
||||
*/
|
||||
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6 };
|
||||
const RESTRICT_SINCE: Readonly<Partial<Record<number, number>>> = { 14: 19, 15: 14, 16: 10, 17: 6, 18: 0 };
|
||||
const LAST_MAJOR_WITHOUT_IT = Math.max(...Object.keys(RESTRICT_SINCE).map(Number));
|
||||
|
||||
/** Whether `version` (`pg_restore (PostgreSQL) 17.11 (Debian …)`) carries the fix. */
|
||||
@@ -24,7 +25,7 @@ export function hasRestrictFix(version: string): boolean {
|
||||
return firstFixedMinor !== undefined && minor >= firstFixedMinor;
|
||||
}
|
||||
|
||||
/** The minimum versions as the error names them, newest major first: "17.6, 16.10, 15.14 or 14.19". */
|
||||
/** The minimum versions as the error names them, newest major first: "18.0, 17.6, 16.10, 15.14 or 14.19". */
|
||||
function minimumVersionsText(): string {
|
||||
const floors = Object.keys(RESTRICT_SINCE)
|
||||
.map(Number)
|
||||
@@ -45,6 +46,6 @@ export async function refuseToolsWithoutRestrict(binDir: string | undefined): Pr
|
||||
if (stale < 0) return;
|
||||
throw new Error(
|
||||
`${tools[stale]} is too old to restore safely (${versions[stale].trim()}): it must be at least ${minimumVersionsText()} ` +
|
||||
'(the Docker image has 17). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
|
||||
'(the Docker image has 18). Upgrade the PostgreSQL client tools or point TREK_DB_PG_BIN_DIR at newer ones.',
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,12 @@ import type { AnyOrm } from '../../orm';
|
||||
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
|
||||
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
|
||||
import { pgDump } from './pg-tools';
|
||||
import {
|
||||
PRIVILEGED_ROLE_SQL,
|
||||
PRIVILEGED_ROLE_WARNING,
|
||||
refuseUnsupportedServer,
|
||||
serverMajor,
|
||||
} from './server-checks';
|
||||
import type { Connection } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { Logger } from '@nestjs/common';
|
||||
@@ -77,6 +83,49 @@ async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDep
|
||||
refuseNewerDatabase(executed, deps.known, await deps.safetyNet.restoreHint(thisRelease));
|
||||
}
|
||||
|
||||
/** The migration history's table, without a schema. */
|
||||
const HISTORY_TABLE = 'mikro_orm_migrations';
|
||||
|
||||
/** What the boot reads about the server before it writes anything. */
|
||||
interface ServerFacts {
|
||||
schema: string | null;
|
||||
version: string;
|
||||
privileged: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a server version TREK does not support, warns about a role that
|
||||
* could reach the database host (a restore refuses it), and points MikroORM's
|
||||
* migration history at the current schema, where TREK's tables live.
|
||||
*
|
||||
* The history table is created without a schema, so it lands in the current
|
||||
* schema beside the tables, but MikroORM looks for it in `public` unless told
|
||||
* otherwise. On a role whose search_path resolves to another schema (a role
|
||||
* `trek` with a schema `trek`, which the default `"$user", public` picks) it
|
||||
* would find none there on the second boot and try to create it again. The
|
||||
* name is qualified here, before the migrator reads it; the migrator reads
|
||||
* the option on every use, so a restore's re-bootstrap sets it again alike.
|
||||
* Returns the table's name without the schema.
|
||||
*/
|
||||
async function checkServerAndPlaceHistory(orm: AnyOrm, connection: Connection): Promise<string> {
|
||||
const [facts] = await connection.execute<ServerFacts[]>(
|
||||
"select current_schema() as schema, current_setting('server_version_num') as version, " +
|
||||
`(${PRIVILEGED_ROLE_SQL}) as privileged`,
|
||||
);
|
||||
refuseUnsupportedServer(serverMajor(facts?.version));
|
||||
if (facts?.privileged) logger.warn(PRIVILEGED_ROLE_WARNING);
|
||||
const schema = facts?.schema;
|
||||
if (!schema || schema.includes('.')) {
|
||||
const why = schema ? `the current schema "${schema}", whose name holds a dot` : 'no current schema';
|
||||
throw new Error(
|
||||
`${REFUSAL} the role in TREK_DB_URL has ${why}. Set its search_path to a schema TREK may create its tables in.`,
|
||||
);
|
||||
}
|
||||
const migrations = orm.config.get('migrations');
|
||||
migrations.tableName = `${schema}.${HISTORY_TABLE}`;
|
||||
return HISTORY_TABLE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings a Postgres database to the current schema: inspect what it holds,
|
||||
* then create the schema or verify the history it has, then migrate.
|
||||
@@ -92,7 +141,7 @@ async function refuseNewerHistory(executed: string[], deps: PostgresBootstrapDep
|
||||
export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstrapDeps): Promise<void> {
|
||||
const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
|
||||
const connection = orm.em.getConnection();
|
||||
const historyTable = orm.config.get('migrations').tableName ?? 'mikro_orm_migrations';
|
||||
const historyTable = await checkServerAndPlaceHistory(orm, connection);
|
||||
|
||||
const tables = await currentTables(connection);
|
||||
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { REFUSAL } from '../../legacy-baseline';
|
||||
|
||||
/**
|
||||
* What TREK checks about the Postgres server and its own role: the server
|
||||
* versions it supports, and whether the role could reach the database host.
|
||||
* Read at boot (`schema-bootstrap.ts`) and before a dump or a restore
|
||||
* (`pg-tools.ts`).
|
||||
*/
|
||||
|
||||
/** The PostgreSQL majors TREK supports and CI runs: the oldest and the newest. */
|
||||
export const SUPPORTED_SERVER_MAJORS = { oldest: 14, newest: 18 } as const;
|
||||
|
||||
/**
|
||||
* True when the current role is a superuser or may run programs or write
|
||||
* files on the database server (the predefined roles
|
||||
* `pg_execute_server_program` and `pg_write_server_files`, which a superuser
|
||||
* holds implicitly). Such a role turns a restored archive, which is a SQL
|
||||
* script, into a way onto the database host.
|
||||
*/
|
||||
export const PRIVILEGED_ROLE_SQL =
|
||||
'coalesce((select r.rolsuper from pg_roles r where r.rolname = current_user), false)' +
|
||||
" or pg_has_role(current_user, 'pg_execute_server_program', 'MEMBER')" +
|
||||
" or pg_has_role(current_user, 'pg_write_server_files', 'MEMBER')";
|
||||
|
||||
/** The major version in `server_version_num` (`170011` is 17, `180001` is 18). Throws on anything else. */
|
||||
export function serverMajor(versionNum: string | number | undefined): number {
|
||||
const value = Number(versionNum);
|
||||
if (!Number.isInteger(value) || value < 10000) {
|
||||
throw new Error(`The Postgres server reported the version number "${String(versionNum)}", which TREK cannot read.`);
|
||||
}
|
||||
return Math.floor(value / 10000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a server outside {@link SUPPORTED_SERVER_MAJORS} before TREK writes
|
||||
* anything to it: an older one lacks what the schema uses, and a newer one is
|
||||
* untested and may be newer than the client tools that back it up.
|
||||
*/
|
||||
export function refuseUnsupportedServer(major: number): void {
|
||||
const { oldest, newest } = SUPPORTED_SERVER_MAJORS;
|
||||
if (major >= oldest && major <= newest) return;
|
||||
throw new Error(
|
||||
`${REFUSAL} the Postgres server is version ${major}; TREK supports PostgreSQL ${oldest} to ${newest}. ` +
|
||||
'Point TREK_DB_URL at a server of a supported version.',
|
||||
);
|
||||
}
|
||||
|
||||
/** The boot warning for a privileged role (see {@link PRIVILEGED_ROLE_SQL}); a restore refuses such a role outright. */
|
||||
export const PRIVILEGED_ROLE_WARNING =
|
||||
'TREK_DB_URL connects as a superuser, or as a role that may run programs or write files on the database server. ' +
|
||||
'Restores are refused for such a role. Connect TREK as a role that owns its database and is not a superuser ' +
|
||||
'(see the wiki page "Database: Postgres").';
|
||||
@@ -1,6 +1,11 @@
|
||||
import { pgRestore, pgTableNames } from '../../db/engine/postgres/pg-tools';
|
||||
import { stageExtractedPluginTrees } from '../plugins/plugin-backup';
|
||||
import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive';
|
||||
import {
|
||||
checkBackupDatabase,
|
||||
extractBackupArchive,
|
||||
markReclaimResetPending,
|
||||
MAX_BACKUP_DECOMPRESSED_SIZE,
|
||||
} from './backup-archive';
|
||||
import { checkPostgresDump, PG_ARCHIVE_ENTRY } from './postgres-database-backup';
|
||||
|
||||
import fs from 'fs';
|
||||
@@ -79,7 +84,8 @@ async function checkArchiveDatabase(plan: BootRestorePlan, staging: string): Pro
|
||||
/** Puts the archive's database in place: renamed in beside the file, or loaded into Postgres in one transaction. */
|
||||
async function putDatabaseInPlace(plan: BootRestorePlan, staging: string): Promise<void> {
|
||||
if (plan.postgres) {
|
||||
await pgRestore(plan.postgres.url, path.join(staging, PG_ARCHIVE_ENTRY), plan.postgres.binDir);
|
||||
const dump = path.join(staging, PG_ARCHIVE_ENTRY);
|
||||
await pgRestore(plan.postgres.url, dump, plan.postgres.binDir, MAX_BACKUP_DECOMPRESSED_SIZE);
|
||||
return;
|
||||
}
|
||||
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
|
||||
|
||||
@@ -9,6 +9,7 @@ import { DomainError } from '../common/domain-error';
|
||||
import type { DatabaseBackupStrategy } from '../database/database-backup.interface';
|
||||
import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../database/database-engine.module';
|
||||
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
|
||||
import { MAX_BACKUP_DECOMPRESSED_SIZE } from './backup-archive';
|
||||
import { keepRestoreCopy, replaceThenReopen } from './database-swap';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
|
||||
@@ -117,7 +118,7 @@ export class PostgresDatabaseBackup implements DatabaseBackupStrategy {
|
||||
async () => {
|
||||
const { url, binDir } = this.connection;
|
||||
// A rejection with no reason still has to fail the swap, so it is never swallowed.
|
||||
await pgRestore(url, source, binDir).catch((err: unknown) => {
|
||||
await pgRestore(url, source, binDir, MAX_BACKUP_DECOMPRESSED_SIZE).catch((err: unknown) => {
|
||||
throw err ?? new Error('pg_restore failed');
|
||||
});
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user