mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fix(db): check the Postgres server and tools before a dump or a restore
- The client tools are looked for newest first, from 18 down: a client handles every server up to its own major and refuses a newer one. A dump or a restore now refuses a client older than the server up front, naming both versions, and the boot refuses a server outside PostgreSQL 14 to 18. - sslrootcert=system is used only by a client from libpq 16 on; an older client is refused with a message that names sslrootcert and the bin dir. - The restore's CVE-2025-8714 table covers 18. - A restore refuses a role that is a superuser or may run programs or write files on the database host, since the archive's SQL runs as that role; the boot warns about such a role. - pg_restore's script is streamed to its work file and stopped once it passes BACKUP_MAX_DECOMPRESSED_MB, so a small dump can no longer expand without limit on the data volume. - A failed tool's error keeps only the summarized stderr; its cause names the exit code and signal instead of carrying the process error, whose message and stderr quoted row data into the log. - The migration history is looked for in the current schema, where the tables and the backups are, so a role whose search_path is not public boots a second time.
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { execFile, spawn } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Transform } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -10,10 +12,13 @@ export type PgTool = 'pg_dump' | 'pg_restore' | 'psql';
|
||||
|
||||
/**
|
||||
* Where the client tools are looked for when `TREK_DB_PG_BIN_DIR` is unset:
|
||||
* Debian's versioned directories newest first (the Docker image installs 17),
|
||||
* then the usual prefixes. Never PATH: a binary is run by its absolute path.
|
||||
* Debian's versioned directories newest first (the Docker image installs 18),
|
||||
* then the usual prefixes. Newest first because a client dumps and restores
|
||||
* every server up to its own major version and refuses a newer one. Never
|
||||
* PATH: a binary is run by its absolute path.
|
||||
*/
|
||||
const DEFAULT_BIN_DIRS = [
|
||||
'/usr/lib/postgresql/18/bin',
|
||||
'/usr/lib/postgresql/17/bin',
|
||||
'/usr/lib/postgresql/16/bin',
|
||||
'/usr/lib/postgresql/15/bin',
|
||||
@@ -29,8 +34,12 @@ const TOOL_MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
||||
const CONNECT_TIMEOUT_SECONDS = '10';
|
||||
/** How much of a tool's stderr an error keeps. */
|
||||
const STDERR_LIMIT = 2000;
|
||||
/** How much of a streaming tool's stderr is held while it runs, before it is summarized. */
|
||||
const STDERR_CAPTURE_LIMIT = 64 * 1024;
|
||||
/** stderr lines that quote SQL or row data: they stay out of errors and logs. */
|
||||
const DATA_LINE = /^\s*(Command was:|CONTEXT:|DETAIL:|LINE \d+:|QUERY:)/;
|
||||
/** libpq reads `sslrootcert=system` as the system's trusted CAs from this major on; older clients read it as a file name. */
|
||||
const SYSTEM_ROOT_CERT_SINCE = 16;
|
||||
|
||||
/** The absolute path of `tool`, or null when it is not installed where TREK looks. */
|
||||
export function resolvePgTool(tool: PgTool, binDir: string | undefined): string | null {
|
||||
@@ -51,7 +60,8 @@ export function resolvePgTool(tool: PgTool, binDir: string | undefined): string
|
||||
* PGPASSWORD, so it never appears in an argument list another process can
|
||||
* read. An sslmode that asks for SSL verifies the server's certificate, as the
|
||||
* app's own pool does (`postgres-engine.ts`): against the URL's sslrootcert,
|
||||
* else the system's trusted CAs.
|
||||
* else the system's trusted CAs (`system`, which `run` accepts only from a
|
||||
* client that understands it).
|
||||
*/
|
||||
export function pgConnectionEnv(url: string): Record<string, string> {
|
||||
const parsed = new URL(url);
|
||||
@@ -83,6 +93,17 @@ export class PgToolMissingError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** A tool that wrote more than the caller allowed; it was stopped and its output is incomplete. */
|
||||
export class PgOutputLimitError extends Error {
|
||||
constructor(
|
||||
tool: PgTool,
|
||||
readonly limitBytes: number,
|
||||
) {
|
||||
super(`${tool} wrote more than ${limitBytes} bytes and was stopped`);
|
||||
this.name = 'PgOutputLimitError';
|
||||
}
|
||||
}
|
||||
|
||||
/** The tool's own message, without the lines that quote SQL or rows, and bounded. */
|
||||
function summarizeStderr(stderr: string | undefined): string {
|
||||
const kept = (stderr ?? '')
|
||||
@@ -92,10 +113,104 @@ function summarizeStderr(stderr: string | undefined): string {
|
||||
return kept.length > STDERR_LIMIT ? `${kept.slice(0, STDERR_LIMIT)}…` : kept;
|
||||
}
|
||||
|
||||
/**
|
||||
* The error a failed tool run throws: `<tool> failed: <its stderr, summarized>`.
|
||||
* Its cause names only how the process ended (exit code, signal, whether the
|
||||
* timeout killed it), never the process error itself, whose message and
|
||||
* `stderr` hold the tool's whole stderr, row data included, and would reach
|
||||
* the log through the cause chain.
|
||||
*/
|
||||
function toolFailure(tool: PgTool, failure: unknown): Error {
|
||||
const { stderr, code, signal, killed } = (failure ?? {}) as {
|
||||
stderr?: unknown;
|
||||
code?: unknown;
|
||||
signal?: unknown;
|
||||
killed?: unknown;
|
||||
};
|
||||
const message = summarizeStderr(typeof stderr === 'string' ? stderr : undefined);
|
||||
const ending = signal ? `signal ${String(signal)}` : `code ${String(code)}`;
|
||||
const cause = Object.assign(new Error(`${tool} ended with ${ending}${killed === true ? ' (timed out)' : ''}`), {
|
||||
code,
|
||||
signal,
|
||||
killed,
|
||||
});
|
||||
return new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause });
|
||||
}
|
||||
|
||||
/** The major version in a tool's `--version` line (`pg_dump (PostgreSQL) 18.1 (Debian …)`), or null. */
|
||||
function toolMajor(version: string): number | null {
|
||||
const match = /\(PostgreSQL\) (\d+)/.exec(version);
|
||||
return match ? Number(match[1]) : null;
|
||||
}
|
||||
|
||||
/** `binary --version`, read directly: the check before a run must not go through `run` itself. */
|
||||
async function binaryVersion(tool: PgTool, binary: string): Promise<string> {
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, ['--version'], { env: {}, timeout: TOOL_TIMEOUT_MS });
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
throw toolFailure(tool, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** The absolute path of `tool`, or the refusal that says how to install it. */
|
||||
function locate(tool: PgTool, binDir: string | undefined): string {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
return binary;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses to run a client older than libpq 16 with `sslrootcert=system`: it
|
||||
* would read `system` as a file name and fail on every connection. Such a
|
||||
* client needs the CA file named in the URL.
|
||||
*/
|
||||
async function refuseSystemRootCertOnOldClient(
|
||||
tool: PgTool,
|
||||
binary: string,
|
||||
env: Record<string, string>,
|
||||
): Promise<void> {
|
||||
if (env.PGSSLROOTCERT !== 'system') return;
|
||||
const version = await binaryVersion(tool, binary);
|
||||
const major = toolMajor(version);
|
||||
if (major !== null && major >= SYSTEM_ROOT_CERT_SINCE) return;
|
||||
throw new Error(
|
||||
`${tool} (${version.trim()}) cannot verify the server against the system's trusted certificate authorities, ` +
|
||||
`which needs a client of version ${SYSTEM_ROOT_CERT_SINCE} or later. Add sslrootcert=<CA file> to TREK_DB_URL, ` +
|
||||
'or point TREK_DB_PG_BIN_DIR at newer client tools (the Docker image has them).',
|
||||
);
|
||||
}
|
||||
|
||||
/** The major version of `tool` as TREK would run it. */
|
||||
async function pgToolMajor(tool: PgTool, binDir: string | undefined): Promise<number | null> {
|
||||
return toolMajor(await binaryVersion(tool, locate(tool, binDir)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses `tools` when one is older than the server it is about to talk to:
|
||||
* pg_dump refuses a newer server outright, and what an older pg_restore or
|
||||
* psql makes of a newer server's dump is not something a restore should find
|
||||
* out halfway. A newer client is fine; it handles every older server.
|
||||
*/
|
||||
export async function refuseClientOlderThanServer(
|
||||
tools: readonly PgTool[],
|
||||
binDir: string | undefined,
|
||||
serverMajor: number,
|
||||
): Promise<void> {
|
||||
for (const tool of tools) {
|
||||
const major = await pgToolMajor(tool, binDir);
|
||||
if (major !== null && major >= serverMajor) continue;
|
||||
throw new Error(
|
||||
`${tool} is version ${major ?? 'unknown'}, older than the PostgreSQL ${serverMajor} server, which it cannot ` +
|
||||
`back up or restore. Point TREK_DB_PG_BIN_DIR at client tools of version ${serverMajor} or later ` +
|
||||
'(the Docker image has the newest supported ones).',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `tool` by its absolute path and resolves with its stdout. A failure
|
||||
* throws `<tool> failed: <its stderr, summarized>`, with the original error as
|
||||
* the cause.
|
||||
* throws `<tool> failed: <its stderr, summarized>`.
|
||||
*/
|
||||
export async function run(
|
||||
tool: PgTool,
|
||||
@@ -103,8 +218,8 @@ export async function run(
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
): Promise<string> {
|
||||
const binary = resolvePgTool(tool, binDir);
|
||||
if (!binary) throw new PgToolMissingError(tool);
|
||||
const binary = locate(tool, binDir);
|
||||
await refuseSystemRootCertOnOldClient(tool, binary, env);
|
||||
try {
|
||||
const { stdout } = await execFileAsync(binary, args, {
|
||||
env,
|
||||
@@ -113,7 +228,55 @@ export async function run(
|
||||
});
|
||||
return stdout;
|
||||
} catch (err) {
|
||||
const message = summarizeStderr((err as { stderr?: string }).stderr);
|
||||
throw new Error(`${tool} failed${message ? `: ${message}` : ''}`, { cause: err });
|
||||
throw toolFailure(tool, err);
|
||||
}
|
||||
}
|
||||
|
||||
/** How a spawned tool ended: its exit, or the error that kept it from starting. */
|
||||
type Ending = { code: number | null; signal: NodeJS.Signals | null } | { error: Error };
|
||||
|
||||
/**
|
||||
* Runs `tool` and streams its stdout into `target` (which must exist; its mode
|
||||
* is kept), stopping the tool with {@link PgOutputLimitError} once more than
|
||||
* `maxBytes` have come out. The caller removes `target` on failure. A failure
|
||||
* of the tool itself throws as `run` does.
|
||||
*/
|
||||
export async function runToFile(
|
||||
tool: PgTool,
|
||||
binDir: string | undefined,
|
||||
args: string[],
|
||||
env: Record<string, string>,
|
||||
target: string,
|
||||
maxBytes: number,
|
||||
): Promise<void> {
|
||||
const binary = locate(tool, binDir);
|
||||
await refuseSystemRootCertOnOldClient(tool, binary, env);
|
||||
const child = spawn(binary, args, { env, stdio: ['ignore', 'pipe', 'pipe'], timeout: TOOL_TIMEOUT_MS });
|
||||
const ended = new Promise<Ending>((resolve) => {
|
||||
child.once('error', (error) => resolve({ error }));
|
||||
child.once('close', (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
let stderr = '';
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
if (stderr.length < STDERR_CAPTURE_LIMIT) stderr += chunk;
|
||||
});
|
||||
let written = 0;
|
||||
const limit = new Transform({
|
||||
transform(chunk: Buffer, _encoding, callback) {
|
||||
written += chunk.length;
|
||||
if (written > maxBytes) callback(new PgOutputLimitError(tool, maxBytes));
|
||||
else callback(null, chunk);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await pipeline(child.stdout, limit, fs.createWriteStream(target, { flags: 'w' }));
|
||||
} catch (err) {
|
||||
child.kill('SIGKILL');
|
||||
await ended;
|
||||
throw err instanceof PgOutputLimitError ? err : toolFailure(tool, { stderr, code: 'stream' });
|
||||
}
|
||||
const ending = await ended;
|
||||
if ('error' in ending) throw toolFailure(tool, { stderr, code: (ending.error as { code?: unknown }).code });
|
||||
if (ending.code !== 0) throw toolFailure(tool, { stderr, code: ending.code, signal: ending.signal });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user