feat(db): run TREK on PostgreSQL

TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before
otherwise: an engine port under src/db/engine, the dialect helpers under
src/db/dialect, a Postgres baseline for the migrations up to the cut,
pg_dump/pg_restore backups, a probe of every repository method against a
real server and a Postgres test lane in CI.
This commit is contained in:
Isaias Tavares
2026-11-05 01:15:21 -03:00
committed by Maurice
parent 17b47b38b9
commit 2e3762afe1
176 changed files with 6728 additions and 777 deletions
@@ -0,0 +1,170 @@
/**
* The committed Postgres baseline (server/assets/postgres/) and what keeps it
* honest: reading and writing it (src/db/engine/postgres/baseline.ts) and the
* generator's --check (scripts/generate-pg-baseline.ts).
*/
import { checkBaseline, rowStatements, triggerPortProblems } from '../../../../../scripts/generate-pg-baseline';
import {
baselineMatches,
readPostgresBaseline,
renderBaseline,
writePostgresBaseline,
type PostgresBaseline,
} from '../../../../../src/db/engine/postgres/baseline';
import { POSTGRES_TRIGGERS } from '../../../../../src/db/engine/postgres/postgres-triggers';
import { knownMigrationNames } from '../../../../../src/db/known-migrations';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
const A = 'Migration20200101000000_a';
const B = 'Migration20200101000100_b';
const C = 'Migration20200101000200_c';
const sample: PostgresBaseline = {
through: B,
migrations: [A, B],
statements: ['create table "t" ("note" text default \'x;y\')', 'create index "i" on "t" ("note")'],
};
let dir: string | undefined;
afterEach(() => {
if (dir) rmSync(dir, { recursive: true, force: true });
dir = undefined;
});
const scratch = () => (dir = mkdtempSync(path.join(tmpdir(), 'trek-pg-baseline-')));
describe('Postgres baseline files', () => {
it('PGBASE-001: what is written reads back the same, comment line and all', () => {
const target = scratch();
writePostgresBaseline(sample, target);
expect(readPostgresBaseline(target)).toEqual(sample);
expect(baselineMatches(sample, target)).toBe(true);
expect(baselineMatches({ ...sample, statements: [sample.statements[0]] }, target)).toBe(false);
expect(renderBaseline(sample).sql.startsWith('-- ')).toBe(true);
});
it('PGBASE-002: a manifest that does not list its own cut, or an empty SQL file, is refused', () => {
const target = scratch();
writePostgresBaseline(sample, target);
writeFileSync(path.join(target, 'baseline.json'), JSON.stringify({ through: C, migrations: [A, B] }));
expect(() => readPostgresBaseline(target)).toThrow(/must name "through"/);
writePostgresBaseline(sample, target);
writeFileSync(path.join(target, 'baseline.sql'), '-- nothing\n');
expect(() => readPostgresBaseline(target)).toThrow(/holds no statements/);
writeFileSync(path.join(target, 'baseline.sql'), 'create table "q" ("a" text default \'?\');\n');
expect(() => readPostgresBaseline(target)).toThrow(/holds a "\?"/);
expect(baselineMatches(sample, path.join(target, 'missing'))).toBe(false);
});
it('PGBASE-003: the committed baseline names every migration on disk up to its cut, and only those', () => {
const committed = readPostgresBaseline();
const known = [...knownMigrationNames()].sort();
expect(committed.migrations).toEqual(known.filter((name) => name <= committed.through));
expect(committed.statements.some((statement) => /^create table "users"/.test(statement))).toBe(true);
expect(committed.statements.join('\n')).not.toMatch(/\binteger\b/);
});
});
describe('checkBaseline', () => {
const PORTED = Object.keys(POSTGRES_TRIGGERS);
const generated = (through: string, migrations: string[]): PostgresBaseline => ({
through,
migrations,
statements: [],
});
it('PGBASE-010: at the newest migration the baseline must be exactly what the entities generate', () => {
expect(checkBaseline(sample, generated(B, [A, B]), true, PORTED)).toEqual([]);
expect(checkBaseline(sample, generated(B, [A, B]), false, PORTED)).toEqual([
'the entities generate a different schema; run npm run gen:pg-baseline',
]);
});
it('PGBASE-011: once a migration lands after the cut, the baseline is frozen and only its manifest is checked', () => {
expect(checkBaseline(sample, generated(C, [A, B, C]), false, PORTED)).toEqual([]);
});
it('PGBASE-013: while not frozen, a SQLite trigger without a port is reported; once frozen the triggers are not checked', () => {
const withNewTrigger = [...PORTED, 'trg_new_trigger'];
expect(checkBaseline(sample, generated(B, [A, B]), true, withNewTrigger)).toEqual([
'SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: trg_new_trigger',
]);
expect(checkBaseline(sample, generated(C, [A, B, C]), false, withNewTrigger)).toEqual([]);
});
it('PGBASE-012: a manifest naming a migration that is gone, or missing one before its cut, is reported', () => {
expect(checkBaseline(sample, generated(C, [B, C]), true, PORTED)).toEqual([
`the baseline names migrations that do not exist: ${A}`,
]);
const skipped = { ...sample, migrations: [B] };
expect(checkBaseline(skipped, generated(B, [A, B]), true, PORTED)).toEqual([
`migrations up to ${B} are missing from it: ${A}`,
]);
});
});
describe('readPostgresBaseline manifest', () => {
it('PGBASE-004: a manifest whose migrations are not all names, or are empty, is refused', () => {
const target = scratch();
for (const manifest of [
{ through: B, migrations: [A, 3] },
{ through: B, migrations: [] },
{ through: 7, migrations: [B] },
]) {
writePostgresBaseline(sample, target);
writeFileSync(path.join(target, 'baseline.json'), JSON.stringify(manifest));
expect(() => readPostgresBaseline(target), JSON.stringify(manifest)).toThrow(/must name "through"/);
}
});
});
describe('triggerPortProblems', () => {
it('PGBASE-020: every SQLite trigger needs a Postgres port, and a port without its trigger is stale', () => {
expect(triggerPortProblems(['a', 'b'], ['b', 'a'])).toEqual([]);
expect(triggerPortProblems(['a', 'b'], ['a', 'c'])).toEqual([
'SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: b',
'POSTGRES_TRIGGERS ports triggers the migrations no longer create: c',
]);
});
it('PGBASE-021: the committed baseline carries every trigger port after the tables', () => {
const statements = readPostgresBaseline().statements;
const firstTrigger = statements.findIndex((statement) => statement.startsWith('CREATE FUNCTION'));
expect(firstTrigger).toBeGreaterThan(0);
expect(statements.slice(firstTrigger).every((statement) => /^CREATE (FUNCTION|TRIGGER) /.test(statement))).toBe(
true,
);
});
});
describe('rowStatements', () => {
it("PGBASE-030: a migration's rows go in as quoted literals the column casts, and a numeric id moves its sequence past them", () => {
expect(
rowStatements('photo_provider_fields', [
{ id: 1, provider_id: 'immich', label: "Server's URL", required: 1, hint: null },
{ id: 2, provider_id: 'immich', label: 'API key', required: 0, hint: 'x' },
]),
).toEqual([
'insert into "photo_provider_fields" ("id", "provider_id", "label", "required", "hint") values ' +
"('1', 'immich', 'Server''s URL', '1', NULL), ('2', 'immich', 'API key', '0', 'x')",
"select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('\"photo_provider_fields\"', 'id') as seq, " +
'max("id") as max_id from "photo_provider_fields") ids where seq is not null',
]);
});
it('PGBASE-031: a text key has no sequence to move, an empty table writes nothing, and binary data is refused', () => {
expect(rowStatements('addons', [{ id: 'mcp', enabled: 0 }])).toEqual([
'insert into "addons" ("id", "enabled") values (\'mcp\', \'0\')',
]);
expect(rowStatements('addons', [])).toEqual([]);
expect(() => rowStatements('blobs', [{ data: Buffer.from('x') }])).toThrow(/binary data/);
});
it('PGBASE-032: the committed baseline carries the reference rows the migrations insert', () => {
const sql = readPostgresBaseline().statements.join('\n');
expect(sql).toContain('insert into "tour_types"');
expect(sql).toContain('insert into "app_settings"');
});
});
@@ -0,0 +1,260 @@
/**
* The Postgres client tools (src/db/engine/postgres/pg-tools.ts, with
* pg-process.ts and restore-safety.ts), run against stand-in pg_dump,
* pg_restore and psql scripts (tests/helpers/fake-pg-tools.ts): the arguments
* and environment each call hands the binary, the password kept out of the
* arguments, the files a dump and a restore leave behind, and the refusal when
* a tool is missing or too old. The real tools against a real database are
* exercised by the Postgres lane.
*/
import { pgConnectionEnv, PgToolMissingError, resolvePgTool } from '../../../../../src/db/engine/postgres/pg-process';
import { pgDump, pgDumpMigrations, pgRestore } from '../../../../../src/db/engine/postgres/pg-tools';
import { hasRestrictFix } from '../../../../../src/db/engine/postgres/restore-safety';
import {
answersSchema,
copiesFileEndingWith,
failsWith,
flagValues,
installFakeTool,
listsMigrations,
printsWhen,
readToolCall,
touches,
writesFile,
writesRestrictedScript,
} from '../../../../helpers/fake-pg-tools';
import fs, { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
import os, { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const URL_WITH_SECRET = 'postgresql://trek:s3cr%40t@db.example:6543/trek_db?sslmode=require';
/** The restore's session settings, which psql gets through PGOPTIONS. */
const RESTORE_OPTIONS = '-c lock_timeout=60s -c client_min_messages=warning';
let bin: string;
let out: string;
beforeEach(() => {
bin = mkdtempSync(path.join(tmpdir(), 'trek-pg-bin-'));
out = mkdtempSync(path.join(tmpdir(), 'trek-pg-out-'));
});
afterEach(() => {
rmSync(bin, { recursive: true, force: true });
rmSync(out, { recursive: true, force: true });
});
describe('pg-tools', () => {
it('PGTOOLS-001: finds a tool by absolute path in the configured directory, and nowhere else', () => {
installFakeTool(bin, 'pg_dump');
writeFileSync(path.join(bin, 'psql'), 'not executable');
expect(resolvePgTool('pg_dump', bin)).toBe(path.join(bin, 'pg_dump'));
expect(resolvePgTool('pg_restore', bin)).toBeNull();
expect(resolvePgTool('psql', bin)).toBeNull();
});
it('PGTOOLS-002: the connection goes in the libpq environment, the password decoded and never an argument', () => {
expect(pgConnectionEnv(URL_WITH_SECRET)).toEqual({
PGHOST: 'db.example',
PGPORT: '6543',
PGDATABASE: 'trek_db',
PGUSER: 'trek',
PGPASSWORD: 's3cr@t',
PGCONNECT_TIMEOUT: '10',
PGSSLMODE: 'verify-full',
PGSSLROOTCERT: 'system',
});
expect(pgConnectionEnv('postgres://u@[::1]/d')).toEqual({
PGHOST: '::1',
PGPORT: '5432',
PGDATABASE: 'd',
PGUSER: 'u',
PGCONNECT_TIMEOUT: '10',
});
});
it('PGTOOLS-003: SSL is verified as the app pool verifies it, against the URL root certificate when it names one', () => {
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable')).toMatchObject({ PGSSLMODE: 'disable' });
expect(pgConnectionEnv('postgres://u@h/d?sslmode=disable').PGSSLROOTCERT).toBeUndefined();
expect(pgConnectionEnv('postgres://u@h/d?sslmode=verify-ca&sslrootcert=%2Fca%2Froot.crt')).toMatchObject({
PGSSLMODE: 'verify-full',
PGSSLROOTCERT: '/ca/root.crt',
});
});
it('PGTOOLS-004: pg_dump writes a private custom-format dump beside the target and renames it into place', async () => {
installFakeTool(bin, 'psql', { snippets: [answersSchema()] });
const log = installFakeTool(bin, 'pg_dump', { snippets: [writesFile('dump')] });
const target = path.join(out, 'x.pgdump');
await pgDump(URL_WITH_SECRET, target, bin);
const call = readToolCall(log);
expect(call.args).toHaveLength(5);
expect(call.args).toEqual(
expect.arrayContaining([
'--format=custom',
'--no-owner',
'--no-privileges',
`--file=${target}.partial`,
'--schema="public"',
]),
);
expect(call.env.PGPASSWORD).toBe('s3cr@t');
expect(call.args.join(' ')).not.toContain('s3cr');
expect(readFileSync(target, 'utf8')).toBe('dump');
expect(statSync(target).mode & 0o777).toBe(0o600);
expect(readdirSync(out)).toEqual(['x.pgdump']);
});
it('PGTOOLS-005: a restore clears the schema and loads the script in one psql transaction, then removes its work files', async () => {
const clearCopy = path.join(bin, 'clear.sql');
const restoreLog = installFakeTool(bin, 'pg_restore', { snippets: [writesRestrictedScript()] });
const psqlLog = installFakeTool(bin, 'psql', {
snippets: [answersSchema(), copiesFileEndingWith('clear-schema.sql', clearCopy)],
});
const source = path.join(out, 'in.pgdump');
await pgRestore(URL_WITH_SECRET, source, bin);
const restore = readToolCall(restoreLog);
expect(restore.args).toEqual(expect.arrayContaining(['--no-owner', '--no-privileges', '--schema=public']));
const [restoreScript] = flagValues(restore.args, '--file=');
expect(restoreScript).toMatch(/\.pg-restore-[^/]+\/restore\.sql$/);
expect(restore.args.slice(-2)).toEqual(['--', source]);
expect(restore.env).toEqual({});
const psql = readToolCall(psqlLog);
expect(psql.args).toEqual(
expect.arrayContaining([
'--no-psqlrc',
'--quiet',
'--single-transaction',
'--set=ON_ERROR_STOP=1',
`--output=${os.devNull}`,
]),
);
// Order is the point here: the schema is cleared before the script loads.
const [clearFile, loadFile] = flagValues(psql.args, '--file=');
expect(clearFile).toMatch(/clear-schema\.sql$/);
expect(loadFile).toBe(restoreScript);
expect(psql.env.PGOPTIONS).toBe(RESTORE_OPTIONS);
expect(readFileSync(clearCopy, 'utf8')).toMatch(/DROP %s IF EXISTS %s CASCADE/);
expect(readdirSync(out)).toEqual([]);
});
it('PGTOOLS-005b: the restore script is checked line by line, so its COPY data is never read whole', async () => {
const source = path.join(out, 'in.pgdump');
installFakeTool(bin, 'pg_restore', {
snippets: [writesRestrictedScript(), 'seq 1 200000 >> "$(fake_file_target "$@")"'],
});
installFakeTool(bin, 'psql', { snippets: [answersSchema()] });
const readWhole = vi.spyOn(fs, 'readFileSync');
try {
await pgRestore(URL_WITH_SECRET, source, bin);
expect(readWhole.mock.calls.filter(([file]) => String(file).endsWith('restore.sql'))).toEqual([]);
} finally {
readWhole.mockRestore();
}
});
it("PGTOOLS-006: reads the dump's recorded migrations from its COPY block, and none from a dump without them", async () => {
const log = installFakeTool(bin, 'pg_restore', {
snippets: [listsMigrations(['Migration20200101000000_baseline_schema', 'Migration20200101000100_second'])],
});
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([
'Migration20200101000000_baseline_schema',
'Migration20200101000100_second',
]);
const call = readToolCall(log);
expect(call.args).toEqual(expect.arrayContaining(['--data-only', '--table=mikro_orm_migrations', '--file=-']));
expect(call.args.slice(-2)).toEqual(['--', '/data/in.pgdump']);
installFakeTool(bin, 'pg_restore', { snippets: [printsWhen('--data-only', 'SET statement_timeout = 0;\n')] });
await expect(pgDumpMigrations('/data/in.pgdump', bin)).resolves.toEqual([]);
});
it.each([
{
id: 'PGTOOLS-007a',
stderr: 'pg_dump: error: query failed\nCommand was: SELECT secret_row\nDETAIL: Key (email)=(a@b.c)\n',
message: /^pg_dump failed: pg_dump: error: query failed$/,
},
{ id: 'PGTOOLS-007b', stderr: 'x'.repeat(5000), message: /^pg_dump failed: x{2000}…$/ },
])(
'$id: a failing tool reports its own message without the lines that quote SQL or rows, and leaves no file',
async ({ stderr, message }) => {
installFakeTool(bin, 'psql', { snippets: [answersSchema()] });
installFakeTool(bin, 'pg_dump', { snippets: [failsWith(stderr)] });
const target = path.join(out, 'x.pgdump');
await expect(pgDump(URL_WITH_SECRET, target, bin)).rejects.toThrow(message);
expect(readdirSync(out)).toEqual([]);
},
);
it('PGTOOLS-008: a missing tool says how to install it, and a restore that cannot run leaves no work files', async () => {
const source = path.join(out, 'in.pgdump');
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(PgToolMissingError);
await expect(pgDump(URL_WITH_SECRET, path.join(out, 'x.pgdump'), bin)).rejects.toThrow(/TREK_DB_PG_BIN_DIR/);
installFakeTool(bin, 'pg_restore', { snippets: [writesRestrictedScript()] });
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(/psql was not found/);
expect(existsSync(out) && readdirSync(out)).toEqual([]);
});
it.each([
{
id: 'PGTOOLS-009a',
tool: 'pg_restore' as const,
version: '16.9',
message:
/pg_restore is too old to restore safely \(pg_restore \(PostgreSQL\) 16\.9 \(Debian\)\): it must be at least 17\.6, 16\.10, 15\.14 or 14\.19/,
},
{ id: 'PGTOOLS-009b', tool: 'psql' as const, version: '17.5', message: /psql is too old/ },
])(
'$id: client tools without the \\restrict fix (CVE-2025-8714) never turn a dump into a script',
async ({ tool, version, message }) => {
const ran = path.join(bin, 'ran');
installFakeTool(bin, 'pg_restore', { snippets: [touches(ran)] });
installFakeTool(bin, 'psql', { snippets: [touches(ran)] });
installFakeTool(bin, tool, { version, snippets: [touches(ran)] });
await expect(pgRestore(URL_WITH_SECRET, path.join(out, 'in.pgdump'), bin)).rejects.toThrow(message);
expect(existsSync(ran)).toBe(false);
expect(readdirSync(out)).toEqual([]);
},
);
it('PGTOOLS-010: knows which client versions carry the \\restrict fix', () => {
const fixed = ['14.19', '15.14', '16.10', '17.6', '17.11', '18.0', '19.1'];
const unfixed = ['13.30', '14.18', '15.13', '16.9', '17.5'];
for (const v of fixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(true);
for (const v of unfixed) expect(hasRestrictFix(`pg_restore (PostgreSQL) ${v}`), v).toBe(false);
expect(hasRestrictFix('not a version')).toBe(false);
});
it('PGTOOLS-011: a dump holds the current schema only, matched by its exact name, which psql reads without the password in an argument', async () => {
const psqlLog = installFakeTool(bin, 'psql', { snippets: [answersSchema('My"Trek', '"My""Trek"')] });
const dumpLog = installFakeTool(bin, 'pg_dump', { snippets: [writesFile('dump')] });
await pgDump(URL_WITH_SECRET, path.join(out, 'x.pgdump'), bin);
expect(readToolCall(dumpLog).args).toContain('--schema="My""Trek"');
const psql = readToolCall(psqlLog);
expect(psql.args.join(' ')).toContain('current_schema()');
expect(psql.args.join(' ')).not.toContain('s3cr');
installFakeTool(bin, 'psql', { snippets: [printsWhen('quote_ident', '\n')] });
await expect(pgDump(URL_WITH_SECRET, path.join(out, 'y.pgdump'), bin)).rejects.toThrow(/no current schema/);
expect(readdirSync(out)).toEqual(['x.pgdump']);
});
it('PGTOOLS-012: a restore keeps to the current schema, and refuses a dump without TREK tables there before clearing anything', async () => {
const source = path.join(out, 'in.pgdump');
const restoreLog = installFakeTool(bin, 'pg_restore', {
snippets: [writesFile('\\restrict k1\nCREATE TABLE auth.users (\n);\n')],
});
const psqlLog = installFakeTool(bin, 'psql', { snippets: [answersSchema('trek')] });
await expect(pgRestore(URL_WITH_SECRET, source, bin)).rejects.toThrow(
'The backup holds no TREK tables in the schema "trek"; nothing was replaced.',
);
expect(readToolCall(restoreLog).args).toContain('--schema=trek');
expect(readToolCall(psqlLog).args.join(' ')).not.toContain('--single-transaction');
expect(readdirSync(out)).toEqual([]);
});
});
@@ -0,0 +1,47 @@
/**
* The Postgres driver reads `bigint` as a number, on its own pool only
* (src/db/engine/postgres/postgres-driver.ts). Nothing here connects.
*/
import { parseInt8, parseNumeric, TrekPostgreSqlDriver } from '../../../../../src/db/engine/postgres/postgres-driver';
import { Configuration } from '@mikro-orm/core';
import type { PostgreSqlConnection } from '@mikro-orm/postgresql';
import { describe, expect, it } from 'vitest';
type Parsers = { getTypeParser(oid: number): (text: string) => unknown };
describe('postgres driver', () => {
it('PGDRIVER-001: a bigint is a number up to 2^53 and refused past it, never rounded', () => {
expect(parseInt8('0')).toBe(0);
expect(parseInt8('1793636850006')).toBe(1793636850006);
expect(parseInt8('-42')).toBe(-42);
expect(() => parseInt8('9007199254740993')).toThrow(/beyond what TREK reads exactly/);
});
it('PGDRIVER-003: a numeric (sum of a bigint, avg) is a number like SQLite answers, an integral one past 2^53 refused', () => {
expect(parseNumeric('4200')).toBe(4200);
expect(parseNumeric('2.5000000000000000')).toBe(2.5);
expect(parseNumeric('-7')).toBe(-7);
expect(() => parseNumeric('9007199254740993')).toThrow(/beyond what TREK reads exactly/);
expect(() => parseNumeric('NaN')).toThrow(/not a finite number/);
expect(() => parseNumeric('Infinity')).toThrow(/not a finite number/);
});
it("PGDRIVER-002: the pool parses int8 as a number and keeps MikroORM's text dates", () => {
const config = new Configuration(
{
driver: TrekPostgreSqlDriver,
clientUrl: 'postgresql://u@localhost/trek',
entities: [],
discovery: { warnWhenNoEntities: false },
},
false,
);
const driver = new TrekPostgreSqlDriver(config);
const options = (driver.getConnection() as PostgreSqlConnection).mapOptions({});
const types = options.types as unknown as Parsers;
expect(types.getTypeParser(20)('12')).toBe(12);
expect(types.getTypeParser(1700)('12.5')).toBe(12.5);
expect(types.getTypeParser(1114)('2026-01-01 00:00:00')).toBe('2026-01-01 00:00:00');
});
});
@@ -0,0 +1,102 @@
/**
* How the entities' schema is written in Postgres
* (src/db/engine/postgres/schema-adjust.ts): SQLite-only defaults, TEXT timestamps, the
* NOCASE collation, and splitting the generated DDL so one refused statement
* does not take the rest with it.
*/
import { POSTGRES_TRIGGERS } from '../../../../../src/db/engine/postgres/postgres-triggers';
import {
applyPostgresColumnAdjustments,
type ColumnMeta,
NOCASE_COLLATION_SQL,
PG_NOW_TEXT_DEFAULT,
portableDefault,
} from '../../../../../src/db/engine/postgres/schema-adjust';
import { splitStatements } from '../../../../../src/db/engine/postgres/sql-split';
import { DbTimestampType } from '../../../../../src/db/types';
import { describe, expect, it } from 'vitest';
class TextType {}
describe('pg-probe schema', () => {
it('PGPROBE-040: maps the SQLite clock defaults and leaves every other default alone', () => {
expect(PG_NOW_TEXT_DEFAULT).toBe("to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), 'YYYY-MM-DD HH24:MI:SS')");
expect(portableDefault('CURRENT_TIMESTAMP')).toBe(PG_NOW_TEXT_DEFAULT);
expect(portableDefault("(datetime('now'))")).toBe(PG_NOW_TEXT_DEFAULT);
expect(portableDefault("datetime( 'now' )")).toBe(PG_NOW_TEXT_DEFAULT);
expect(portableDefault("(strftime('%s','now'))")).toBe('CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)');
for (const kept of ['0', '1', 'NULL', "'none'", "'#10b981'", "'{}'"]) expect(portableDefault(kept)).toBe(kept);
});
it('PGPROBE-041: makes timestamp columns text, rewrites defaults and the NOCASE collation, and names what it changed', () => {
const columns: ColumnMeta[] = [
{
name: 'created_at',
customType: new DbTimestampType(),
columnTypes: ['timestamptz'],
defaultRaw: 'CURRENT_TIMESTAMP',
},
{ name: 'title', customType: new TextType(), columnTypes: ['text'], defaultRaw: "'x'" },
{ name: 'stamp', columnTypes: ['int'], defaultRaw: "(strftime('%s','now'))" },
{ name: 'region', columnTypes: ['text'], collation: 'NOCASE' },
{ name: 'body', columnTypes: ['text'] },
];
expect(applyPostgresColumnAdjustments(columns)).toEqual(['created_at', 'stamp', 'region']);
expect(columns[0]).toMatchObject({ columnTypes: ['text'], defaultRaw: PG_NOW_TEXT_DEFAULT });
expect(columns[1]).toMatchObject({ columnTypes: ['text'], defaultRaw: "'x'" });
expect(columns[2]).toMatchObject({
columnTypes: ['bigint'],
defaultRaw: 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)',
});
expect(columns[3]).toMatchObject({ collation: 'nocase' });
expect(NOCASE_COLLATION_SQL).toContain('CREATE COLLATION IF NOT EXISTS nocase');
});
it('PGPROBE-043: an integer column is bigint, as wide as SQLite INTEGER (epoch milliseconds overflow integer)', () => {
const columns: ColumnMeta[] = [
{ name: 'id', columnTypes: ['integer'] },
{ name: 'next_run_at', columnTypes: ['int'] },
{ name: 'enabled', columnTypes: ['boolean'] },
{ name: 'note', columnTypes: ['text'] },
];
expect(applyPostgresColumnAdjustments(columns)).toEqual(['id', 'next_run_at']);
expect(columns.map((column) => column.columnTypes)).toEqual([['bigint'], ['bigint'], ['boolean'], ['text']]);
});
it('PGPROBE-042: splits DDL on semicolons outside quotes and drops session settings', () => {
const ddl = [
"set names 'utf8';",
'create table "a" ("id" serial primary key, "note" text default \'a;b\');',
'create index "a_x" on "a" ("note") where "note" <> \';\';',
'',
'create table "b ;" ("id" int)',
].join('\n');
expect(splitStatements(ddl)).toEqual([
'create table "a" ("id" serial primary key, "note" text default \'a;b\')',
'create index "a_x" on "a" ("note") where "note" <> \';\'',
'create table "b ;" ("id" int)',
]);
expect(splitStatements(' ; ; ')).toEqual([]);
});
it('PGPROBE-045: refuses a tagged dollar quote, which it does not understand, rather than splitting inside it', () => {
expect(() => splitStatements('create function f() as $f$ select 1; $f$;')).toThrow(
/only the \$\$ dollar quote, not \$f\$/,
);
expect(splitStatements("select '$f$'; select 2")).toEqual(["select '$f$'", 'select 2']);
});
it('PGPROBE-044: a $$ body stays one statement, semicolons and all, and every trigger port splits into its two', () => {
const fn =
"CREATE FUNCTION f() RETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n UPDATE t SET x = ';';\n RETURN NEW;\nEND\n$$";
expect(splitStatements(`${fn};\nCREATE TRIGGER g AFTER UPDATE ON t FOR EACH ROW EXECUTE FUNCTION f();`)).toEqual([
fn,
'CREATE TRIGGER g AFTER UPDATE ON t FOR EACH ROW EXECUTE FUNCTION f()',
]);
for (const [name, statements] of Object.entries(POSTGRES_TRIGGERS)) {
expect(splitStatements(statements.join(';\n')), name).toEqual([...statements]);
expect(statements.at(-1), name).toContain(`CREATE TRIGGER ${name} `);
}
});
});
@@ -0,0 +1,267 @@
/**
* How a Postgres database reaches the schema (src/db/engine/postgres/schema-bootstrap.ts):
* the three states the plan names (empty, foreign tables, TREK's own) and the
* safety net before pending migrations, over a fake migrator and connection.
*/
import type { AppEnv } from '../../../../../src/app-config';
import {
bootstrapPostgresSchema,
postgresSafetyNet,
type PostgresBootstrapDeps,
} from '../../../../../src/db/engine/postgres/schema-bootstrap';
import type { MigrationSafetyNet } from '../../../../../src/db/legacy-baseline';
import type { AnyOrm } from '../../../../../src/db/orm';
import { answersSchema, failsWith, installFakeTool, writesFile } from '../../../../helpers/fake-pg-tools';
import { Logger } from '@nestjs/common';
import { mkdtempSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const M1 = 'Migration20200101000000_baseline_schema';
const M2 = 'Migration20200101000100_second';
const M3 = 'Migration20200101000200_after_the_cut';
interface FakeDb {
executed: string[];
pending: string[];
tables: string[];
/** Whether mikro_orm_migrations exists: from the start when anything is recorded. */
history: boolean;
ensureTable: ReturnType<typeof vi.fn>;
sql: string[];
logged: string[];
inTransaction: boolean[];
up: ReturnType<typeof vi.fn>;
}
function fakeOrm(state: Pick<FakeDb, 'executed' | 'pending' | 'tables'> & { history?: boolean }): {
orm: AnyOrm;
db: FakeDb;
} {
const db: FakeDb = {
...state,
history: state.history ?? state.executed.length > 0,
ensureTable: vi.fn(async () => {
db.history = true;
}),
sql: [],
logged: [],
inTransaction: [],
up: vi.fn(async () => undefined),
};
const trx = { trx: true };
const connection = {
execute: vi.fn(async (sql: string, _params?: unknown[], _method?: string, ctx?: unknown) => {
if (sql.includes('information_schema.tables')) {
return [...db.tables, ...(db.history ? ['mikro_orm_migrations'] : [])].map((table_name) => ({ table_name }));
}
db.sql.push(sql);
db.inTransaction.push(ctx === trx);
return [];
}),
transactional: vi.fn(async (work: (ctx: unknown) => Promise<void>) => work(trx)),
};
const storage = {
ensureTable: db.ensureTable,
logMigration: vi.fn(async ({ name }: { name: string }, ctx?: unknown) => {
db.logged.push(name);
db.inTransaction.push(ctx === trx);
db.executed = [...db.executed, name];
db.pending = db.pending.filter((pending) => pending !== name);
}),
};
const migrator = {
getStorage: () => storage,
getExecuted: vi.fn(async () => db.executed.map((name) => ({ name }))),
getPending: vi.fn(async () => db.pending.map((name) => ({ name }))),
up: db.up,
};
const orm = {
config: { getExtension: () => migrator, get: () => ({}) },
em: { getConnection: () => connection },
} as unknown as AnyOrm;
return { orm, db };
}
function deps(
safetyNet: MigrationSafetyNet = { beforeMigrate: vi.fn(async () => undefined), restoreHint: async () => null },
) {
const value: PostgresBootstrapDeps = {
baseline: () => ({
through: M2,
migrations: [M1, M2],
statements: ['create table "a" ("id" bigserial)', 'create index "x" on "a" ("id")'],
}),
known: new Set([M1, M2, M3]),
safetyNet,
};
return value;
}
afterEach(() => vi.restoreAllMocks());
describe('bootstrapPostgresSchema', () => {
beforeEach(() => {
vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
});
it('PGBOOT-001: an empty database gets the baseline and its migrations recorded, all in one transaction', async () => {
const { orm, db } = fakeOrm({ executed: [], pending: [M1, M2], tables: [] });
const net = deps();
await bootstrapPostgresSchema(orm, net);
expect(db.sql).toEqual(['create table "a" ("id" bigserial);\ncreate index "x" on "a" ("id")']);
expect(db.logged).toEqual([M1, M2]);
expect(db.inTransaction.every(Boolean)).toBe(true);
expect(db.up).not.toHaveBeenCalled();
expect(net.safetyNet.beforeMigrate).not.toHaveBeenCalled();
});
it('PGBOOT-002: a fresh database runs the migrations after the cut without the safety net (nothing to lose)', async () => {
const { orm, db } = fakeOrm({ executed: [], pending: [M1, M2, M3], tables: [] });
const net = deps();
await bootstrapPostgresSchema(orm, net);
expect(db.logged).toEqual([M1, M2]);
expect(db.up).toHaveBeenCalledTimes(1);
expect(net.safetyNet.beforeMigrate).not.toHaveBeenCalled();
});
it('PGBOOT-003: tables without TREK history are refused, named, and nothing is written', async () => {
const { orm, db } = fakeOrm({ executed: [], pending: [M1], tables: ['customers', 'orders'] });
await expect(bootstrapPostgresSchema(orm, deps())).rejects.toThrow(
/Refusing to boot: the database holds 2 table\(s\) \(customers, orders\) but no TREK migration history/,
);
expect(db.sql).toEqual([]);
expect(db.logged).toEqual([]);
expect(db.ensureTable).not.toHaveBeenCalled();
});
it('PGBOOT-008: an empty history table beside no other table is a fresh database (a first boot cut short)', async () => {
const { orm, db } = fakeOrm({ executed: [], pending: [M1, M2], tables: [], history: true });
await bootstrapPostgresSchema(orm, deps());
expect(db.logged).toEqual([M1, M2]);
});
it('PGBOOT-004: a database a newer TREK migrated is refused before anything runs', async () => {
const { orm, db } = fakeOrm({ executed: [M1, 'Migration20990101000000_from_the_future'], pending: [], tables: [] });
await expect(bootstrapPostgresSchema(orm, deps())).rejects.toThrow(/migrated by a newer TREK/);
expect(db.up).not.toHaveBeenCalled();
});
it('PGBOOT-005: pending migrations on a database in use run after the safety net, labelled by the last one', async () => {
const { orm, db } = fakeOrm({ executed: [M2, M1], pending: [M3], tables: ['users'] });
const net = deps();
await bootstrapPostgresSchema(orm, net);
expect(net.safetyNet.beforeMigrate).toHaveBeenCalledWith('20200101000100');
expect(db.up).toHaveBeenCalledTimes(1);
expect(db.sql).toEqual([]);
});
it('PGBOOT-006: a refusing safety net stops the migration', async () => {
const { orm, db } = fakeOrm({ executed: [M1, M2], pending: [M3], tables: ['users'] });
const refusing = {
beforeMigrate: vi.fn(async () => Promise.reject(new Error('no copy'))),
restoreHint: async () => null,
};
await expect(bootstrapPostgresSchema(orm, deps(refusing))).rejects.toThrow('no copy');
expect(db.up).not.toHaveBeenCalled();
});
it('PGBOOT-007: a database at head does nothing', async () => {
const { orm, db } = fakeOrm({ executed: [M1, M2, M3], pending: [], tables: ['users'] });
const net = deps();
await bootstrapPostgresSchema(orm, net);
expect(db.sql).toEqual([]);
expect(db.up).not.toHaveBeenCalled();
expect(net.safetyNet.beforeMigrate).not.toHaveBeenCalled();
});
});
describe('postgresSafetyNet', () => {
const URL = 'postgresql://trek:pw@db/trek';
const NOW = new Date('2026-11-02T13:05:09Z');
let dir: string;
let bin: string;
beforeEach(() => {
dir = mkdtempSync(path.join(tmpdir(), 'trek-pg-net-'));
bin = mkdtempSync(path.join(tmpdir(), 'trek-pg-net-bin-'));
vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
rmSync(bin, { recursive: true, force: true });
});
/**
* A stand-in pg_dump that writes its --file, or fails like one that cannot
* connect, beside the psql that names the schema the dump is limited to.
*/
const fakePgDump = (ok: boolean) => {
installFakeTool(bin, 'pg_dump', { snippets: [ok ? writesFile('dump') : failsWith('could not connect')] });
installFakeTool(bin, 'psql', { snippets: [answersSchema()] });
};
const envWith =
(preMigrateSnapshot: boolean | undefined, isTest: boolean, keep = 3) =>
() =>
({
db: { preMigrateSnapshot, preMigrateSnapshotKeep: keep, pgBinDir: bin },
app: { isTest },
}) as unknown as AppEnv;
const net = (env: () => AppEnv) =>
postgresSafetyNet(
URL,
env,
() => NOW,
() => dir,
);
it.each([undefined, true])(
'PGBOOT-010: dumps the database before migrating it (setting %s), keeping the newest copies and no half-written one',
async (setting) => {
fakePgDump(true);
for (const old of ['20200101000001', '20200101000002']) {
writeFileSync(path.join(dir, `pre-migrate-${old}-20250101T000000Z.pgdump`), 'old');
utimesSync(path.join(dir, `pre-migrate-${old}-20250101T000000Z.pgdump`), 1, 1);
}
writeFileSync(path.join(dir, 'pre-migrate-20200101000003-20250101T000000Z.pgdump.partial'), 'killed');
await net(envWith(setting, false, 2)).beforeMigrate('20200101000100');
const kept = readdirSync(dir).sort();
expect(kept).toHaveLength(2);
expect(kept).toContain('pre-migrate-20200101000100-20261102T130509Z.pgdump');
expect(kept.some((file) => file.endsWith('.partial'))).toBe(false);
},
);
it('PGBOOT-011: migrates without a copy when the operator turned it off, and says so', async () => {
const warn = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
await expect(net(envWith(false, false)).beforeMigrate('1')).resolves.toBeUndefined();
expect(warn).toHaveBeenCalledWith(expect.stringContaining('without a copy'));
expect(readdirSync(dir)).toEqual([]);
});
it('PGBOOT-012: stays out of the way under test unless a suite asks for it', async () => {
await expect(net(envWith(undefined, true)).beforeMigrate('1')).resolves.toBeUndefined();
expect(readdirSync(dir)).toEqual([]);
});
it('PGBOOT-013: a copy that cannot be written refuses the boot, names the switch, and leaves no partial file', async () => {
fakePgDump(false);
await expect(net(envWith(true, false)).beforeMigrate('1')).rejects.toThrow(
/Refusing to boot: could not write a copy.*pg_dump failed: could not connect.*TREK_DB_PRE_MIGRATE_SNAPSHOT=false/s,
);
expect(readdirSync(dir)).toEqual([]);
});
it('PGBOOT-014: the newer-database refusal points at the copy taken from this release, or at the others', async () => {
const hint = net(envWith(true, false));
await expect(hint.restoreHint('20200101000100')).resolves.toBeNull();
writeFileSync(path.join(dir, 'pre-migrate-20200101000050-20250101T000000Z.pgdump'), 'old');
await expect(hint.restoreHint('20200101000100')).resolves.toMatch(/none of them from this release/);
writeFileSync(path.join(dir, 'pre-migrate-20200101000100-20250102T000000Z.pgdump'), 'mine');
await expect(hint.restoreHint('20200101000100')).resolves.toMatch(
/pre-migrate-20200101000100-20250102T000000Z\.pgdump: stop TREK, empty the database .*pg_restore --no-owner/,
);
});
});