From 5503db48833c15edc549d1f1ee84c0f70a9ffa2d Mon Sep 17 00:00:00 2001 From: Isaias Tavares Date: Thu, 5 Nov 2026 00:36:42 -0300 Subject: [PATCH] refactor(db): generate the Postgres baseline in one pass, check it in two modes --check migrated a temporary SQLite database twice: once inside the generation and again to read the triggers for the port check. The generation now returns the triggers it already read, so a check is one migration. main is split into writeMode and checkMode, and the frozen-or-not decision lives in one place, checkBaseline. A baseline whose `through` is the newest migration must match the entities and port every SQLite trigger; a baseline frozen behind a newer migration is only checked by its manifest. The old code made the same decision twice, once for the match and once for the triggers, and the two could drift apart. The migrated-database read is split into listing the objects and reading them, so the temporary file is removed in one place. Manifest parsing uses a type guard, isBaselineManifest, instead of casts after a check, and PGBASE-004 pins what it refuses. --- server/scripts/generate-pg-baseline.ts | 128 +++++++++++------- server/src/db/engine/postgres/baseline.ts | 27 ++-- .../unit/db/engine/postgres/baseline.test.ts | 35 ++++- 3 files changed, 128 insertions(+), 62 deletions(-) diff --git a/server/scripts/generate-pg-baseline.ts b/server/scripts/generate-pg-baseline.ts index 558199023..3c578c88d 100644 --- a/server/scripts/generate-pg-baseline.ts +++ b/server/scripts/generate-pg-baseline.ts @@ -36,13 +36,18 @@ import { MikroORM } from '@mikro-orm/postgresql'; import Database from 'better-sqlite3'; import fs from 'node:fs'; -/** The baseline the entities and the migrations on disk describe now. */ -export async function generatePostgresBaseline(known: Set = knownMigrationNames()): Promise { +/** What one pass over the migrated SQLite database yields: the baseline, and the triggers the migrations leave there. */ +interface BaselinePass { + baseline: PostgresBaseline; + sqliteTriggers: string[]; +} + +/** The baseline the entities and the migrations on disk describe now, from one migration of the SQLite database. */ +async function generateBaseline(known: Set = knownMigrationNames()): Promise { const migrations = [...known].sort(); const through = migrations.at(-1); if (!through) throw new Error('no migrations found'); const orm = await MikroORM.init({ - // Never contacted: init() only discovers metadata, and the schema generator needs nothing else. clientUrl: 'postgresql://generator@localhost/trek', entities: [...ALL_ENTITIES], discovery: { warnWhenNoEntities: false }, @@ -56,19 +61,26 @@ export async function generatePostgresBaseline(known: Set = knownMigrati ...migrated.tables.flatMap((table) => rowStatements(table.name, table.rows)), ...Object.values(POSTGRES_TRIGGERS).flat(), ]; - return { through, migrations, statements }; + return { baseline: { through, migrations, statements }, sqliteTriggers: migrated.triggers }; } finally { await orm.close(); } } /** - * What `--check` holds. While no migration is newer than the committed - * baseline, the baseline must be exactly what the entities generate. Once one - * is, the baseline is frozen and only its manifest is checked: every migration - * it names exists, and it names every migration up to `through`. + * What `--check` holds, as the problems it finds. Every manifest must name + * migrations that exist and every migration up to `through`. While no migration + * is newer than the committed baseline (`through` matches), the baseline is + * not frozen: it must be exactly what the entities generate, and each SQLite + * trigger needs a port. Once a newer migration lands, the baseline is frozen + * and only the manifest is checked. */ -export function checkBaseline(committed: PostgresBaseline, generated: PostgresBaseline, matches: boolean): string[] { +export function checkBaseline( + committed: PostgresBaseline, + generated: PostgresBaseline, + matches: boolean, + sqliteTriggers: readonly string[], +): string[] { const known = new Set(generated.migrations); const problems: string[] = []; const missing = committed.migrations.filter((name) => !known.has(name)); @@ -78,9 +90,10 @@ export function checkBaseline(committed: PostgresBaseline, generated: PostgresBa ); if (unlisted.length > 0) problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`); - if (committed.through === generated.through && !matches) { - problems.push('the entities generate a different schema; run npm run gen:pg-baseline'); - } + const frozen = committed.through !== generated.through; + if (frozen) return problems; + if (!matches) problems.push('the entities generate a different schema; run npm run gen:pg-baseline'); + problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS))); return problems; } @@ -90,32 +103,48 @@ const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schem type Row = Record; /** What the migrations leave in a SQLite database: its triggers, and the rows they insert, table by table in creation order. */ -async function readMigratedSqlite(): Promise<{ triggers: string[]; tables: Array<{ name: string; rows: Row[] }> }> { +interface MigratedSchema { + triggers: string[]; + tables: Array<{ name: string; rows: Row[] }>; +} + +/** The tables and triggers of a SQLite database in creation order, each as its type and name. */ +function listSchemaObjects(conn: Database.Database): Array<{ type: string; name: string }> { + return conn + .prepare( + "select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid", + ) + .all() as Array<{ type: string; name: string }>; +} + +/** Reads a migrated SQLite database, read-only. */ +function readMigratedSchema(dbPath: string): MigratedSchema { + const conn = new Database(dbPath, { readonly: true }); + try { + const objects = listSchemaObjects(conn); + const tables = objects + .filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name)) + .map((object) => ({ + name: object.name, + rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[], + })) + .filter((table) => table.rows.length > 0); + const triggers = objects + .filter((object) => object.type === 'trigger') + .map((object) => object.name) + .sort(); + return { triggers, tables }; + } finally { + conn.close(); + } +} + +/** Migrates a temporary SQLite database and reads what the migrations leave in it; the file is removed afterwards. */ +async function readMigratedSqlite(): Promise { const dbPath = tempDbPath(); try { await migrateTempDb(dbPath); - const conn = new Database(dbPath, { readonly: true }); - try { - const objects = conn - .prepare( - "select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid", - ) - .all() as Array<{ type: string; name: string }>; - const tables = objects - .filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name)) - .map((object) => ({ - name: object.name, - rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[], - })) - .filter((table) => table.rows.length > 0); - const triggers = objects - .filter((object) => object.type === 'trigger') - .map((object) => object.name) - .sort(); - return { triggers, tables }; - } finally { - conn.close(); - } + return readMigratedSchema(dbPath); } finally { fs.rmSync(dbPath, { force: true }); } @@ -143,7 +172,6 @@ export function rowStatements(table: string, rows: readonly Row[]): string[] { const statements = [ `insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`, ]; - // A text key (addons.id) has no sequence; a numeric one may. if (rows.every((row) => typeof row.id === 'number')) { statements.push( `select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('"${table}"', 'id') as seq, ` + @@ -169,19 +197,18 @@ export function triggerPortProblems(sqliteTriggers: readonly string[], ported: r return problems; } -async function main(): Promise { - const generated = await generatePostgresBaseline(); - if (!process.argv.slice(2).includes('--check')) { - writePostgresBaseline(generated); - console.log(`[generate-pg-baseline] wrote ${generated.statements.length} statements through ${generated.through}`); - return; - } +/** The default mode: writes the baseline the entities and the migrations describe. */ +async function writeMode(): Promise { + const { baseline } = await generateBaseline(); + writePostgresBaseline(baseline); + console.log(`[generate-pg-baseline] wrote ${baseline.statements.length} statements through ${baseline.through}`); +} + +/** `--check` (CI): exits 1, naming every problem, when the committed baseline has drifted. */ +async function checkMode(): Promise { + const { baseline: generated, sqliteTriggers } = await generateBaseline(); const committed = readPostgresBaseline(); - const problems = checkBaseline(committed, generated, baselineMatches(generated)); - // Once frozen, a later migration's trigger reaches Postgres through the migrator, not this list. - if (committed.through === generated.through) { - problems.push(...triggerPortProblems((await readMigratedSqlite()).triggers, Object.keys(POSTGRES_TRIGGERS))); - } + const problems = checkBaseline(committed, generated, baselineMatches(generated), sqliteTriggers); if (problems.length > 0) { for (const problem of problems) console.error(`[generate-pg-baseline] ${problem}`); process.exit(1); @@ -189,6 +216,11 @@ async function main(): Promise { console.log('[generate-pg-baseline] --check: the Postgres baseline matches the entities and the migrations'); } +async function main(): Promise { + if (process.argv.slice(2).includes('--check')) await checkMode(); + else await writeMode(); +} + if (require.main === module) { main().catch((error: unknown) => { console.error(error); diff --git a/server/src/db/engine/postgres/baseline.ts b/server/src/db/engine/postgres/baseline.ts index 03e308226..bb370d2f7 100644 --- a/server/src/db/engine/postgres/baseline.ts +++ b/server/src/db/engine/postgres/baseline.ts @@ -41,17 +41,26 @@ export function renderBaseline(baseline: PostgresBaseline): { sql: string; manif }; } +/** + * Whether `data` is a manifest a baseline can be read from: a `through` naming + * one of a non-empty list of `migrations`, every entry a string. + */ +function isBaselineManifest(data: unknown): data is BaselineManifest { + if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false; + const { through, migrations } = data; + return ( + typeof through === 'string' && + Array.isArray(migrations) && + migrations.length > 0 && + migrations.every((name): name is string => typeof name === 'string') && + migrations.includes(through) + ); +} + function parseManifest(text: string, file: string): BaselineManifest { const data: unknown = JSON.parse(text); - const manifest = data as Partial; - const valid = - typeof manifest.through === 'string' && - Array.isArray(manifest.migrations) && - manifest.migrations.length > 0 && - manifest.migrations.every((name) => typeof name === 'string') && - manifest.migrations.includes(manifest.through); - if (!valid) throw new Error(`${file} must name "through" and list it among "migrations"`); - return { through: manifest.through as string, migrations: manifest.migrations as string[] }; + if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`); + return { through: data.through, migrations: data.migrations }; } /** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */ diff --git a/server/tests/unit/db/engine/postgres/baseline.test.ts b/server/tests/unit/db/engine/postgres/baseline.test.ts index 825058187..5041d9e12 100644 --- a/server/tests/unit/db/engine/postgres/baseline.test.ts +++ b/server/tests/unit/db/engine/postgres/baseline.test.ts @@ -11,6 +11,7 @@ import { writePostgresBaseline, type PostgresBaseline, } from '../../../../../src/db/engine/postgres/baseline'; +import { POSTGRES_TRIGGERS } from '../../../../../src/db/engine/postgres/postgres-triggers'; import { knownMigrationNames } from '../../../../../src/db/known-migrations'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; @@ -67,6 +68,7 @@ describe('Postgres baseline files', () => { }); describe('checkBaseline', () => { + const PORTED = Object.keys(POSTGRES_TRIGGERS); const generated = (through: string, migrations: string[]): PostgresBaseline => ({ through, migrations, @@ -74,27 +76,50 @@ describe('checkBaseline', () => { }); it('PGBASE-010: at the newest migration the baseline must be exactly what the entities generate', () => { - expect(checkBaseline(sample, generated(B, [A, B]), true)).toEqual([]); - expect(checkBaseline(sample, generated(B, [A, B]), false)).toEqual([ + expect(checkBaseline(sample, generated(B, [A, B]), true, PORTED)).toEqual([]); + expect(checkBaseline(sample, generated(B, [A, B]), false, PORTED)).toEqual([ 'the entities generate a different schema; run npm run gen:pg-baseline', ]); }); it('PGBASE-011: once a migration lands after the cut, the baseline is frozen and only its manifest is checked', () => { - expect(checkBaseline(sample, generated(C, [A, B, C]), false)).toEqual([]); + expect(checkBaseline(sample, generated(C, [A, B, C]), false, PORTED)).toEqual([]); + }); + + it('PGBASE-013: while not frozen, a SQLite trigger without a port is reported; once frozen the triggers are not checked', () => { + const withNewTrigger = [...PORTED, 'trg_new_trigger']; + expect(checkBaseline(sample, generated(B, [A, B]), true, withNewTrigger)).toEqual([ + 'SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: trg_new_trigger', + ]); + expect(checkBaseline(sample, generated(C, [A, B, C]), false, withNewTrigger)).toEqual([]); }); it('PGBASE-012: a manifest naming a migration that is gone, or missing one before its cut, is reported', () => { - expect(checkBaseline(sample, generated(C, [B, C]), true)).toEqual([ + expect(checkBaseline(sample, generated(C, [B, C]), true, PORTED)).toEqual([ `the baseline names migrations that do not exist: ${A}`, ]); const skipped = { ...sample, migrations: [B] }; - expect(checkBaseline(skipped, generated(B, [A, B]), true)).toEqual([ + expect(checkBaseline(skipped, generated(B, [A, B]), true, PORTED)).toEqual([ `migrations up to ${B} are missing from it: ${A}`, ]); }); }); +describe('readPostgresBaseline manifest', () => { + it('PGBASE-004: a manifest whose migrations are not all names, or are empty, is refused', () => { + const target = scratch(); + for (const manifest of [ + { through: B, migrations: [A, 3] }, + { through: B, migrations: [] }, + { through: 7, migrations: [B] }, + ]) { + writePostgresBaseline(sample, target); + writeFileSync(path.join(target, 'baseline.json'), JSON.stringify(manifest)); + expect(() => readPostgresBaseline(target), JSON.stringify(manifest)).toThrow(/must name "through"/); + } + }); +}); + describe('triggerPortProblems', () => { it('PGBASE-020: every SQLite trigger needs a Postgres port, and a port without its trigger is stale', () => { expect(triggerPortProblems(['a', 'b'], ['b', 'a'])).toEqual([]);