diff --git a/server/scripts/generate-pg-baseline.ts b/server/scripts/generate-pg-baseline.ts index 558199023..3c578c88d 100644 --- a/server/scripts/generate-pg-baseline.ts +++ b/server/scripts/generate-pg-baseline.ts @@ -36,13 +36,18 @@ import { MikroORM } from '@mikro-orm/postgresql'; import Database from 'better-sqlite3'; import fs from 'node:fs'; -/** The baseline the entities and the migrations on disk describe now. */ -export async function generatePostgresBaseline(known: Set = knownMigrationNames()): Promise { +/** What one pass over the migrated SQLite database yields: the baseline, and the triggers the migrations leave there. */ +interface BaselinePass { + baseline: PostgresBaseline; + sqliteTriggers: string[]; +} + +/** The baseline the entities and the migrations on disk describe now, from one migration of the SQLite database. */ +async function generateBaseline(known: Set = knownMigrationNames()): Promise { const migrations = [...known].sort(); const through = migrations.at(-1); if (!through) throw new Error('no migrations found'); const orm = await MikroORM.init({ - // Never contacted: init() only discovers metadata, and the schema generator needs nothing else. clientUrl: 'postgresql://generator@localhost/trek', entities: [...ALL_ENTITIES], discovery: { warnWhenNoEntities: false }, @@ -56,19 +61,26 @@ export async function generatePostgresBaseline(known: Set = knownMigrati ...migrated.tables.flatMap((table) => rowStatements(table.name, table.rows)), ...Object.values(POSTGRES_TRIGGERS).flat(), ]; - return { through, migrations, statements }; + return { baseline: { through, migrations, statements }, sqliteTriggers: migrated.triggers }; } finally { await orm.close(); } } /** - * What `--check` holds. While no migration is newer than the committed - * baseline, the baseline must be exactly what the entities generate. Once one - * is, the baseline is frozen and only its manifest is checked: every migration - * it names exists, and it names every migration up to `through`. + * What `--check` holds, as the problems it finds. Every manifest must name + * migrations that exist and every migration up to `through`. While no migration + * is newer than the committed baseline (`through` matches), the baseline is + * not frozen: it must be exactly what the entities generate, and each SQLite + * trigger needs a port. Once a newer migration lands, the baseline is frozen + * and only the manifest is checked. */ -export function checkBaseline(committed: PostgresBaseline, generated: PostgresBaseline, matches: boolean): string[] { +export function checkBaseline( + committed: PostgresBaseline, + generated: PostgresBaseline, + matches: boolean, + sqliteTriggers: readonly string[], +): string[] { const known = new Set(generated.migrations); const problems: string[] = []; const missing = committed.migrations.filter((name) => !known.has(name)); @@ -78,9 +90,10 @@ export function checkBaseline(committed: PostgresBaseline, generated: PostgresBa ); if (unlisted.length > 0) problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`); - if (committed.through === generated.through && !matches) { - problems.push('the entities generate a different schema; run npm run gen:pg-baseline'); - } + const frozen = committed.through !== generated.through; + if (frozen) return problems; + if (!matches) problems.push('the entities generate a different schema; run npm run gen:pg-baseline'); + problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS))); return problems; } @@ -90,32 +103,48 @@ const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schem type Row = Record; /** What the migrations leave in a SQLite database: its triggers, and the rows they insert, table by table in creation order. */ -async function readMigratedSqlite(): Promise<{ triggers: string[]; tables: Array<{ name: string; rows: Row[] }> }> { +interface MigratedSchema { + triggers: string[]; + tables: Array<{ name: string; rows: Row[] }>; +} + +/** The tables and triggers of a SQLite database in creation order, each as its type and name. */ +function listSchemaObjects(conn: Database.Database): Array<{ type: string; name: string }> { + return conn + .prepare( + "select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid", + ) + .all() as Array<{ type: string; name: string }>; +} + +/** Reads a migrated SQLite database, read-only. */ +function readMigratedSchema(dbPath: string): MigratedSchema { + const conn = new Database(dbPath, { readonly: true }); + try { + const objects = listSchemaObjects(conn); + const tables = objects + .filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name)) + .map((object) => ({ + name: object.name, + rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[], + })) + .filter((table) => table.rows.length > 0); + const triggers = objects + .filter((object) => object.type === 'trigger') + .map((object) => object.name) + .sort(); + return { triggers, tables }; + } finally { + conn.close(); + } +} + +/** Migrates a temporary SQLite database and reads what the migrations leave in it; the file is removed afterwards. */ +async function readMigratedSqlite(): Promise { const dbPath = tempDbPath(); try { await migrateTempDb(dbPath); - const conn = new Database(dbPath, { readonly: true }); - try { - const objects = conn - .prepare( - "select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid", - ) - .all() as Array<{ type: string; name: string }>; - const tables = objects - .filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name)) - .map((object) => ({ - name: object.name, - rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[], - })) - .filter((table) => table.rows.length > 0); - const triggers = objects - .filter((object) => object.type === 'trigger') - .map((object) => object.name) - .sort(); - return { triggers, tables }; - } finally { - conn.close(); - } + return readMigratedSchema(dbPath); } finally { fs.rmSync(dbPath, { force: true }); } @@ -143,7 +172,6 @@ export function rowStatements(table: string, rows: readonly Row[]): string[] { const statements = [ `insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`, ]; - // A text key (addons.id) has no sequence; a numeric one may. if (rows.every((row) => typeof row.id === 'number')) { statements.push( `select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('"${table}"', 'id') as seq, ` + @@ -169,19 +197,18 @@ export function triggerPortProblems(sqliteTriggers: readonly string[], ported: r return problems; } -async function main(): Promise { - const generated = await generatePostgresBaseline(); - if (!process.argv.slice(2).includes('--check')) { - writePostgresBaseline(generated); - console.log(`[generate-pg-baseline] wrote ${generated.statements.length} statements through ${generated.through}`); - return; - } +/** The default mode: writes the baseline the entities and the migrations describe. */ +async function writeMode(): Promise { + const { baseline } = await generateBaseline(); + writePostgresBaseline(baseline); + console.log(`[generate-pg-baseline] wrote ${baseline.statements.length} statements through ${baseline.through}`); +} + +/** `--check` (CI): exits 1, naming every problem, when the committed baseline has drifted. */ +async function checkMode(): Promise { + const { baseline: generated, sqliteTriggers } = await generateBaseline(); const committed = readPostgresBaseline(); - const problems = checkBaseline(committed, generated, baselineMatches(generated)); - // Once frozen, a later migration's trigger reaches Postgres through the migrator, not this list. - if (committed.through === generated.through) { - problems.push(...triggerPortProblems((await readMigratedSqlite()).triggers, Object.keys(POSTGRES_TRIGGERS))); - } + const problems = checkBaseline(committed, generated, baselineMatches(generated), sqliteTriggers); if (problems.length > 0) { for (const problem of problems) console.error(`[generate-pg-baseline] ${problem}`); process.exit(1); @@ -189,6 +216,11 @@ async function main(): Promise { console.log('[generate-pg-baseline] --check: the Postgres baseline matches the entities and the migrations'); } +async function main(): Promise { + if (process.argv.slice(2).includes('--check')) await checkMode(); + else await writeMode(); +} + if (require.main === module) { main().catch((error: unknown) => { console.error(error); diff --git a/server/src/db/engine/postgres/baseline.ts b/server/src/db/engine/postgres/baseline.ts index 03e308226..bb370d2f7 100644 --- a/server/src/db/engine/postgres/baseline.ts +++ b/server/src/db/engine/postgres/baseline.ts @@ -41,17 +41,26 @@ export function renderBaseline(baseline: PostgresBaseline): { sql: string; manif }; } +/** + * Whether `data` is a manifest a baseline can be read from: a `through` naming + * one of a non-empty list of `migrations`, every entry a string. + */ +function isBaselineManifest(data: unknown): data is BaselineManifest { + if (typeof data !== 'object' || data === null || !('through' in data) || !('migrations' in data)) return false; + const { through, migrations } = data; + return ( + typeof through === 'string' && + Array.isArray(migrations) && + migrations.length > 0 && + migrations.every((name): name is string => typeof name === 'string') && + migrations.includes(through) + ); +} + function parseManifest(text: string, file: string): BaselineManifest { const data: unknown = JSON.parse(text); - const manifest = data as Partial; - const valid = - typeof manifest.through === 'string' && - Array.isArray(manifest.migrations) && - manifest.migrations.length > 0 && - manifest.migrations.every((name) => typeof name === 'string') && - manifest.migrations.includes(manifest.through); - if (!valid) throw new Error(`${file} must name "through" and list it among "migrations"`); - return { through: manifest.through as string, migrations: manifest.migrations as string[] }; + if (!isBaselineManifest(data)) throw new Error(`${file} must name "through" and list it among "migrations"`); + return { through: data.through, migrations: data.migrations }; } /** Reads the committed baseline; a missing or malformed one throws, so a boot cannot start from half a schema. */ diff --git a/server/tests/unit/db/engine/postgres/baseline.test.ts b/server/tests/unit/db/engine/postgres/baseline.test.ts index 825058187..5041d9e12 100644 --- a/server/tests/unit/db/engine/postgres/baseline.test.ts +++ b/server/tests/unit/db/engine/postgres/baseline.test.ts @@ -11,6 +11,7 @@ import { writePostgresBaseline, type PostgresBaseline, } from '../../../../../src/db/engine/postgres/baseline'; +import { POSTGRES_TRIGGERS } from '../../../../../src/db/engine/postgres/postgres-triggers'; import { knownMigrationNames } from '../../../../../src/db/known-migrations'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; @@ -67,6 +68,7 @@ describe('Postgres baseline files', () => { }); describe('checkBaseline', () => { + const PORTED = Object.keys(POSTGRES_TRIGGERS); const generated = (through: string, migrations: string[]): PostgresBaseline => ({ through, migrations, @@ -74,27 +76,50 @@ describe('checkBaseline', () => { }); it('PGBASE-010: at the newest migration the baseline must be exactly what the entities generate', () => { - expect(checkBaseline(sample, generated(B, [A, B]), true)).toEqual([]); - expect(checkBaseline(sample, generated(B, [A, B]), false)).toEqual([ + expect(checkBaseline(sample, generated(B, [A, B]), true, PORTED)).toEqual([]); + expect(checkBaseline(sample, generated(B, [A, B]), false, PORTED)).toEqual([ 'the entities generate a different schema; run npm run gen:pg-baseline', ]); }); it('PGBASE-011: once a migration lands after the cut, the baseline is frozen and only its manifest is checked', () => { - expect(checkBaseline(sample, generated(C, [A, B, C]), false)).toEqual([]); + expect(checkBaseline(sample, generated(C, [A, B, C]), false, PORTED)).toEqual([]); + }); + + it('PGBASE-013: while not frozen, a SQLite trigger without a port is reported; once frozen the triggers are not checked', () => { + const withNewTrigger = [...PORTED, 'trg_new_trigger']; + expect(checkBaseline(sample, generated(B, [A, B]), true, withNewTrigger)).toEqual([ + 'SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: trg_new_trigger', + ]); + expect(checkBaseline(sample, generated(C, [A, B, C]), false, withNewTrigger)).toEqual([]); }); it('PGBASE-012: a manifest naming a migration that is gone, or missing one before its cut, is reported', () => { - expect(checkBaseline(sample, generated(C, [B, C]), true)).toEqual([ + expect(checkBaseline(sample, generated(C, [B, C]), true, PORTED)).toEqual([ `the baseline names migrations that do not exist: ${A}`, ]); const skipped = { ...sample, migrations: [B] }; - expect(checkBaseline(skipped, generated(B, [A, B]), true)).toEqual([ + expect(checkBaseline(skipped, generated(B, [A, B]), true, PORTED)).toEqual([ `migrations up to ${B} are missing from it: ${A}`, ]); }); }); +describe('readPostgresBaseline manifest', () => { + it('PGBASE-004: a manifest whose migrations are not all names, or are empty, is refused', () => { + const target = scratch(); + for (const manifest of [ + { through: B, migrations: [A, 3] }, + { through: B, migrations: [] }, + { through: 7, migrations: [B] }, + ]) { + writePostgresBaseline(sample, target); + writeFileSync(path.join(target, 'baseline.json'), JSON.stringify(manifest)); + expect(() => readPostgresBaseline(target), JSON.stringify(manifest)).toThrow(/must name "through"/); + } + }); +}); + describe('triggerPortProblems', () => { it('PGBASE-020: every SQLite trigger needs a Postgres port, and a port without its trigger is stale', () => { expect(triggerPortProblems(['a', 'b'], ['b', 'a'])).toEqual([]);