From aa77e21d4b51922dcb1df4ecb0f4c4110b38920c Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 21:39:27 +0100 Subject: [PATCH] fix(db): refuse to regenerate a frozen Postgres baseline and check its trigger ports once frozen gen:pg-baseline rewrote the baseline with its cut at the newest migration on disk, so after a migration landed after the cut, running it moved the cut over that migration while Postgres installs created from the old baseline would still run it through the migrator. It now refuses while a migration follows the committed cut, names those migrations and the routine, and moves the cut only with --move-cut (lint:dialect keeps such a migration under the post-cut rules either way). check:pg-baseline stopped comparing SQLite triggers with POSTGRES_TRIGGERS once the baseline was frozen, so a trigger a later migration created on SQLite went unnoticed. The comparison now runs in both states, with advice that points at the migration when the baseline is frozen. --- .github/workflows/test.yml | 1 + server/CLAUDE.md | 6 +- server/scripts/generate-pg-baseline.ts | 82 +++++++++++++++---- .../unit/db/engine/postgres/baseline.test.ts | 27 +++++- 4 files changed, 94 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39facabf8..9109af968 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -292,6 +292,7 @@ jobs: # lands after its cut it must match them exactly; after, it is frozen, # its list of migrations is checked and the migrations after the cut are # listed: lint:dialect holds their engineDdl and PARITY-PG their result. + # Either way every SQLite trigger needs its Postgres port. run: cd server && npm run check:pg-baseline server-lint-checks: diff --git a/server/CLAUDE.md b/server/CLAUDE.md index 22bbd3c40..5ef1c30a6 100644 --- a/server/CLAUDE.md +++ b/server/CLAUDE.md @@ -20,8 +20,8 @@ npm run gen:plugin-facts # regenerate the plugin-protocol tables into plugin-sd npm run check:plugin-facts # CI gate — fails if the generated copies drifted npm run gen:db-types # regenerate the Kysely table types in src/db/kysely/ from the migrated schema npm run check:db-types # CI gate: fails when src/db/kysely/ no longer matches the migrations -npm run gen:pg-baseline # regenerate the Postgres baseline in assets/postgres/ from the entities (never for a migration after its cut) -npm run check:pg-baseline # CI gate: fails when the manifest drifted from the migrations; until a migration follows the cut, also when the baseline drifted from the entities or a SQLite trigger has no Postgres port; lists the migrations after the cut +npm run gen:pg-baseline # regenerate the Postgres baseline in assets/postgres/ from the entities; refused while a migration follows its cut (-- --move-cut folds them in on purpose) +npm run check:pg-baseline # CI gate: fails when the manifest drifted from the migrations or a SQLite trigger has no Postgres port; until a migration follows the cut, also when the baseline drifted from the entities; lists the migrations after the cut npm run db:move-to-postgres # copy a SQLite install (data/travel.db, --from=) into the empty Postgres database TREK_DB_URL names; dist/move-to-postgres.js in the image npm run gen:pg-migration # print the Postgres DDL of the newest migration (--migration= for another) for its engineDdl call; needs TREK_TEST_DB_URL or --url node scripts/coverage-thresholds.mjs # after test:coverage — prints the ratchet block for vitest.config.ts @@ -122,7 +122,7 @@ SQLite is the default engine and Postgres the second (`TREK_DB_URL`, see Engines 3. Run `npm run gen:pg-migration` (with `TREK_TEST_DB_URL` naming a Postgres server it may create a scratch database on, or `--url=`). It builds a database the way a boot does up to the new migration, compares it with the entities by PARITY-PG's own rules (`src/db/engine/postgres/entity-schema-diff.ts`) and prints the `postgres: [...]` member to paste. Nothing to do is printed as `postgres: [],`; keep it only with a `// sqlite-only: ` comment above it. 4. Push. `lint:dialect` fails a schema statement outside `engineDdl`, a SQLite spelling in its postgres member, anything but the schema change in its sqlite member and an unexplained empty postgres member, and the Postgres lane runs the migration and PARITY-PG. - The catalog helpers (`tableExists`, `columnNames`, which read `src/db/dialect/catalog.ts`) answer on both engines; `tableSql` and `createPlaceRegionsFollowPlaceTrigger` are SQLite's only. A migration never branches on the engine itself (`lint:engine-dispatch`). The baseline is not regenerated for a migration after its cut: an install created from the old one only gets the change through the migration, so `check:pg-baseline` then verifies the manifest and lists the migrations after the cut, and nothing moves the cut on its own. A trigger after the cut is not yet supported by PARITY-PG's rules (only `POSTGRES_TRIGGERS` are known); add one only together with that. + The catalog helpers (`tableExists`, `columnNames`, which read `src/db/dialect/catalog.ts`) answer on both engines; `tableSql` and `createPlaceRegionsFollowPlaceTrigger` are SQLite's only. A migration never branches on the engine itself (`lint:engine-dispatch`). The baseline is not regenerated for a migration after its cut: an install created from the old one only gets the change through the migration, so `check:pg-baseline` then verifies the manifest and the trigger ports and lists the migrations after the cut, and `gen:pg-baseline` refuses to rewrite it. `--move-cut` folds them into a new baseline on purpose; that is safe because `lint:dialect` holds every migration after `POSTGRES_FIRST_CUT` to the post-cut rules wherever the cut stands, so an install created from the old baseline reaches the same schema through the migrator. A trigger after the cut is not yet supported by PARITY-PG's rules (only `POSTGRES_TRIGGERS` are known); add one only together with that. - **Storage model**: the generated Kysely types describe what SQLite stores (0/1 integers for flags, TEXT timestamps, JSON in TEXT). Convert at the service boundary, as the contracts already require. - **MySQL/MariaDB are out of scope.** The repositories rely on `INSERT ... ON CONFLICT` (Kysely's `onConflict`, often a target-less `doNothing()`) and on `RETURNING`, and MySQL has neither: it would need an insert-ignore/upsert wrapper in the dialect layer and a different way to read new ids before it could be considered. diff --git a/server/scripts/generate-pg-baseline.ts b/server/scripts/generate-pg-baseline.ts index dd3e8e438..887e3dec8 100644 --- a/server/scripts/generate-pg-baseline.ts +++ b/server/scripts/generate-pg-baseline.ts @@ -14,12 +14,18 @@ * generated (`through`). Once a migration lands after it, the baseline is * frozen like a shipped migration: the newer migration runs on Postgres * through the migrator, written through `engineDdl` with the Postgres side - * from `npm run gen:pg-migration`, and `--check` verifies the manifest and - * lists the migrations after the cut. Nothing here moves the cut. + * from `npm run gen:pg-migration`, and `--check` verifies the manifest, that + * every SQLite trigger still has its port, and lists the migrations after the + * cut. Rewriting a frozen baseline moves the cut, so the default mode refuses + * it and only `--move-cut` does it. That is safe because lint:dialect holds + * every migration after POSTGRES_FIRST_CUT to the post-cut rules wherever the + * cut stands, so an install created from the old baseline still reaches the + * same schema through the migrator. * * Usage: - * node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/ - * node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI) + * node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/ (not while frozen) + * node --import tsx scripts/generate-pg-baseline.ts --move-cut # rewrite it over the migrations after the cut + * node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI) */ import { baselineMatches, @@ -114,18 +120,36 @@ export function checkBaseline( ); } const frozen = committed.through !== generated.through; - if (frozen) return problems; - if (!matches) { + if (!frozen && !matches) { problems.push( 'server/assets/postgres/ is not what the entities, POSTGRES_TRIGGERS and the reference rows the migrations ' + 'insert generate now, and no migration follows its cut: run npm run gen:pg-baseline and commit ' + 'server/assets/postgres/.', ); } - problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS))); + // Frozen or not: the entities carry no triggers, so this is the only check + // that a trigger a migration leaves on SQLite exists on Postgres too. + problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS), frozen)); return problems; } +/** + * Why the default mode refuses to rewrite the baseline, or null when it may: + * the migrations after the committed cut. Rewriting the baseline would move + * the cut over them, which only `--move-cut` does. + */ +export function cutMoveRefusal(committed: PostgresBaseline, known: readonly string[]): string | null { + const after = postCutMigrations(committed, known); + if (after.length === 0) return null; + return ( + `the baseline is frozen at ${committed.through}, and ${after.length} migration(s) follow its cut: ` + + `${after.join(', ')}. A Postgres install created from it reaches them through the migrator, so the baseline ` + + 'is not regenerated for them: write their schema change through engineDdl with the postgres member that npm ' + + 'run gen:pg-migration prints (server/CLAUDE.md, SQL dialect, Migrations). To fold them into a new baseline on ' + + 'purpose, run npm run gen:pg-baseline -- --move-cut.' + ); +} + /** Migration bookkeeping: the baseline's manifest stands for it on Postgres. */ const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schema_version']); @@ -213,29 +237,52 @@ export function rowStatements(table: string, rows: readonly Row[]): string[] { /** * The SQLite triggers without a Postgres port in `POSTGRES_TRIGGERS`, and the * ports whose trigger is gone. The entities carry no triggers, so this is - * what keeps the hand-written ports level with the migrations. + * what keeps the hand-written ports level with the migrations. `frozen` says + * a migration follows the baseline's cut, so the fix goes into that + * migration rather than into a regenerated baseline. */ -export function triggerPortProblems(sqliteTriggers: readonly string[], ported: readonly string[]): string[] { +export function triggerPortProblems( + sqliteTriggers: readonly string[], + ported: readonly string[], + frozen = false, +): string[] { const problems: string[] = []; const missing = sqliteTriggers.filter((name) => !ported.includes(name)); if (missing.length > 0) { problems.push( - `SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}. Add each one's ` + - 'function and trigger to src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.', + frozen + ? `SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}. A migration after ` + + "the baseline's cut created them, and a trigger after the cut has no Postgres port (server/CLAUDE.md, " + + 'SQL dialect, Migrations): take it out of the migration and do its work in the repository instead.' + : `SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}. Add each one's ` + + 'function and trigger to src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.', ); } const stale = ported.filter((name) => !sqliteTriggers.includes(name)); if (stale.length > 0) { problems.push( - `POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}. Remove them from ` + - 'src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.', + frozen + ? `POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}. A migration ` + + "after the baseline's cut dropped them on SQLite: drop them on Postgres in the postgres member of its " + + 'engineDdl call too, and remove them from src/db/engine/postgres/postgres-triggers.ts.' + : `POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}. Remove them from ` + + 'src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.', ); } return problems; } -/** The default mode: writes the baseline the entities and the migrations describe. */ -async function writeMode(): Promise { +/** + * The default mode: writes the baseline the entities and the migrations + * describe, refused while the committed one is frozen unless `moveCut`. + */ +async function writeMode(moveCut: boolean): Promise { + const known = [...knownMigrationNames()].sort(); + const refusal = cutMoveRefusal(readPostgresBaseline(), known); + if (refusal && !moveCut) { + console.error(`[generate-pg-baseline] ${refusal}`); + process.exit(1); + } const { baseline } = await generateBaseline(); writePostgresBaseline(baseline); console.log(`[generate-pg-baseline] wrote ${baseline.statements.length} statements through ${baseline.through}`); @@ -268,8 +315,9 @@ async function checkMode(): Promise { } async function main(): Promise { - if (process.argv.slice(2).includes('--check')) await checkMode(); - else await writeMode(); + const args = process.argv.slice(2); + if (args.includes('--check')) await checkMode(); + else await writeMode(args.includes('--move-cut')); } if (require.main === module) { diff --git a/server/tests/unit/db/engine/postgres/baseline.test.ts b/server/tests/unit/db/engine/postgres/baseline.test.ts index bc0f7a777..4c7336b12 100644 --- a/server/tests/unit/db/engine/postgres/baseline.test.ts +++ b/server/tests/unit/db/engine/postgres/baseline.test.ts @@ -5,6 +5,7 @@ */ import { checkBaseline, + cutMoveRefusal, postCutMigrations, rowStatements, triggerPortProblems, @@ -98,13 +99,27 @@ describe('checkBaseline', () => { expect(checkBaseline(sample, generated(C, [A, B, C]), false, PORTED)).toEqual([]); }); - it('PGBASE-013: while not frozen, a SQLite trigger without a port is reported; once frozen the triggers are not checked', () => { + it('PGBASE-013: a SQLite trigger without a port is reported, before the cut and once frozen alike', () => { const withNewTrigger = [...PORTED, 'trg_new_trigger']; expect(checkBaseline(sample, generated(B, [A, B]), true, withNewTrigger)).toEqual([ "SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: trg_new_trigger. Add each one's function and " + 'trigger to src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.', ]); - expect(checkBaseline(sample, generated(C, [A, B, C]), false, withNewTrigger)).toEqual([]); + // A post-cut migration that left a trigger on SQLite alone: frozen, the + // baseline is not regenerated, so the advice points at the migration. + const frozen = checkBaseline(sample, generated(C, [A, B, C]), false, withNewTrigger); + expect(frozen).toHaveLength(1); + expect(frozen[0]).toContain('SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: trg_new_trigger.'); + expect(frozen[0]).toContain('do its work in the repository instead'); + expect(frozen[0]).not.toContain('gen:pg-baseline'); + }); + + it('PGBASE-015: rewriting a frozen baseline is refused, naming the migrations after its cut and --move-cut', () => { + expect(cutMoveRefusal(sample, [A, B])).toBeNull(); + const refusal = cutMoveRefusal(sample, [A, B, C]); + expect(refusal).toContain(`the baseline is frozen at ${B}, and 1 migration(s) follow its cut: ${C}.`); + expect(refusal).toContain('npm run gen:pg-migration'); + expect(refusal).toContain('npm run gen:pg-baseline -- --move-cut'); }); it('PGBASE-012: a manifest naming a migration that is gone, or missing one before its cut, is reported', () => { @@ -146,6 +161,14 @@ describe('triggerPortProblems', () => { ]); }); + it('PGBASE-022: once frozen, the advice goes to the migration after the cut, not to a regenerated baseline', () => { + const [missing, stale] = triggerPortProblems(['a', 'b'], ['a', 'c'], true); + expect(missing).toContain('SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: b.'); + expect(missing).toContain('take it out of the migration'); + expect(stale).toContain('POSTGRES_TRIGGERS ports triggers the migrations no longer create: c.'); + expect(stale).toContain('in the postgres member of its engineDdl call'); + }); + it('PGBASE-021: the committed baseline carries every trigger port after the tables', () => { const statements = readPostgresBaseline().statements; const firstTrigger = statements.findIndex((statement) => statement.startsWith('CREATE FUNCTION'));