From 0f338d5484d52dce9235e12a778f9f6102f49599 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 11:48:17 +0100 Subject: [PATCH] refactor(db): move PARITY-PG's schema comparison into the Postgres engine Which comparator statements change nothing (drop not null on a live primary key, spelling-only defaults and checks proven by a rolled-back fingerprint, the hand-ported triggers) is now one module the parity test and the migration generator share, with unit tests for the sorting and the rollback. --- .../db/engine/postgres/entity-schema-diff.ts | 191 ++++++++++++++++++ server/tests/postgres/schema-parity.test.ts | 130 ++---------- .../postgres/entity-schema-diff.test.ts | 124 ++++++++++++ 3 files changed, 329 insertions(+), 116 deletions(-) create mode 100644 server/src/db/engine/postgres/entity-schema-diff.ts create mode 100644 server/tests/unit/db/engine/postgres/entity-schema-diff.test.ts diff --git a/server/src/db/engine/postgres/entity-schema-diff.ts b/server/src/db/engine/postgres/entity-schema-diff.ts new file mode 100644 index 000000000..697098162 --- /dev/null +++ b/server/src/db/engine/postgres/entity-schema-diff.ts @@ -0,0 +1,191 @@ +/** + * What still separates a live Postgres database from the schema the entities + * describe for Postgres (`schema-adjust.ts`): the one definition PARITY-PG + * (`tests/postgres/schema-parity.test.ts`) holds at nothing and + * `npm run gen:pg-migration` prints as the Postgres side of a new migration. + * + * MikroORM's comparator also reports statements that change nothing, and each + * kind is proven so rather than recognised by its text alone: + * - `drop not null` on a primary key column: SQLite lets a key column be null + * and the entities say so; Postgres keeps every primary key NOT NULL (and + * refuses the statement), checked against the live keys. + * - a column default or a check constraint Postgres stores in its own + * spelling (casts added, EXTRACT rewritten, BETWEEN as two comparisons): + * such statements are applied in a transaction that is rolled back, and the + * catalogue's own rendering of every default and check must read the same + * after as before. A default or a bound that really differs changes it. + * - dropping a trigger the entities cannot carry: only the triggers ported by + * hand (`POSTGRES_TRIGGERS`), by exact name, and only while the trigger is in + * the database on its table. + */ +import { ALL_ENTITIES } from '../../entities'; +import { POSTGRES_TRIGGERS } from './postgres-triggers'; +import { applyPostgresColumnAdjustments } from './schema-adjust'; +import type { Transaction } from '@mikro-orm/core'; +import { MikroORM } from '@mikro-orm/postgresql'; +import type { AbstractSqlConnection } from '@mikro-orm/sql'; + +const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/; +const SPELLING = + /^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s; +/** MikroORM drops a trigger by its name, and the function it would have made for it as `__fn`. */ +const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/; +const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/; + +/** Every column default and check constraint of the schema, as the catalogue renders them. */ +const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from ( + select concat('default ', c.relname, '.', a.attname, ' = ', pg_get_expr(d.adbin, d.adrelid)) as item + from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum + where c.relnamespace = current_schema()::regnamespace + union all + select concat('check ', conrelid::regclass, '.', conname, ' = ', pg_get_constraintdef(oid)) + from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace +) items`; + +const PRIMARY_KEYS_SQL = `select concat(c.relname, '.', a.attname) as name from pg_index i + join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey) + where i.indisprimary and c.relnamespace = current_schema()::regnamespace`; + +const TRIGGERS_SQL = `select concat(c.relname, '.', t.tgname) as name from pg_trigger t join pg_class c on c.oid = t.tgrelid + where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`; + +/** The facts of the live schema the classification needs, each as `
.`. */ +export interface LiveSchemaFacts { + primaryKeys: ReadonlySet; + triggers: ReadonlySet; +} + +/** The comparator's statements sorted by what they would do. */ +interface ClassifiedDiff { + /** Statements that change the schema: what a migration has to send. */ + changes: string[]; + /** Default and check statements that may only be Postgres's own spelling; `spellingChangesNothing` decides. */ + spelling: string[]; + /** `drop not null` on a live primary key, which Postgres keeps NOT NULL anyway. */ + keyNotNull: string[]; + /** Dropping a hand-ported trigger or its function. */ + portedTriggers: string[]; +} + +/** The comparator's SQL (`getUpdateSchemaSQL({ wrap: false })`) as single statements without their semicolons. */ +export function splitSchemaDiff(sql: string): string[] { + return sql + .split(';\n') + .map((statement) => statement.trim().replace(/;$/, '')) + .filter((statement) => statement !== ''); +} + +/** Sorts the comparator's statements; `ported` names the triggers `POSTGRES_TRIGGERS` carries. */ +export function classifySchemaDiff( + statements: readonly string[], + live: LiveSchemaFacts, + ported: readonly string[] = Object.keys(POSTGRES_TRIGGERS), +): ClassifiedDiff { + const isKeyNotNull = (s: string) => { + const m = DROP_NOT_NULL.exec(s); + return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`); + }; + const isPortedTrigger = (s: string) => { + const trigger = DROP_TRIGGER.exec(s); + if (trigger) return ported.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`); + const fn = DROP_FUNCTION.exec(s); + return fn !== null && ported.some((name) => fn[1].endsWith(`_${name}_fn`)); + }; + const diff: ClassifiedDiff = { changes: [], spelling: [], keyNotNull: [], portedTriggers: [] }; + for (const statement of statements) { + if (isKeyNotNull(statement)) diff.keyNotNull.push(statement); + else if (isPortedTrigger(statement)) diff.portedTriggers.push(statement); + else if (SPELLING.test(statement)) diff.spelling.push(statement); + else diff.changes.push(statement); + } + return diff; +} + +/** The hand-ported triggers that are not in the live database. */ +export function missingPortedTriggers( + live: LiveSchemaFacts, + ported: readonly string[] = Object.keys(POSTGRES_TRIGGERS), +): string[] { + return ported.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`))); +} + +/** Reads the live primary keys and triggers of the current schema. */ +async function readLiveSchemaFacts(connection: AbstractSqlConnection): Promise { + const names = (sql: string) => + connection.execute>(sql).then((rows) => new Set(rows.map((row) => row.name))); + const [primaryKeys, triggers] = await Promise.all([names(PRIMARY_KEYS_SQL), names(TRIGGERS_SQL)]); + return { primaryKeys, triggers }; +} + +const ROLLBACK = new Error('roll back the spelling check'); + +/** + * The catalogue's rendering of every default and check before and after + * `statements`, applied in a transaction that is always rolled back: equal + * means they change nothing. + */ +export async function spellingChangesNothing( + connection: AbstractSqlConnection, + statements: readonly string[], +): Promise<{ before: string; after: string }> { + const fingerprint = async (trx?: Transaction) => + (await connection.execute>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint; + const before = await fingerprint(); + let after = ''; + await connection + .transactional(async (trx) => { + // One after the other: a drop and the add that follows name the same constraint. + await statements.reduce>( + (previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)), + Promise.resolve(), + ); + after = await fingerprint(trx); + throw ROLLBACK; + }) + .catch((err: unknown) => { + if (err !== ROLLBACK) throw err; + }); + return { before, after }; +} + +/** Everything the comparison of one database with the entities found. */ +export interface EntitySchemaDiff extends ClassifiedDiff { + /** Whether the `spelling` statements leave every default and check as it was. */ + spellingHolds: boolean; + /** The catalogue's defaults and checks before and after the spelling statements, for a failure message. */ + fingerprint: { before: string; after: string }; + /** Hand-ported triggers missing from the database. */ + missingTriggers: string[]; +} + +/** The statements a migration must still send: the changes, and the spelling ones when they are more than spelling. */ +export function remainingStatements(diff: EntitySchemaDiff): string[] { + return diff.spellingHolds ? [...diff.changes] : [...diff.changes, ...diff.spelling]; +} + +/** + * Compares the database `url` names with the entities, read for Postgres the + * way the baseline writes them. Nothing in the database changes. + */ +export async function diffAgainstEntities(url: string): Promise { + const orm = await MikroORM.init({ + clientUrl: url, + entities: [...ALL_ENTITIES], + discovery: { warnWhenNoEntities: false }, + }); + try { + for (const meta of orm.getMetadata().getAll().values()) applyPostgresColumnAdjustments(meta.props); + const connection = orm.em.getConnection(); + const live = await readLiveSchemaFacts(connection); + const classified = classifySchemaDiff(splitSchemaDiff(await orm.schema.getUpdateSchemaSQL({ wrap: false })), live); + const fingerprint = await spellingChangesNothing(connection, classified.spelling); + return { + ...classified, + spellingHolds: fingerprint.before === fingerprint.after, + fingerprint, + missingTriggers: missingPortedTriggers(live), + }; + } finally { + await orm.close(true); + } +} diff --git a/server/tests/postgres/schema-parity.test.ts b/server/tests/postgres/schema-parity.test.ts index 3c1559e75..917098976 100644 --- a/server/tests/postgres/schema-parity.test.ts +++ b/server/tests/postgres/schema-parity.test.ts @@ -7,127 +7,25 @@ * the cut that left Postgres behind SQLite, or a baseline that drifted from * the entities, fails here. It runs on the lane's database, never skipped. * - * The comparator also reports statements that change nothing, and each kind - * is proven so rather than recognised by its text: - * - `drop not null` on a primary key column: SQLite lets a key column be null - * and the entities say so; Postgres keeps every primary key NOT NULL (and - * refuses the statement), checked against the live keys. - * - a column default or a check constraint Postgres stores in its own spelling - * (casts added, EXTRACT rewritten, BETWEEN as two comparisons): such - * statements are applied in a transaction that is rolled back, and the - * catalogue's own rendering of every default and check must read the same - * after as before. A default or a bound that really differs changes it. - * - dropping a trigger the entities cannot carry: only the triggers ported by - * hand (POSTGRES_TRIGGERS), by exact name, and each of them must exist. + * Which of the comparator's statements change nothing, and how each kind is + * proven so, is src/db/engine/postgres/entity-schema-diff.ts: the same + * definition `npm run gen:pg-migration` uses to print the Postgres side of a + * new migration, so what the tool prints is what this test holds. */ -import { POSTGRES_TRIGGERS } from '../../src/db/engine/postgres/postgres-triggers'; -import { applyPostgresColumnAdjustments } from '../../src/db/engine/postgres/schema-adjust'; -import { ALL_ENTITIES } from '../../src/db/entities'; -import type { Transaction } from '@mikro-orm/core'; -import { MikroORM } from '@mikro-orm/postgresql'; +import { diffAgainstEntities } from '../../src/db/engine/postgres/entity-schema-diff'; import { describe, expect, it } from 'vitest'; -const PORTED = Object.keys(POSTGRES_TRIGGERS); -const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/; -const SPELLING = - /^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s; -/** MikroORM drops a trigger by its name, and the function it would have made for it as `
__fn`. */ -const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/; -const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/; - -/** Every column default and check constraint of the schema, as the catalogue renders them. */ -const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from ( - select 'default ' || c.relname || '.' || a.attname || ' = ' || pg_get_expr(d.adbin, d.adrelid) as item - from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum - where c.relnamespace = current_schema()::regnamespace - union all - select 'check ' || conrelid::regclass || '.' || conname || ' = ' || pg_get_constraintdef(oid) - from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace -) items`; - -const ROLLBACK = new Error('roll back the spelling check'); - -async function liveSets(orm: MikroORM): Promise<{ primaryKeys: Set; triggers: Set }> { - const connection = orm.em.getConnection(); - const names = (sql: string) => - connection.execute>(sql).then((rows) => new Set(rows.map((row) => row.name))); - const [primaryKeys, triggers] = await Promise.all([ - names( - `select c.relname || '.' || a.attname as name from pg_index i - join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey) - where i.indisprimary and c.relnamespace = current_schema()::regnamespace`, - ), - names( - `select c.relname || '.' || t.tgname as name from pg_trigger t join pg_class c on c.oid = t.tgrelid - where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`, - ), - ]); - return { primaryKeys, triggers }; -} - -/** Whether applying `statements` leaves every default and check constraint reading as it did; nothing is kept. */ -async function changesNothing( - orm: MikroORM, - statements: readonly string[], -): Promise<{ before: string; after: string }> { - const connection = orm.em.getConnection(); - const fingerprint = async (trx?: Transaction) => - (await connection.execute>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint; - const before = await fingerprint(); - let after = ''; - await connection - .transactional(async (trx) => { - // One after the other: a drop and the add that follows name the same constraint. - await statements.reduce>( - (previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)), - Promise.resolve(), - ); - after = await fingerprint(trx); - throw ROLLBACK; - }) - .catch((err: unknown) => { - if (err !== ROLLBACK) throw err; - }); - return { before, after }; -} - describe('PARITY-PG', () => { it('PARITY-PG-001: the bootstrapped database holds exactly the schema the entities generate for Postgres', async () => { - const orm = await MikroORM.init({ - clientUrl: process.env.TREK_DB_URL, - entities: [...ALL_ENTITIES], - discovery: { warnWhenNoEntities: false }, - }); - try { - for (const meta of orm.getMetadata().getAll().values()) { - applyPostgresColumnAdjustments(meta.props); - } - const live = await liveSets(orm); - const statements = (await orm.schema.getUpdateSchemaSQL({ wrap: false })) - .split(';\n') - .map((statement) => statement.trim().replace(/;$/, '')) - .filter((statement) => statement !== ''); - - const isKeyNotNull = (s: string) => { - const m = DROP_NOT_NULL.exec(s); - return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`); - }; - const isPortedTrigger = (s: string) => { - const trigger = DROP_TRIGGER.exec(s); - if (trigger) return PORTED.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`); - const fn = DROP_FUNCTION.exec(s); - return fn !== null && PORTED.some((name) => fn[1].endsWith(`_${name}_fn`)); - }; - const spelling = statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && SPELLING.test(s)); - - expect(statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && !SPELLING.test(s))).toEqual([]); - const { before, after } = await changesNothing(orm, spelling); - expect(after).toBe(before); - // Every hand-ported trigger is in the database, on its table. - expect(PORTED.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`)))).toEqual([]); - } finally { - await orm.close(true); - } + const url = process.env.TREK_DB_URL; + if (!url) throw new Error('PARITY-PG runs in the Postgres lane, which sets TREK_DB_URL'); + const diff = await diffAgainstEntities(url); + // A statement here is a change a migration after the cut still owes Postgres: + // `npm run gen:pg-migration` prints it for the migration's engineDdl postgres array. + expect(diff.changes).toEqual([]); + expect(diff.fingerprint.after).toBe(diff.fingerprint.before); + // Every hand-ported trigger is in the database, on its table. + expect(diff.missingTriggers).toEqual([]); }, 60_000); }); diff --git a/server/tests/unit/db/engine/postgres/entity-schema-diff.test.ts b/server/tests/unit/db/engine/postgres/entity-schema-diff.test.ts new file mode 100644 index 000000000..1960fb37f --- /dev/null +++ b/server/tests/unit/db/engine/postgres/entity-schema-diff.test.ts @@ -0,0 +1,124 @@ +/** + * Which of the comparator's statements change nothing + * (src/db/engine/postgres/entity-schema-diff.ts), the definition PARITY-PG + * and `npm run gen:pg-migration` share. The live database half runs in the + * Postgres lane; this pins the sorting and the rolled-back spelling check. + */ +import { + classifySchemaDiff, + missingPortedTriggers, + remainingStatements, + spellingChangesNothing, + splitSchemaDiff, + type EntitySchemaDiff, + type LiveSchemaFacts, +} from '../../../../../src/db/engine/postgres/entity-schema-diff'; +import type { Transaction } from '@mikro-orm/core'; +import type { AbstractSqlConnection } from '@mikro-orm/sql'; + +import { describe, expect, it, vi } from 'vitest'; + +const live: LiveSchemaFacts = { + primaryKeys: new Set(['trips.id']), + triggers: new Set(['places.trg_place_regions_follow_place']), +}; +const PORTED = ['trg_place_regions_follow_place', 'trg_release_stop_on_stay_delete']; + +describe('entity schema diff', () => { + it('SCHEMA-DIFF-001: splits the comparator output into statements without semicolons or blanks', () => { + expect(splitSchemaDiff('alter table "a" add column "b" text null;\n\ncreate index "x" on "a" ("b");\n')).toEqual([ + 'alter table "a" add column "b" text null', + 'create index "x" on "a" ("b")', + ]); + }); + + it('SCHEMA-DIFF-002: a drop not null counts as nothing only on a live primary key', () => { + const diff = classifySchemaDiff( + ['alter table "trips" alter column "id" drop not null', 'alter table "trips" alter column "title" drop not null'], + live, + PORTED, + ); + expect(diff.keyNotNull).toEqual(['alter table "trips" alter column "id" drop not null']); + expect(diff.changes).toEqual(['alter table "trips" alter column "title" drop not null']); + }); + + it('SCHEMA-DIFF-003: dropping a trigger counts as nothing only for a ported one that is in the database', () => { + const diff = classifySchemaDiff( + [ + 'drop trigger if exists "trg_place_regions_follow_place" on "places"', + 'drop function if exists "places_trg_place_regions_follow_place_fn"()', + // Ported, but not in the database on that table. + 'drop trigger if exists "trg_release_stop_on_stay_delete" on "day_accommodations"', + 'drop trigger if exists "trg_unknown" on "places"', + 'drop function if exists "places_trg_unknown_fn"()', + ], + live, + PORTED, + ); + expect(diff.portedTriggers).toHaveLength(2); + expect(diff.changes).toEqual([ + 'drop trigger if exists "trg_release_stop_on_stay_delete" on "day_accommodations"', + 'drop trigger if exists "trg_unknown" on "places"', + 'drop function if exists "places_trg_unknown_fn"()', + ]); + expect(missingPortedTriggers(live, PORTED)).toEqual(['trg_release_stop_on_stay_delete']); + }); + + it('SCHEMA-DIFF-004: defaults and checks wait for the spelling proof, everything else is a change', () => { + const diff = classifySchemaDiff( + [ + 'alter table "trips" alter column "created_at" set default to_char(now())', + 'alter table "trips" drop constraint "trips_status_check"', + 'alter table "trips" add constraint "trips_status_check" check ("status" in (\'a\'))', + 'alter table "trips" add column "note" text null', + ], + live, + PORTED, + ); + expect(diff.spelling).toHaveLength(3); + expect(diff.changes).toEqual(['alter table "trips" add column "note" text null']); + }); + + it('SCHEMA-DIFF-005: the spelling statements run in order inside a transaction that is always rolled back', async () => { + const fingerprints = ['default a = 1', 'default a = 1']; + const executed: string[] = []; + const trx = {} as Transaction; + const connection = { + execute: vi.fn(async (sql: string, _params?: unknown[], _method?: string, ctx?: Transaction) => { + if (sql.includes('string_agg')) return [{ fingerprint: fingerprints.shift() }]; + expect(ctx).toBe(trx); + executed.push(sql); + return []; + }), + transactional: vi.fn(async (cb: (t: Transaction) => Promise) => cb(trx)), + } as unknown as AbstractSqlConnection; + const result = await spellingChangesNothing(connection, ['first', 'second']); + expect(executed).toEqual(['first', 'second']); + expect(result).toEqual({ before: 'default a = 1', after: 'default a = 1' }); + }); + + it('SCHEMA-DIFF-006: a statement that fails inside the proof is not swallowed', async () => { + const connection = { + execute: vi.fn(async (sql: string) => { + if (sql.includes('string_agg')) return [{ fingerprint: 'x' }]; + throw new Error('syntax error'); + }), + transactional: vi.fn(async (cb: (t: Transaction) => Promise) => cb({} as Transaction)), + } as unknown as AbstractSqlConnection; + await expect(spellingChangesNothing(connection, ['broken'])).rejects.toThrow('syntax error'); + }); + + it('SCHEMA-DIFF-007: what a migration still owes includes the spelling statements only when they change something', () => { + const base: EntitySchemaDiff = { + changes: ['alter table "a" add column "b" text null'], + spelling: ['alter table "a" alter column "c" set default 1'], + keyNotNull: [], + portedTriggers: [], + spellingHolds: true, + fingerprint: { before: '', after: '' }, + missingTriggers: [], + }; + expect(remainingStatements(base)).toEqual(base.changes); + expect(remainingStatements({ ...base, spellingHolds: false })).toEqual([...base.changes, ...base.spelling]); + }); +});