refactor(db): move PARITY-PG's schema comparison into the Postgres engine

Which comparator statements change nothing (drop not null on a live primary
key, spelling-only defaults and checks proven by a rolled-back fingerprint, the
hand-ported triggers) is now one module the parity test and the migration
generator share, with unit tests for the sorting and the rollback.
This commit is contained in:
Maurice
2026-11-17 11:48:17 +01:00
parent a99cd40f4d
commit 0f338d5484
3 changed files with 329 additions and 116 deletions
@@ -0,0 +1,191 @@
/**
* What still separates a live Postgres database from the schema the entities
* describe for Postgres (`schema-adjust.ts`): the one definition PARITY-PG
* (`tests/postgres/schema-parity.test.ts`) holds at nothing and
* `npm run gen:pg-migration` prints as the Postgres side of a new migration.
*
* MikroORM's comparator also reports statements that change nothing, and each
* kind is proven so rather than recognised by its text alone:
* - `drop not null` on a primary key column: SQLite lets a key column be null
* and the entities say so; Postgres keeps every primary key NOT NULL (and
* refuses the statement), checked against the live keys.
* - a column default or a check constraint Postgres stores in its own
* spelling (casts added, EXTRACT rewritten, BETWEEN as two comparisons):
* such statements are applied in a transaction that is rolled back, and the
* catalogue's own rendering of every default and check must read the same
* after as before. A default or a bound that really differs changes it.
* - dropping a trigger the entities cannot carry: only the triggers ported by
* hand (`POSTGRES_TRIGGERS`), by exact name, and only while the trigger is in
* the database on its table.
*/
import { ALL_ENTITIES } from '../../entities';
import { POSTGRES_TRIGGERS } from './postgres-triggers';
import { applyPostgresColumnAdjustments } from './schema-adjust';
import type { Transaction } from '@mikro-orm/core';
import { MikroORM } from '@mikro-orm/postgresql';
import type { AbstractSqlConnection } from '@mikro-orm/sql';
const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/;
const SPELLING =
/^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s;
/** MikroORM drops a trigger by its name, and the function it would have made for it as `<table>_<trigger>_fn`. */
const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/;
const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/;
/** Every column default and check constraint of the schema, as the catalogue renders them. */
const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from (
select concat('default ', c.relname, '.', a.attname, ' = ', pg_get_expr(d.adbin, d.adrelid)) as item
from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum
where c.relnamespace = current_schema()::regnamespace
union all
select concat('check ', conrelid::regclass, '.', conname, ' = ', pg_get_constraintdef(oid))
from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace
) items`;
const PRIMARY_KEYS_SQL = `select concat(c.relname, '.', a.attname) as name from pg_index i
join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey)
where i.indisprimary and c.relnamespace = current_schema()::regnamespace`;
const TRIGGERS_SQL = `select concat(c.relname, '.', t.tgname) as name from pg_trigger t join pg_class c on c.oid = t.tgrelid
where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`;
/** The facts of the live schema the classification needs, each as `<table>.<name>`. */
export interface LiveSchemaFacts {
primaryKeys: ReadonlySet<string>;
triggers: ReadonlySet<string>;
}
/** The comparator's statements sorted by what they would do. */
interface ClassifiedDiff {
/** Statements that change the schema: what a migration has to send. */
changes: string[];
/** Default and check statements that may only be Postgres's own spelling; `spellingChangesNothing` decides. */
spelling: string[];
/** `drop not null` on a live primary key, which Postgres keeps NOT NULL anyway. */
keyNotNull: string[];
/** Dropping a hand-ported trigger or its function. */
portedTriggers: string[];
}
/** The comparator's SQL (`getUpdateSchemaSQL({ wrap: false })`) as single statements without their semicolons. */
export function splitSchemaDiff(sql: string): string[] {
return sql
.split(';\n')
.map((statement) => statement.trim().replace(/;$/, ''))
.filter((statement) => statement !== '');
}
/** Sorts the comparator's statements; `ported` names the triggers `POSTGRES_TRIGGERS` carries. */
export function classifySchemaDiff(
statements: readonly string[],
live: LiveSchemaFacts,
ported: readonly string[] = Object.keys(POSTGRES_TRIGGERS),
): ClassifiedDiff {
const isKeyNotNull = (s: string) => {
const m = DROP_NOT_NULL.exec(s);
return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`);
};
const isPortedTrigger = (s: string) => {
const trigger = DROP_TRIGGER.exec(s);
if (trigger) return ported.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`);
const fn = DROP_FUNCTION.exec(s);
return fn !== null && ported.some((name) => fn[1].endsWith(`_${name}_fn`));
};
const diff: ClassifiedDiff = { changes: [], spelling: [], keyNotNull: [], portedTriggers: [] };
for (const statement of statements) {
if (isKeyNotNull(statement)) diff.keyNotNull.push(statement);
else if (isPortedTrigger(statement)) diff.portedTriggers.push(statement);
else if (SPELLING.test(statement)) diff.spelling.push(statement);
else diff.changes.push(statement);
}
return diff;
}
/** The hand-ported triggers that are not in the live database. */
export function missingPortedTriggers(
live: LiveSchemaFacts,
ported: readonly string[] = Object.keys(POSTGRES_TRIGGERS),
): string[] {
return ported.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`)));
}
/** Reads the live primary keys and triggers of the current schema. */
async function readLiveSchemaFacts(connection: AbstractSqlConnection): Promise<LiveSchemaFacts> {
const names = (sql: string) =>
connection.execute<Array<{ name: string }>>(sql).then((rows) => new Set(rows.map((row) => row.name)));
const [primaryKeys, triggers] = await Promise.all([names(PRIMARY_KEYS_SQL), names(TRIGGERS_SQL)]);
return { primaryKeys, triggers };
}
const ROLLBACK = new Error('roll back the spelling check');
/**
* The catalogue's rendering of every default and check before and after
* `statements`, applied in a transaction that is always rolled back: equal
* means they change nothing.
*/
export async function spellingChangesNothing(
connection: AbstractSqlConnection,
statements: readonly string[],
): Promise<{ before: string; after: string }> {
const fingerprint = async (trx?: Transaction) =>
(await connection.execute<Array<{ fingerprint: string }>>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint;
const before = await fingerprint();
let after = '';
await connection
.transactional(async (trx) => {
// One after the other: a drop and the add that follows name the same constraint.
await statements.reduce<Promise<unknown>>(
(previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)),
Promise.resolve(),
);
after = await fingerprint(trx);
throw ROLLBACK;
})
.catch((err: unknown) => {
if (err !== ROLLBACK) throw err;
});
return { before, after };
}
/** Everything the comparison of one database with the entities found. */
export interface EntitySchemaDiff extends ClassifiedDiff {
/** Whether the `spelling` statements leave every default and check as it was. */
spellingHolds: boolean;
/** The catalogue's defaults and checks before and after the spelling statements, for a failure message. */
fingerprint: { before: string; after: string };
/** Hand-ported triggers missing from the database. */
missingTriggers: string[];
}
/** The statements a migration must still send: the changes, and the spelling ones when they are more than spelling. */
export function remainingStatements(diff: EntitySchemaDiff): string[] {
return diff.spellingHolds ? [...diff.changes] : [...diff.changes, ...diff.spelling];
}
/**
* Compares the database `url` names with the entities, read for Postgres the
* way the baseline writes them. Nothing in the database changes.
*/
export async function diffAgainstEntities(url: string): Promise<EntitySchemaDiff> {
const orm = await MikroORM.init({
clientUrl: url,
entities: [...ALL_ENTITIES],
discovery: { warnWhenNoEntities: false },
});
try {
for (const meta of orm.getMetadata().getAll().values()) applyPostgresColumnAdjustments(meta.props);
const connection = orm.em.getConnection();
const live = await readLiveSchemaFacts(connection);
const classified = classifySchemaDiff(splitSchemaDiff(await orm.schema.getUpdateSchemaSQL({ wrap: false })), live);
const fingerprint = await spellingChangesNothing(connection, classified.spelling);
return {
...classified,
spellingHolds: fingerprint.before === fingerprint.after,
fingerprint,
missingTriggers: missingPortedTriggers(live),
};
} finally {
await orm.close(true);
}
}
+14 -116
View File
@@ -7,127 +7,25 @@
* the cut that left Postgres behind SQLite, or a baseline that drifted from
* the entities, fails here. It runs on the lane's database, never skipped.
*
* The comparator also reports statements that change nothing, and each kind
* is proven so rather than recognised by its text:
* - `drop not null` on a primary key column: SQLite lets a key column be null
* and the entities say so; Postgres keeps every primary key NOT NULL (and
* refuses the statement), checked against the live keys.
* - a column default or a check constraint Postgres stores in its own spelling
* (casts added, EXTRACT rewritten, BETWEEN as two comparisons): such
* statements are applied in a transaction that is rolled back, and the
* catalogue's own rendering of every default and check must read the same
* after as before. A default or a bound that really differs changes it.
* - dropping a trigger the entities cannot carry: only the triggers ported by
* hand (POSTGRES_TRIGGERS), by exact name, and each of them must exist.
* Which of the comparator's statements change nothing, and how each kind is
* proven so, is src/db/engine/postgres/entity-schema-diff.ts: the same
* definition `npm run gen:pg-migration` uses to print the Postgres side of a
* new migration, so what the tool prints is what this test holds.
*/
import { POSTGRES_TRIGGERS } from '../../src/db/engine/postgres/postgres-triggers';
import { applyPostgresColumnAdjustments } from '../../src/db/engine/postgres/schema-adjust';
import { ALL_ENTITIES } from '../../src/db/entities';
import type { Transaction } from '@mikro-orm/core';
import { MikroORM } from '@mikro-orm/postgresql';
import { diffAgainstEntities } from '../../src/db/engine/postgres/entity-schema-diff';
import { describe, expect, it } from 'vitest';
const PORTED = Object.keys(POSTGRES_TRIGGERS);
const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/;
const SPELLING =
/^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s;
/** MikroORM drops a trigger by its name, and the function it would have made for it as `<table>_<trigger>_fn`. */
const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/;
const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/;
/** Every column default and check constraint of the schema, as the catalogue renders them. */
const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from (
select 'default ' || c.relname || '.' || a.attname || ' = ' || pg_get_expr(d.adbin, d.adrelid) as item
from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum
where c.relnamespace = current_schema()::regnamespace
union all
select 'check ' || conrelid::regclass || '.' || conname || ' = ' || pg_get_constraintdef(oid)
from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace
) items`;
const ROLLBACK = new Error('roll back the spelling check');
async function liveSets(orm: MikroORM): Promise<{ primaryKeys: Set<string>; triggers: Set<string> }> {
const connection = orm.em.getConnection();
const names = (sql: string) =>
connection.execute<Array<{ name: string }>>(sql).then((rows) => new Set(rows.map((row) => row.name)));
const [primaryKeys, triggers] = await Promise.all([
names(
`select c.relname || '.' || a.attname as name from pg_index i
join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey)
where i.indisprimary and c.relnamespace = current_schema()::regnamespace`,
),
names(
`select c.relname || '.' || t.tgname as name from pg_trigger t join pg_class c on c.oid = t.tgrelid
where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`,
),
]);
return { primaryKeys, triggers };
}
/** Whether applying `statements` leaves every default and check constraint reading as it did; nothing is kept. */
async function changesNothing(
orm: MikroORM,
statements: readonly string[],
): Promise<{ before: string; after: string }> {
const connection = orm.em.getConnection();
const fingerprint = async (trx?: Transaction) =>
(await connection.execute<Array<{ fingerprint: string }>>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint;
const before = await fingerprint();
let after = '';
await connection
.transactional(async (trx) => {
// One after the other: a drop and the add that follows name the same constraint.
await statements.reduce<Promise<unknown>>(
(previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)),
Promise.resolve(),
);
after = await fingerprint(trx);
throw ROLLBACK;
})
.catch((err: unknown) => {
if (err !== ROLLBACK) throw err;
});
return { before, after };
}
describe('PARITY-PG', () => {
it('PARITY-PG-001: the bootstrapped database holds exactly the schema the entities generate for Postgres', async () => {
const orm = await MikroORM.init({
clientUrl: process.env.TREK_DB_URL,
entities: [...ALL_ENTITIES],
discovery: { warnWhenNoEntities: false },
});
try {
for (const meta of orm.getMetadata().getAll().values()) {
applyPostgresColumnAdjustments(meta.props);
}
const live = await liveSets(orm);
const statements = (await orm.schema.getUpdateSchemaSQL({ wrap: false }))
.split(';\n')
.map((statement) => statement.trim().replace(/;$/, ''))
.filter((statement) => statement !== '');
const isKeyNotNull = (s: string) => {
const m = DROP_NOT_NULL.exec(s);
return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`);
};
const isPortedTrigger = (s: string) => {
const trigger = DROP_TRIGGER.exec(s);
if (trigger) return PORTED.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`);
const fn = DROP_FUNCTION.exec(s);
return fn !== null && PORTED.some((name) => fn[1].endsWith(`_${name}_fn`));
};
const spelling = statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && SPELLING.test(s));
expect(statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && !SPELLING.test(s))).toEqual([]);
const { before, after } = await changesNothing(orm, spelling);
expect(after).toBe(before);
// Every hand-ported trigger is in the database, on its table.
expect(PORTED.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`)))).toEqual([]);
} finally {
await orm.close(true);
}
const url = process.env.TREK_DB_URL;
if (!url) throw new Error('PARITY-PG runs in the Postgres lane, which sets TREK_DB_URL');
const diff = await diffAgainstEntities(url);
// A statement here is a change a migration after the cut still owes Postgres:
// `npm run gen:pg-migration` prints it for the migration's engineDdl postgres array.
expect(diff.changes).toEqual([]);
expect(diff.fingerprint.after).toBe(diff.fingerprint.before);
// Every hand-ported trigger is in the database, on its table.
expect(diff.missingTriggers).toEqual([]);
}, 60_000);
});
@@ -0,0 +1,124 @@
/**
* Which of the comparator's statements change nothing
* (src/db/engine/postgres/entity-schema-diff.ts), the definition PARITY-PG
* and `npm run gen:pg-migration` share. The live database half runs in the
* Postgres lane; this pins the sorting and the rolled-back spelling check.
*/
import {
classifySchemaDiff,
missingPortedTriggers,
remainingStatements,
spellingChangesNothing,
splitSchemaDiff,
type EntitySchemaDiff,
type LiveSchemaFacts,
} from '../../../../../src/db/engine/postgres/entity-schema-diff';
import type { Transaction } from '@mikro-orm/core';
import type { AbstractSqlConnection } from '@mikro-orm/sql';
import { describe, expect, it, vi } from 'vitest';
const live: LiveSchemaFacts = {
primaryKeys: new Set(['trips.id']),
triggers: new Set(['places.trg_place_regions_follow_place']),
};
const PORTED = ['trg_place_regions_follow_place', 'trg_release_stop_on_stay_delete'];
describe('entity schema diff', () => {
it('SCHEMA-DIFF-001: splits the comparator output into statements without semicolons or blanks', () => {
expect(splitSchemaDiff('alter table "a" add column "b" text null;\n\ncreate index "x" on "a" ("b");\n')).toEqual([
'alter table "a" add column "b" text null',
'create index "x" on "a" ("b")',
]);
});
it('SCHEMA-DIFF-002: a drop not null counts as nothing only on a live primary key', () => {
const diff = classifySchemaDiff(
['alter table "trips" alter column "id" drop not null', 'alter table "trips" alter column "title" drop not null'],
live,
PORTED,
);
expect(diff.keyNotNull).toEqual(['alter table "trips" alter column "id" drop not null']);
expect(diff.changes).toEqual(['alter table "trips" alter column "title" drop not null']);
});
it('SCHEMA-DIFF-003: dropping a trigger counts as nothing only for a ported one that is in the database', () => {
const diff = classifySchemaDiff(
[
'drop trigger if exists "trg_place_regions_follow_place" on "places"',
'drop function if exists "places_trg_place_regions_follow_place_fn"()',
// Ported, but not in the database on that table.
'drop trigger if exists "trg_release_stop_on_stay_delete" on "day_accommodations"',
'drop trigger if exists "trg_unknown" on "places"',
'drop function if exists "places_trg_unknown_fn"()',
],
live,
PORTED,
);
expect(diff.portedTriggers).toHaveLength(2);
expect(diff.changes).toEqual([
'drop trigger if exists "trg_release_stop_on_stay_delete" on "day_accommodations"',
'drop trigger if exists "trg_unknown" on "places"',
'drop function if exists "places_trg_unknown_fn"()',
]);
expect(missingPortedTriggers(live, PORTED)).toEqual(['trg_release_stop_on_stay_delete']);
});
it('SCHEMA-DIFF-004: defaults and checks wait for the spelling proof, everything else is a change', () => {
const diff = classifySchemaDiff(
[
'alter table "trips" alter column "created_at" set default to_char(now())',
'alter table "trips" drop constraint "trips_status_check"',
'alter table "trips" add constraint "trips_status_check" check ("status" in (\'a\'))',
'alter table "trips" add column "note" text null',
],
live,
PORTED,
);
expect(diff.spelling).toHaveLength(3);
expect(diff.changes).toEqual(['alter table "trips" add column "note" text null']);
});
it('SCHEMA-DIFF-005: the spelling statements run in order inside a transaction that is always rolled back', async () => {
const fingerprints = ['default a = 1', 'default a = 1'];
const executed: string[] = [];
const trx = {} as Transaction;
const connection = {
execute: vi.fn(async (sql: string, _params?: unknown[], _method?: string, ctx?: Transaction) => {
if (sql.includes('string_agg')) return [{ fingerprint: fingerprints.shift() }];
expect(ctx).toBe(trx);
executed.push(sql);
return [];
}),
transactional: vi.fn(async (cb: (t: Transaction) => Promise<unknown>) => cb(trx)),
} as unknown as AbstractSqlConnection;
const result = await spellingChangesNothing(connection, ['first', 'second']);
expect(executed).toEqual(['first', 'second']);
expect(result).toEqual({ before: 'default a = 1', after: 'default a = 1' });
});
it('SCHEMA-DIFF-006: a statement that fails inside the proof is not swallowed', async () => {
const connection = {
execute: vi.fn(async (sql: string) => {
if (sql.includes('string_agg')) return [{ fingerprint: 'x' }];
throw new Error('syntax error');
}),
transactional: vi.fn(async (cb: (t: Transaction) => Promise<unknown>) => cb({} as Transaction)),
} as unknown as AbstractSqlConnection;
await expect(spellingChangesNothing(connection, ['broken'])).rejects.toThrow('syntax error');
});
it('SCHEMA-DIFF-007: what a migration still owes includes the spelling statements only when they change something', () => {
const base: EntitySchemaDiff = {
changes: ['alter table "a" add column "b" text null'],
spelling: ['alter table "a" alter column "c" set default 1'],
keyNotNull: [],
portedTriggers: [],
spellingHolds: true,
fingerprint: { before: '', after: '' },
missingTriggers: [],
};
expect(remainingStatements(base)).toEqual(base.changes);
expect(remainingStatements({ ...base, spellingHolds: false })).toEqual([...base.changes, ...base.spelling]);
});
});