From 830cd6e3b55085d1c7dbe3c66525538f7b8037d2 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 11:48:17 +0100 Subject: [PATCH] feat(db): print the Postgres side of a new migration with gen:pg-migration The script builds a scratch database the way a boot does (the baseline, then the migrations after its cut) up to the target migration, compares it with the entities by PARITY-PG's rules and prints the remaining statements as the postgres member of engineDdl. The scratch database is always dropped. --- server/package.json | 1 + server/scripts/generate-pg-migration.ts | 210 ++++++++++++++++++ .../scripts/generate-pg-migration.test.ts | 59 +++++ server/tsconfig.scripts.json | 2 +- 4 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 server/scripts/generate-pg-migration.ts create mode 100644 server/tests/unit/scripts/generate-pg-migration.test.ts diff --git a/server/package.json b/server/package.json index 8e77040e6..5fbf7a76c 100644 --- a/server/package.json +++ b/server/package.json @@ -39,6 +39,7 @@ "check:db-types": "node --import tsx scripts/generate-db-types.ts --check", "gen:pg-baseline": "node --import tsx scripts/generate-pg-baseline.ts", "check:pg-baseline": "node --import tsx scripts/generate-pg-baseline.ts --check", + "gen:pg-migration": "node --import tsx scripts/generate-pg-migration.ts", "probe:pg": "node --import tsx scripts/pg-probe.ts", "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"", "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\"", diff --git a/server/scripts/generate-pg-migration.ts b/server/scripts/generate-pg-migration.ts new file mode 100644 index 000000000..4514a2262 --- /dev/null +++ b/server/scripts/generate-pg-migration.ts @@ -0,0 +1,210 @@ +/** + * Prints the Postgres side of a new migration: the `postgres` array of its + * `engineDdl(this, { sqlite: [...], postgres: [...] })` call + * (src/db/migration-utils.ts). + * + * A migration after the Postgres baseline's cut runs on both engines, and the + * Postgres result has to be exactly what MikroORM generates from the entities + * (PARITY-PG). So the Postgres DDL is not written by hand: this script builds + * a scratch database the way a boot builds an empty one (the baseline, then + * every migration after its cut) but stops before the target migration, asks + * MikroORM's comparator what still separates it from the entities, drops what + * changes nothing by the rules PARITY-PG uses + * (src/db/engine/postgres/entity-schema-diff.ts), and prints the rest. + * + * The routine: change the entities, write the migration's SQLite statements + * as `engineDdl(this, { sqlite: [...], postgres: [] })`, run this, paste what + * it prints as the `postgres` array. Nothing is written to the repository. + * + * It needs a Postgres server where it may create and drop a database: + * TREK_TEST_DB_URL or --url. The scratch database is dropped again however + * the run ends. + * + * Usage: + * npm run gen:pg-migration # the newest migration + * npm run gen:pg-migration -- --migration= # another one + * npm run gen:pg-migration -- --url=postgres://user:pw@host:5432/postgres + */ +import { buildOrmConfig } from '../src/db/engine'; +import { readPostgresBaseline } from '../src/db/engine/postgres/baseline'; +import { diffAgainstEntities, remainingStatements } from '../src/db/engine/postgres/entity-schema-diff'; +import { PostgresEngine } from '../src/db/engine/postgres/postgres-engine'; +import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps } from '../src/db/engine/postgres/schema-bootstrap'; +import { knownMigrationNames } from '../src/db/known-migrations'; +import { NO_SAFETY_NET } from '../src/db/legacy-baseline'; +import { MikroORM, type MigrationObject } from '@mikro-orm/core'; +import { MikroORM as PostgresMikroORM } from '@mikro-orm/postgresql'; + +import { randomBytes } from 'node:crypto'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const MIGRATIONS_DIR = path.join(__dirname, '..', 'src', 'db', 'migrations'); +const TAG = '[gen:pg-migration]'; + +/** The scratch databases this script creates: a prefix nothing else uses. */ +export const SCRATCH_PREFIX = 'trek_gen_pg_migration_'; + +/** The migration the run is for: `--migration` (a full class name), else the newest one. */ +export function pickTarget(known: readonly string[], requested: string | undefined): string { + const sorted = [...known].sort(); + if (requested === undefined) { + const newest = sorted.at(-1); + if (!newest) throw new Error('no migrations found'); + return newest; + } + if (!sorted.includes(requested)) { + throw new Error( + `--migration=${requested} names no migration in src/db/migrations; pass the class name, e.g. ${sorted.at(-1)}`, + ); + } + return requested; +} + +/** The migrations a boot runs on Postgres after the baseline's cut and before `target`, in order. */ +export function postCutBefore(known: readonly string[], through: string, target: string): string[] { + return [...known].filter((name) => name > through && name < target).sort(); +} + +/** + * One statement as a TypeScript string literal, in the quote that needs fewer + * escapes (Prettier's choice, so the pasted array passes lint:format as it is). + */ +export function quote(statement: string): string { + const mark = statement.split("'").length > statement.split('"').length ? '"' : "'"; + const escaped = statement.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').split(mark).join(`\\${mark}`); + return `${mark}${escaped}${mark}`; +} + +/** The statements as the `postgres` member of an engineDdl call, ready to paste. */ +export function renderPostgresArray(statements: readonly string[]): string { + if (statements.length === 0) return 'postgres: [],'; + return ['postgres: [', ...statements.map((statement) => ` ${quote(statement)},`), '],'].join('\n'); +} + +/** `adminUrl` pointed at `database`, everything else (credentials, host, sslmode) kept. */ +export function databaseUrl(adminUrl: string, database: string): string { + const url = new URL(adminUrl); + url.pathname = `/${database}`; + return url.toString(); +} + +/** Runs one statement on the admin database, outside any transaction (CREATE and DROP DATABASE refuse one). */ +async function onAdmin(adminUrl: string, sql: string): Promise { + const orm = await PostgresMikroORM.init({ + clientUrl: adminUrl, + entities: [], + discovery: { warnWhenNoEntities: false }, + pool: { min: 0, max: 1 }, + }); + try { + await orm.em.getConnection().execute(sql, [], 'run'); + } finally { + await orm.close(true); + } +} + +/** The migration classes a boot would run, loaded from their files. */ +async function loadMigrations(names: readonly string[]): Promise { + return Promise.all( + names.map(async (name) => { + const module = (await import(pathToFileURL(path.join(MIGRATIONS_DIR, `${name}.ts`)).href)) as Record< + string, + unknown + >; + const migrationClass = module[name]; + if (typeof migrationClass !== 'function') throw new Error(`${name}.ts does not export the class ${name}`); + return { name, class: migrationClass as MigrationObject['class'] }; + }), + ); +} + +/** + * Brings the empty database `url` names to the schema a boot leaves before + * `target`: the engine's own bootstrap (baseline, then the migrations after + * its cut), offered only the migrations in `before`, with no pre-migrate copy. + */ +async function bootstrapBefore(url: string, before: readonly string[]): Promise { + const config = buildOrmConfig(new PostgresEngine({ engine: 'postgres', url, poolMax: 2 })); + const orm = await MikroORM.init({ + ...config, + // As tests/postgres/build-template.ts: the bootstrap runs SQL, which needs no entity metadata. + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + migrations: { ...config.migrations, silent: true, migrationsList: await loadMigrations(before) }, + }); + try { + await bootstrapPostgresSchema(orm, { ...defaultPostgresBootstrapDeps(url), safetyNet: NO_SAFETY_NET }); + } finally { + await orm.close(true); + } +} + +function argument(argv: readonly string[], name: string): string | undefined { + const prefix = `--${name}=`; + return argv.find((arg) => arg.startsWith(prefix))?.slice(prefix.length); +} + +async function main(argv: readonly string[]): Promise { + const adminUrl = argument(argv, 'url') ?? process.env.TREK_TEST_DB_URL; + if (!adminUrl) { + throw new Error( + 'needs a Postgres server where it may create and drop a scratch database: set TREK_TEST_DB_URL or pass ' + + '--url=postgres://user:password@host:5432/postgres (locally: docker run --rm -p 5432:5432 ' + + '-e POSTGRES_PASSWORD=trek postgres:17).', + ); + } + const known = [...knownMigrationNames()].sort(); + const target = pickTarget(known, argument(argv, 'migration')); + const { through } = readPostgresBaseline(); + const before = postCutBefore(known, through, target); + const scratch = `${SCRATCH_PREFIX}${randomBytes(6).toString('hex')}`; + const scratchUrl = databaseUrl(adminUrl, scratch); + + console.error(`${TAG} target ${target}; the Postgres baseline stands for everything through ${through}`); + if (target <= through) { + console.error(`${TAG} ${target} is at or before the cut: it never runs on Postgres. Comparing the baseline alone.`); + } else { + console.error(`${TAG} building ${scratch}: the baseline, then ${before.length} migration(s) after the cut`); + } + + await onAdmin(adminUrl, `create database "${scratch}"`); + try { + await bootstrapBefore(scratchUrl, before); + const diff = await diffAgainstEntities(scratchUrl); + const statements = remainingStatements(diff); + if (!diff.spellingHolds) { + console.error( + `${TAG} some default or check statements change the catalogue, so they are part of the change and printed below.`, + ); + } + if (statements.some((statement) => statement.includes('?'))) { + console.error(`${TAG} a statement holds a "?", which the migrator would read as a parameter: rewrite it.`); + } + if (statements.length === 0) { + console.error( + target <= through + ? `${TAG} nothing to do: the baseline matches the entities.` + : `${TAG} nothing to do: Postgres already matches the entities without ${target}; ` + + 'its engineDdl postgres array stays empty.', + ); + } else if (target <= through) { + console.error( + `${TAG} the baseline differs from the entities by the statements below; npm run check:pg-baseline says why.`, + ); + } else { + console.error(`${TAG} paste this as the postgres array of the engineDdl call in ${target}:`); + } + console.log(renderPostgresArray(statements)); + } finally { + await onAdmin(adminUrl, `drop database if exists "${scratch}" with (force)`); + } +} + +if (require.main === module) { + main(process.argv.slice(2)).catch((error: unknown) => { + console.error(`${TAG} ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/server/tests/unit/scripts/generate-pg-migration.test.ts b/server/tests/unit/scripts/generate-pg-migration.test.ts new file mode 100644 index 000000000..92fa9c492 --- /dev/null +++ b/server/tests/unit/scripts/generate-pg-migration.test.ts @@ -0,0 +1,59 @@ +/** + * scripts/generate-pg-migration.ts prints the Postgres side of a new + * migration. Its database half runs against a real server (the author's, or + * the Postgres lane's); this pins the parts that decide what it builds and + * how it prints: which migration it is for, which migrations the scratch + * database runs, and a paste-ready array. + */ +import { + databaseUrl, + pickTarget, + postCutBefore, + quote, + renderPostgresArray, +} from '../../../scripts/generate-pg-migration'; + +import { describe, expect, it } from 'vitest'; + +const A = 'Migration20200101000100_a'; +const B = 'Migration20200101000200_b'; +const C = 'Migration20200101000300_c'; +const D = 'Migration20200101000400_d'; + +describe('generate-pg-migration', () => { + it('GENPGMIG-001: the target is the newest migration unless one is named, and an unknown name is refused', () => { + expect(pickTarget([C, A, B], undefined)).toBe(C); + expect(pickTarget([A, B, C], B)).toBe(B); + expect(() => pickTarget([A, B, C], 'Migration20200101000300')).toThrow(/names no migration/); + expect(() => pickTarget([], undefined)).toThrow(/no migrations/); + }); + + it('GENPGMIG-002: the scratch database runs the migrations after the cut and before the target, in order', () => { + expect(postCutBefore([D, A, C, B], A, D)).toEqual([B, C]); + expect(postCutBefore([A, B, C, D], A, B)).toEqual([]); + // A target at or before the cut: the baseline alone. + expect(postCutBefore([A, B, C, D], C, B)).toEqual([]); + }); + + it('GENPGMIG-003: statements print as literals in the quote that needs fewer escapes', () => { + expect(quote('create index "x" on "t" ("a")')).toBe('\'create index "x" on "t" ("a")\''); + expect(quote("alter table t add c text default 'a'")).toBe('"alter table t add c text default \'a\'"'); + expect(quote('alter table "t" add "c" text default \'a\'')).toBe( + '\'alter table "t" add "c" text default \\\'a\\\'\'', + ); + expect(quote('a\\b\nc')).toBe("'a\\\\b\\nc'"); + }); + + it('GENPGMIG-004: the array is ready to paste, and empty when there is nothing to do', () => { + expect(renderPostgresArray([])).toBe('postgres: [],'); + expect(renderPostgresArray(['create table "t" ("id" bigserial primary key)'])).toBe( + 'postgres: [\n \'create table "t" ("id" bigserial primary key)\',\n],', + ); + }); + + it('GENPGMIG-005: the scratch database keeps the credentials and options of the server URL', () => { + expect(databaseUrl('postgres://u:p@h:5432/postgres?sslmode=require', 'scratch')).toBe( + 'postgres://u:p@h:5432/scratch?sslmode=require', + ); + }); +}); diff --git a/server/tsconfig.scripts.json b/server/tsconfig.scripts.json index 7fd1ba59e..8538143bf 100644 --- a/server/tsconfig.scripts.json +++ b/server/tsconfig.scripts.json @@ -4,5 +4,5 @@ "rootDir": ".", "noEmit": true }, - "include": ["scripts/generate-entities.ts", "scripts/generate-db-types.ts", "scripts/generate-pg-baseline.ts", "scripts/migrate-encryption.ts", "scripts/pg-probe.ts", "scripts/pg-probe/*.ts"] + "include": ["scripts/generate-entities.ts", "scripts/generate-db-types.ts", "scripts/generate-pg-baseline.ts", "scripts/generate-pg-migration.ts", "scripts/migrate-encryption.ts", "scripts/pg-probe.ts", "scripts/pg-probe/*.ts"] }