refactor(db): split the Postgres schema adjustments by concern

schema-adjust.ts mixed the statement splitter, the trigger ports and the
column adjustments, and every caller repeated `meta.props as EntityProperty[]
as ColumnMeta[]` to reach them. Now:

- sql-split.ts holds splitStatements. Its scope is written down: `;` outside
  quotes, '/" quoted text and the $$ body, and the session-setting lines it
  drops. A tagged dollar quote ($tag$) is refused with an error instead of
  being split inside, which the splitter could not do correctly.
- postgres-triggers.ts holds the hand-written trigger ports, which the
  baseline and the parity test read by name.
- schema-adjust.ts holds the column adjustments. applyPostgresColumnAdjustments
  (renamed from adjustColumns, which hid that it mutates its input) takes an
  entity's props directly: ColumnMeta now reads customType as unknown, so
  the entity's property metadata satisfies it without a cast. Each adjustment
  is a small function that returns whether it changed the column.

The timestamp check is now `instanceof DbTimestampType` rather than a compare
on the constructor's name, which would have broken silently on a rename.
This commit is contained in:
Isaias Tavares
2026-11-05 00:32:18 -03:00
parent d498fe2bea
commit 33a18f29dd
8 changed files with 193 additions and 133 deletions
+33 -18
View File
@@ -19,19 +19,19 @@
* node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/
* node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI)
*/
import { baselineMatches, readPostgresBaseline, writePostgresBaseline, type PostgresBaseline } from '../src/db/engine/postgres/baseline';
import {
adjustColumns,
NOCASE_COLLATION_SQL,
POSTGRES_TRIGGERS,
splitStatements,
type ColumnMeta,
} from '../src/db/engine/postgres/schema-adjust';
baselineMatches,
readPostgresBaseline,
writePostgresBaseline,
type PostgresBaseline,
} from '../src/db/engine/postgres/baseline';
import { POSTGRES_TRIGGERS } from '../src/db/engine/postgres/postgres-triggers';
import { applyPostgresColumnAdjustments, NOCASE_COLLATION_SQL } from '../src/db/engine/postgres/schema-adjust';
import { splitStatements } from '../src/db/engine/postgres/sql-split';
import { ALL_ENTITIES } from '../src/db/entities';
import { knownMigrationNames } from '../src/db/known-migrations';
import type { EntityProperty } from '@mikro-orm/core';
import { MikroORM } from '@mikro-orm/postgresql';
import { migrateTempDb, tempDbPath } from './generate-entities';
import { MikroORM } from '@mikro-orm/postgresql';
import Database from 'better-sqlite3';
import fs from 'node:fs';
@@ -48,7 +48,7 @@ export async function generatePostgresBaseline(known: Set<string> = knownMigrati
discovery: { warnWhenNoEntities: false },
});
try {
for (const meta of orm.getMetadata().getAll().values()) adjustColumns(meta.props as EntityProperty[] as ColumnMeta[]);
for (const meta of orm.getMetadata().getAll().values()) applyPostgresColumnAdjustments(meta.props);
const migrated = await readMigratedSqlite();
const statements = [
NOCASE_COLLATION_SQL,
@@ -73,8 +73,11 @@ export function checkBaseline(committed: PostgresBaseline, generated: PostgresBa
const problems: string[] = [];
const missing = committed.migrations.filter((name) => !known.has(name));
if (missing.length > 0) problems.push(`the baseline names migrations that do not exist: ${missing.join(', ')}`);
const unlisted = generated.migrations.filter((name) => name <= committed.through && !committed.migrations.includes(name));
if (unlisted.length > 0) problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`);
const unlisted = generated.migrations.filter(
(name) => name <= committed.through && !committed.migrations.includes(name),
);
if (unlisted.length > 0)
problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`);
if (committed.through === generated.through && !matches) {
problems.push('the entities generate a different schema; run npm run gen:pg-baseline');
}
@@ -94,13 +97,21 @@ async function readMigratedSqlite(): Promise<{ triggers: string[]; tables: Array
const conn = new Database(dbPath, { readonly: true });
try {
const objects = conn
.prepare("select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid")
.prepare(
"select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid",
)
.all() as Array<{ type: string; name: string }>;
const tables = objects
.filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name))
.map((object) => ({ name: object.name, rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[] }))
.map((object) => ({
name: object.name,
rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[],
}))
.filter((table) => table.rows.length > 0);
const triggers = objects.filter((object) => object.type === 'trigger').map((object) => object.name).sort();
const triggers = objects
.filter((object) => object.type === 'trigger')
.map((object) => object.name)
.sort();
return { triggers, tables };
} finally {
conn.close();
@@ -129,7 +140,9 @@ export function rowStatements(table: string, rows: readonly Row[]): string[] {
if (rows.length === 0) return [];
const columns = Object.keys(rows[0]);
const values = rows.map((row) => `(${columns.map((column) => literal(row[column])).join(', ')})`);
const statements = [`insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`];
const statements = [
`insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`,
];
// A text key (addons.id) has no sequence; a numeric one may.
if (rows.every((row) => typeof row.id === 'number')) {
statements.push(
@@ -148,9 +161,11 @@ export function rowStatements(table: string, rows: readonly Row[]): string[] {
export function triggerPortProblems(sqliteTriggers: readonly string[], ported: readonly string[]): string[] {
const problems: string[] = [];
const missing = sqliteTriggers.filter((name) => !ported.includes(name));
if (missing.length > 0) problems.push(`SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}`);
if (missing.length > 0)
problems.push(`SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}`);
const stale = ported.filter((name) => !sqliteTriggers.includes(name));
if (stale.length > 0) problems.push(`POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}`);
if (stale.length > 0)
problems.push(`POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}`);
return problems;
}
+4 -3
View File
@@ -5,6 +5,8 @@
* SQLite database the helper cases are compared on, and the loop that calls
* every repository method.
*/
import { applyPostgresColumnAdjustments, NOCASE_COLLATION_SQL } from '../../src/db/engine/postgres/schema-adjust';
import { splitStatements } from '../../src/db/engine/postgres/sql-split';
import { ALL_ENTITIES } from '../../src/db/entities';
import type { MethodPlan } from './arg-samples';
import {
@@ -18,8 +20,7 @@ import {
} from './helper-cases';
import { ProbePostgresDialect, type StatementRecorder } from './recorder';
import type { MethodResult } from './report';
import { adjustColumns, NOCASE_COLLATION_SQL, splitStatements, type ColumnMeta } from '../../src/db/engine/postgres/schema-adjust';
import { RequestContext, type Configuration, type EntityManager, type EntityProperty } from '@mikro-orm/core';
import { RequestContext, type Configuration, type EntityManager } from '@mikro-orm/core';
import { MikroORM, PostgreSqlConnection, PostgreSqlDriver, type Options } from '@mikro-orm/postgresql';
import { SqlitePlatform } from '@mikro-orm/sql';
@@ -129,7 +130,7 @@ export async function prepareDatabase(
await run(NOCASE_COLLATION_SQL);
let adjusted = 0;
for (const meta of orm.getMetadata().getAll().values()) {
adjusted += adjustColumns(meta.props as EntityProperty[] as ColumnMeta[]).length;
adjusted += applyPostgresColumnAdjustments(meta.props).length;
}
const statements = splitStatements(await orm.schema.getCreateSchemaSQL({ wrap: false }));
for (const statement of statements) await run(statement);
+1 -1
View File
@@ -1,4 +1,4 @@
import { splitStatements } from './schema-adjust';
import { splitStatements } from './sql-split';
import fs from 'node:fs';
import path from 'node:path';
@@ -0,0 +1,32 @@
/**
* The SQLite schema's triggers as Postgres writes them. The entities describe
* tables only, so the baseline (`scripts/generate-pg-baseline.ts`) adds these
* after the tables, and `check:pg-baseline` fails while a trigger in the
* migrated SQLite schema has no entry here. Each entry is the trigger's
* function and the trigger that runs it, under the SQLite trigger's name.
*/
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
trg_release_stop_on_stay_delete: [
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
RETURN OLD;
END
$$`,
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
],
// A place that moved or was renamed drops the region it was resolved to (#2527).
trg_place_regions_follow_place: [
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM place_regions WHERE place_id = NEW.id;
RETURN NEW;
END
$$`,
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
'EXECUTE FUNCTION trg_place_regions_follow_place()',
],
};
+46 -101
View File
@@ -22,9 +22,11 @@
*
* The baseline keeps the foreign keys; the probe leaves them out, so a
* statement it sends is never refused only because the sample row it writes
* points at nothing.
* points at nothing. The triggers the SQLite schema has are in
* `postgres-triggers.ts`.
*/
import { pgTimestampText, PG_UTC_NOW } from '../../dialect/platform';
import { DbTimestampType } from '../../types';
/** SQLite's timestamp text on Postgres, for a column default. */
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
@@ -45,120 +47,63 @@ export function portableDefault(raw: string): string {
return raw;
}
/**
* The SQLite schema's triggers, as Postgres writes them: a trigger function
* and the trigger that runs it, under the SQLite trigger's name. The entities
* describe tables only, so the baseline adds these after the tables, and
* `check:pg-baseline` fails while a trigger in the migrated SQLite schema has
* no entry here.
*/
export const POSTGRES_TRIGGERS: Readonly<Record<string, readonly string[]>> = {
// A deleted stay releases the day stops it claimed (no foreign key holds the link).
trg_release_stop_on_stay_delete: [
`CREATE FUNCTION trg_release_stop_on_stay_delete() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
UPDATE day_assignments SET accommodation_id = NULL WHERE accommodation_id = OLD.id;
RETURN OLD;
END
$$`,
'CREATE TRIGGER trg_release_stop_on_stay_delete AFTER DELETE ON day_accommodations ' +
'FOR EACH ROW EXECUTE FUNCTION trg_release_stop_on_stay_delete()',
],
// A place that moved or was renamed drops the region it was resolved to (#2527).
trg_place_regions_follow_place: [
`CREATE FUNCTION trg_place_regions_follow_place() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
DELETE FROM place_regions WHERE place_id = NEW.id;
RETURN NEW;
END
$$`,
'CREATE TRIGGER trg_place_regions_follow_place AFTER UPDATE OF lat, lng, address ON places FOR EACH ROW ' +
'WHEN (OLD.lat IS DISTINCT FROM NEW.lat OR OLD.lng IS DISTINCT FROM NEW.lng OR OLD.address IS DISTINCT FROM NEW.address) ' +
'EXECUTE FUNCTION trg_place_regions_follow_place()',
],
};
/**
* Splits generated DDL into single statements on `;` outside quotes and
* `$$` bodies, so one statement Postgres refuses does not take the rest of the
* schema with it.
*/
export function splitStatements(sql: string): string[] {
const statements: string[] = [];
let current = '';
let quote: '"' | "'" | '$$' | null = null;
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const dollar = sql.startsWith('$$', i);
if (quote === '$$') {
if (dollar) {
quote = null;
current += '$';
i++;
}
current += char;
} else if (quote) {
if (char === quote) quote = null;
current += char;
} else if (dollar) {
quote = '$$';
current += '$$';
i++;
} else if (char === "'" || char === '"') {
quote = char;
current += char;
} else if (char === ';') {
if (current.trim()) statements.push(current.trim());
current = '';
} else {
current += char;
}
}
if (current.trim()) statements.push(current.trim());
return statements.filter((statement) => !/^set\s/i.test(statement));
}
/** A column type SQLite stores as its 64-bit INTEGER. */
const INTEGER_TYPE = /^(integer|int)$/i;
/** The subset of MikroORM's property metadata the schema adjustments touch. */
/**
* The subset of MikroORM's property metadata the adjustments read and write.
* An entity's `props` satisfy it as they are, so no cast is needed.
*/
export interface ColumnMeta {
name: string;
defaultRaw?: string;
collation?: string;
columnTypes?: string[];
customType?: { constructor: { name: string } };
customType?: unknown;
}
/** A timestamp column becomes text. Returns whether it changed. */
function timestampAsText(column: ColumnMeta): boolean {
if (!(column.customType instanceof DbTimestampType) || !column.columnTypes) return false;
column.columnTypes = column.columnTypes.map(() => 'text');
return true;
}
/** An integer column becomes bigint. Returns whether it changed. */
function integerAsBigint(column: ColumnMeta): boolean {
if (!column.columnTypes?.some((type) => INTEGER_TYPE.test(type))) return false;
column.columnTypes = column.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
return true;
}
/** A SQLite-only default gets its Postgres spelling. Returns whether it changed. */
function defaultAsPostgres(column: ColumnMeta): boolean {
if (typeof column.defaultRaw !== 'string') return false;
const next = portableDefault(column.defaultRaw);
if (next === column.defaultRaw) return false;
column.defaultRaw = next;
return true;
}
/** A NOCASE column uses the `nocase` collation. Returns whether it changed. */
function nocaseAsCollation(column: ColumnMeta): boolean {
if (column.collation?.toUpperCase() !== 'NOCASE') return false;
column.collation = 'nocase';
return true;
}
/**
* Applies the adjustments above to one entity's properties in place:
* timestamp columns become `text`, integer columns `bigint`, SQLite-only defaults and the NOCASE
* collation their Postgres spelling. Returns the names of the properties
* it changed, for the log.
* Applies the adjustments above to an entity's properties, in place, and
* returns the names of the properties it changed, for the log. A timestamp
* column is text, so the integer adjustment never sees it.
*/
export function adjustColumns(properties: readonly ColumnMeta[]): string[] {
export function applyPostgresColumnAdjustments(properties: readonly ColumnMeta[]): string[] {
const changed: string[] = [];
for (const property of properties) {
let touched = false;
if (property.customType?.constructor.name === 'DbTimestampType' && property.columnTypes) {
property.columnTypes = property.columnTypes.map(() => 'text');
touched = true;
} else if (property.columnTypes?.some((type) => INTEGER_TYPE.test(type))) {
property.columnTypes = property.columnTypes.map((type) => (INTEGER_TYPE.test(type) ? 'bigint' : type));
touched = true;
}
if (typeof property.defaultRaw === 'string') {
const next = portableDefault(property.defaultRaw);
if (next !== property.defaultRaw) {
property.defaultRaw = next;
touched = true;
}
}
if (property.collation?.toUpperCase() === 'NOCASE') {
property.collation = 'nocase';
touched = true;
}
if (touched) changed.push(property.name);
const typeChanged = timestampAsText(property) || integerAsBigint(property);
const defaultChanged = defaultAsPostgres(property);
const collationChanged = nocaseAsCollation(property);
if (typeChanged || defaultChanged || collationChanged) changed.push(property.name);
}
return changed;
}
@@ -0,0 +1,59 @@
/**
* Splits generated DDL into single statements, so one statement Postgres
* refuses does not take the rest of the schema with it.
*
* Understood: `;` outside quotes, `'` and `"` quoted text, and the `$$` dollar
* quote a function body is written in, which is the only dollar quote TREK's
* generated DDL and baseline contain. A tagged dollar quote (`$tag$`) is not
* understood, so it is refused rather than split inside.
*/
/** A tagged dollar quote (`$tag$`), matched where the scan stands. */
const TAGGED_DOLLAR_QUOTE = /\$[A-Za-z_]\w*\$/y;
function refuseTaggedDollarQuote(sql: string, at: number): void {
TAGGED_DOLLAR_QUOTE.lastIndex = at;
const tag = TAGGED_DOLLAR_QUOTE.exec(sql)?.[0];
if (tag) throw new Error(`splitStatements understands only the $$ dollar quote, not ${tag} (at ${at})`);
}
/**
* The statements in `sql`, trimmed and in order. A statement that sets a
* session parameter (`SET ...`) is dropped: a dump's preamble, which a schema
* the entities generate never needs.
*/
export function splitStatements(sql: string): string[] {
const statements: string[] = [];
let current = '';
let quote: '"' | "'" | '$$' | null = null;
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const dollar = sql.startsWith('$$', i);
if (quote === '$$') {
if (dollar) {
quote = null;
current += '$';
i++;
}
current += char;
} else if (quote) {
if (char === quote) quote = null;
current += char;
} else if (dollar) {
quote = '$$';
current += '$$';
i++;
} else if (char === "'" || char === '"') {
quote = char;
current += char;
} else if (char === ';') {
if (current.trim()) statements.push(current.trim());
current = '';
} else {
if (char === '$') refuseTaggedDollarQuote(sql, i);
current += char;
}
}
if (current.trim()) statements.push(current.trim());
return statements.filter((statement) => !/^set\s/i.test(statement));
}
+4 -3
View File
@@ -20,9 +20,10 @@
* - dropping a trigger the entities cannot carry: only the triggers ported by
* hand (POSTGRES_TRIGGERS), by exact name, and each of them must exist.
*/
import { adjustColumns, POSTGRES_TRIGGERS, type ColumnMeta } from '../../src/db/engine/postgres/schema-adjust';
import { POSTGRES_TRIGGERS } from '../../src/db/engine/postgres/postgres-triggers';
import { applyPostgresColumnAdjustments } from '../../src/db/engine/postgres/schema-adjust';
import { ALL_ENTITIES } from '../../src/db/entities';
import type { EntityProperty, Transaction } from '@mikro-orm/core';
import type { Transaction } from '@mikro-orm/core';
import { MikroORM } from '@mikro-orm/postgresql';
import { describe, expect, it } from 'vitest';
@@ -100,7 +101,7 @@ describe('PARITY-PG', () => {
});
try {
for (const meta of orm.getMetadata().getAll().values()) {
adjustColumns(meta.props as EntityProperty[] as ColumnMeta[]);
applyPostgresColumnAdjustments(meta.props);
}
const live = await liveSets(orm);
const statements = (await orm.schema.getUpdateSchemaSQL({ wrap: false }))
@@ -4,19 +4,19 @@
* NOCASE collation, and splitting the generated DDL so one refused statement
* does not take the rest with it.
*/
import { POSTGRES_TRIGGERS } from '../../../../../src/db/engine/postgres/postgres-triggers';
import {
adjustColumns,
applyPostgresColumnAdjustments,
type ColumnMeta,
NOCASE_COLLATION_SQL,
PG_NOW_TEXT_DEFAULT,
portableDefault,
POSTGRES_TRIGGERS,
splitStatements,
type ColumnMeta,
} from '../../../../../src/db/engine/postgres/schema-adjust';
import { splitStatements } from '../../../../../src/db/engine/postgres/sql-split';
import { DbTimestampType } from '../../../../../src/db/types';
import { describe, expect, it } from 'vitest';
class DbTimestampType {}
class TextType {}
describe('pg-probe schema', () => {
@@ -42,7 +42,7 @@ describe('pg-probe schema', () => {
{ name: 'region', columnTypes: ['text'], collation: 'NOCASE' },
{ name: 'body', columnTypes: ['text'] },
];
expect(adjustColumns(columns)).toEqual(['created_at', 'stamp', 'region']);
expect(applyPostgresColumnAdjustments(columns)).toEqual(['created_at', 'stamp', 'region']);
expect(columns[0]).toMatchObject({ columnTypes: ['text'], defaultRaw: PG_NOW_TEXT_DEFAULT });
expect(columns[1]).toMatchObject({ columnTypes: ['text'], defaultRaw: "'x'" });
expect(columns[2]).toMatchObject({
@@ -60,7 +60,7 @@ describe('pg-probe schema', () => {
{ name: 'enabled', columnTypes: ['boolean'] },
{ name: 'note', columnTypes: ['text'] },
];
expect(adjustColumns(columns)).toEqual(['id', 'next_run_at']);
expect(applyPostgresColumnAdjustments(columns)).toEqual(['id', 'next_run_at']);
expect(columns.map((column) => column.columnTypes)).toEqual([['bigint'], ['bigint'], ['boolean'], ['text']]);
});
@@ -80,6 +80,13 @@ describe('pg-probe schema', () => {
expect(splitStatements(' ; ; ')).toEqual([]);
});
it('PGPROBE-045: refuses a tagged dollar quote, which it does not understand, rather than splitting inside it', () => {
expect(() => splitStatements('create function f() as $f$ select 1; $f$;')).toThrow(
/only the \$\$ dollar quote, not \$f\$/,
);
expect(splitStatements("select '$f$'; select 2")).toEqual(["select '$f$'", 'select 2']);
});
it('PGPROBE-044: a $$ body stays one statement, semicolons and all, and every trigger port splits into its two', () => {
const fn =
"CREATE FUNCTION f() RETURNS trigger LANGUAGE plpgsql AS $$\nBEGIN\n UPDATE t SET x = ';';\n RETURN NEW;\nEND\n$$";