mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
293a48c26a
A frozen baseline still has its manifest verified, and the check now lists the migrations that run on Postgres after the cut and what holds them. Every problem it reports says what to do about it.
281 lines
12 KiB
TypeScript
281 lines
12 KiB
TypeScript
/**
|
|
* Generates the Postgres baseline: the schema a fresh Postgres database starts
|
|
* from (src/db/engine/postgres/baseline.ts says how a boot uses it).
|
|
*
|
|
* The entities describe the migrated SQLite schema and are pinned to it
|
|
* (`check:entities`), so the baseline is MikroORM's schema generator over the
|
|
* entities, for the Postgres platform, after the adjustments every Postgres
|
|
* schema in this repository shares (src/db/engine/postgres/schema-adjust.ts):
|
|
* TEXT timestamps, portable defaults, the `nocase` collation. Unlike the
|
|
* probe's schema it keeps the foreign keys. No database is needed: the
|
|
* generator reads only the metadata and the platform.
|
|
*
|
|
* The baseline stands for every migration up to the newest one when it was
|
|
* generated (`through`). Once a migration lands after it, the baseline is
|
|
* frozen like a shipped migration: the newer migration runs on Postgres
|
|
* through the migrator, written through `engineDdl` with the Postgres side
|
|
* from `npm run gen:pg-migration`, and `--check` verifies the manifest and
|
|
* lists the migrations after the cut. Nothing here moves the cut.
|
|
*
|
|
* Usage:
|
|
* node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/
|
|
* node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI)
|
|
*/
|
|
import {
|
|
baselineMatches,
|
|
readPostgresBaseline,
|
|
writePostgresBaseline,
|
|
type PostgresBaseline,
|
|
} from '../src/db/engine/postgres/baseline';
|
|
import { POSTGRES_TRIGGERS } from '../src/db/engine/postgres/postgres-triggers';
|
|
import { applyPostgresColumnAdjustments, NOCASE_COLLATION_SQL } from '../src/db/engine/postgres/schema-adjust';
|
|
import { splitStatements } from '../src/db/engine/postgres/sql-split';
|
|
import { ALL_ENTITIES } from '../src/db/entities';
|
|
import { knownMigrationNames } from '../src/db/known-migrations';
|
|
import { migrateTempDb, tempDbPath } from './generate-entities';
|
|
import { MikroORM } from '@mikro-orm/postgresql';
|
|
|
|
import Database from 'better-sqlite3';
|
|
import fs from 'node:fs';
|
|
|
|
/** What one pass over the migrated SQLite database yields: the baseline, and the triggers the migrations leave there. */
|
|
interface BaselinePass {
|
|
baseline: PostgresBaseline;
|
|
sqliteTriggers: string[];
|
|
}
|
|
|
|
/** The baseline the entities and the migrations on disk describe now, from one migration of the SQLite database. */
|
|
async function generateBaseline(known: Set<string> = knownMigrationNames()): Promise<BaselinePass> {
|
|
const migrations = [...known].sort();
|
|
const through = migrations.at(-1);
|
|
if (!through) throw new Error('no migrations found');
|
|
const orm = await MikroORM.init({
|
|
clientUrl: 'postgresql://generator@localhost/trek',
|
|
entities: [...ALL_ENTITIES],
|
|
discovery: { warnWhenNoEntities: false },
|
|
});
|
|
try {
|
|
for (const meta of orm.getMetadata().getAll().values()) applyPostgresColumnAdjustments(meta.props);
|
|
const migrated = await readMigratedSqlite();
|
|
const statements = [
|
|
NOCASE_COLLATION_SQL,
|
|
...splitStatements(await orm.schema.getCreateSchemaSQL({ wrap: false })),
|
|
...migrated.tables.flatMap((table) => rowStatements(table.name, table.rows)),
|
|
...Object.values(POSTGRES_TRIGGERS).flat(),
|
|
];
|
|
return { baseline: { through, migrations, statements }, sqliteTriggers: migrated.triggers };
|
|
} finally {
|
|
await orm.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The migrations after the committed baseline's cut, in order: what a boot
|
|
* runs on Postgres through the migrator, each written through `engineDdl`
|
|
* (lint:dialect) and held by PARITY-PG. While there is one, the baseline is
|
|
* frozen. `known` is every migration on disk.
|
|
*/
|
|
export function postCutMigrations(committed: PostgresBaseline, known: readonly string[]): string[] {
|
|
return known.filter((name) => name > committed.through).sort();
|
|
}
|
|
|
|
/**
|
|
* What `--check` holds, as the problems it finds, each with what to do about
|
|
* it. Every manifest must name migrations that exist and every migration up to
|
|
* `through`. While no migration is newer than the committed baseline
|
|
* (`through` matches), the baseline is not frozen: it must be exactly what the
|
|
* entities generate, and each SQLite trigger needs a port. Once a newer
|
|
* migration lands, the baseline is frozen: Postgres reaches the newer schema
|
|
* through that migration, so only the manifest is checked here, and
|
|
* `checkMode` lists the migrations after the cut.
|
|
*/
|
|
export function checkBaseline(
|
|
committed: PostgresBaseline,
|
|
generated: PostgresBaseline,
|
|
matches: boolean,
|
|
sqliteTriggers: readonly string[],
|
|
): string[] {
|
|
const known = new Set(generated.migrations);
|
|
const problems: string[] = [];
|
|
const missing = committed.migrations.filter((name) => !known.has(name));
|
|
if (missing.length > 0) {
|
|
problems.push(
|
|
`the manifest names migrations that are not in src/db/migrations: ${missing.join(', ')}. ` +
|
|
'A migration that has shipped is never renamed or deleted; restore it under its name.',
|
|
);
|
|
}
|
|
const unlisted = generated.migrations.filter(
|
|
(name) => name <= committed.through && !committed.migrations.includes(name),
|
|
);
|
|
if (unlisted.length > 0) {
|
|
problems.push(
|
|
`migrations that sort before the cut (${committed.through}) are not in the manifest: ${unlisted.join(', ')}. ` +
|
|
`Postgres would never run them; give each a name that sorts after ${generated.through}.`,
|
|
);
|
|
}
|
|
const frozen = committed.through !== generated.through;
|
|
if (frozen) return problems;
|
|
if (!matches) {
|
|
problems.push(
|
|
'server/assets/postgres/ is not what the entities, POSTGRES_TRIGGERS and the reference rows the migrations ' +
|
|
'insert generate now, and no migration follows its cut: run npm run gen:pg-baseline and commit ' +
|
|
'server/assets/postgres/.',
|
|
);
|
|
}
|
|
problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS)));
|
|
return problems;
|
|
}
|
|
|
|
/** Migration bookkeeping: the baseline's manifest stands for it on Postgres. */
|
|
const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schema_version']);
|
|
|
|
type Row = Record<string, unknown>;
|
|
|
|
/** What the migrations leave in a SQLite database: its triggers, and the rows they insert, table by table in creation order. */
|
|
interface MigratedSchema {
|
|
triggers: string[];
|
|
tables: Array<{ name: string; rows: Row[] }>;
|
|
}
|
|
|
|
/** The tables and triggers of a SQLite database in creation order, each as its type and name. */
|
|
function listSchemaObjects(conn: Database.Database): Array<{ type: string; name: string }> {
|
|
return conn
|
|
.prepare(
|
|
"select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid",
|
|
)
|
|
.all() as Array<{ type: string; name: string }>;
|
|
}
|
|
|
|
/** Reads a migrated SQLite database, read-only. */
|
|
function readMigratedSchema(dbPath: string): MigratedSchema {
|
|
const conn = new Database(dbPath, { readonly: true });
|
|
try {
|
|
const objects = listSchemaObjects(conn);
|
|
const tables = objects
|
|
.filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name))
|
|
.map((object) => ({
|
|
name: object.name,
|
|
rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[],
|
|
}))
|
|
.filter((table) => table.rows.length > 0);
|
|
const triggers = objects
|
|
.filter((object) => object.type === 'trigger')
|
|
.map((object) => object.name)
|
|
.sort();
|
|
return { triggers, tables };
|
|
} finally {
|
|
conn.close();
|
|
}
|
|
}
|
|
|
|
/** Migrates a temporary SQLite database and reads what the migrations leave in it; the file is removed afterwards. */
|
|
async function readMigratedSqlite(): Promise<MigratedSchema> {
|
|
const dbPath = tempDbPath();
|
|
try {
|
|
await migrateTempDb(dbPath);
|
|
return readMigratedSchema(dbPath);
|
|
} finally {
|
|
fs.rmSync(dbPath, { force: true });
|
|
}
|
|
}
|
|
|
|
/** A value as a Postgres literal the column's own type takes in: quoted text, which Postgres casts on insert. */
|
|
function literal(value: unknown): string {
|
|
if (value === null || value === undefined) return 'NULL';
|
|
if (Buffer.isBuffer(value)) throw new Error('a migration inserted binary data, which the baseline cannot carry');
|
|
return `'${String(value).replace(/'/g, "''")}'`;
|
|
}
|
|
|
|
/**
|
|
* The rows a migration inserted into `table` (reference data such as the tour
|
|
* types and the provider catalogues), as the statements that put them in a
|
|
* Postgres database the baseline creates, then move the table's id sequence,
|
|
* if it has one, past them. A migration up to the cut never runs on Postgres,
|
|
* so without these the database would start without what every SQLite
|
|
* install has before its first seeder.
|
|
*/
|
|
export function rowStatements(table: string, rows: readonly Row[]): string[] {
|
|
if (rows.length === 0) return [];
|
|
const columns = Object.keys(rows[0]);
|
|
const values = rows.map((row) => `(${columns.map((column) => literal(row[column])).join(', ')})`);
|
|
const statements = [
|
|
`insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`,
|
|
];
|
|
if (rows.every((row) => typeof row.id === 'number')) {
|
|
statements.push(
|
|
`select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('"${table}"', 'id') as seq, ` +
|
|
`max("id") as max_id from "${table}") ids where seq is not null`,
|
|
);
|
|
}
|
|
return statements;
|
|
}
|
|
|
|
/**
|
|
* The SQLite triggers without a Postgres port in `POSTGRES_TRIGGERS`, and the
|
|
* ports whose trigger is gone. The entities carry no triggers, so this is
|
|
* what keeps the hand-written ports level with the migrations.
|
|
*/
|
|
export function triggerPortProblems(sqliteTriggers: readonly string[], ported: readonly string[]): string[] {
|
|
const problems: string[] = [];
|
|
const missing = sqliteTriggers.filter((name) => !ported.includes(name));
|
|
if (missing.length > 0) {
|
|
problems.push(
|
|
`SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}. Add each one's ` +
|
|
'function and trigger to src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.',
|
|
);
|
|
}
|
|
const stale = ported.filter((name) => !sqliteTriggers.includes(name));
|
|
if (stale.length > 0) {
|
|
problems.push(
|
|
`POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}. Remove them from ` +
|
|
'src/db/engine/postgres/postgres-triggers.ts, then run npm run gen:pg-baseline.',
|
|
);
|
|
}
|
|
return problems;
|
|
}
|
|
|
|
/** The default mode: writes the baseline the entities and the migrations describe. */
|
|
async function writeMode(): Promise<void> {
|
|
const { baseline } = await generateBaseline();
|
|
writePostgresBaseline(baseline);
|
|
console.log(`[generate-pg-baseline] wrote ${baseline.statements.length} statements through ${baseline.through}`);
|
|
}
|
|
|
|
/** `--check` (CI): exits 1, naming every problem, when the committed baseline has drifted. */
|
|
async function checkMode(): Promise<void> {
|
|
const { baseline: generated, sqliteTriggers } = await generateBaseline();
|
|
const committed = readPostgresBaseline();
|
|
const problems = checkBaseline(committed, generated, baselineMatches(generated), sqliteTriggers);
|
|
const after = postCutMigrations(committed, generated.migrations);
|
|
if (after.length > 0) {
|
|
// Frozen: say so, and what keeps those migrations right on Postgres, whether or not the manifest passed.
|
|
console.log(
|
|
`[generate-pg-baseline] --check: the baseline is frozen at ${committed.through}. ${after.length} migration(s) ` +
|
|
`run on Postgres after its cut, through the migrator: ${after.join(', ')}. Each writes its schema change ` +
|
|
'through engineDdl (lint:dialect) and PARITY-PG in the Postgres lane holds the result against the entities; ' +
|
|
'the baseline is not regenerated for them.',
|
|
);
|
|
}
|
|
if (problems.length > 0) {
|
|
for (const problem of problems) console.error(`[generate-pg-baseline] ${problem}`);
|
|
process.exit(1);
|
|
}
|
|
console.log(
|
|
after.length > 0
|
|
? `[generate-pg-baseline] --check: the manifest lists every migration through ${committed.through}`
|
|
: '[generate-pg-baseline] --check: the Postgres baseline matches the entities and the migrations',
|
|
);
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
if (process.argv.slice(2).includes('--check')) await checkMode();
|
|
else await writeMode();
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch((error: unknown) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|
|
}
|