mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-22 13:10:29 +00:00
feat(db): create and migrate the schema on a Postgres database
With TREK_DB_URL set, TREK now boots on Postgres; without it nothing changes. - The database engine contract gains open, close, reopen and the schema bootstrap; DatabaseLifecycle delegates to the engine resolveEngine() picks. - An empty Postgres database gets server/assets/postgres/baseline.sql (generated from the entities by gen:pg-baseline, with the reference rows the migrations up to its cut insert: tour types, provider catalogues, addons, app settings) and every migration up to its cut recorded as executed, in one transaction. A database holding tables without TREK's history is refused before anything is written; a migrated one runs the newer-database refusal and its pending migrations, which refuse without a copy unless TREK_DB_PRE_MIGRATE_SNAPSHOT=false. - INTEGER columns are bigint on Postgres (SQLite's INTEGER is 64-bit: an epoch-millisecond column overflowed int4), and the engine's driver reads bigint and numeric back as numbers, refusing values past 2^53. - The two SQLite triggers are ported to PL/pgSQL; check:pg-baseline fails while a SQLite trigger has no port, and while the baseline drifts from the entities before a later migration freezes it. - Backups and restores on Postgres refuse until they can dump and load one. - Migration helpers read the catalog per engine (dialect/catalog.ts), and the email case index no longer needs group_concat. - CI runs check:pg-baseline and boots the image on postgres:17 twice.
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Generates the Postgres baseline: the schema a fresh Postgres database starts
|
||||
* from (src/db/engine/postgres/baseline.ts says how a boot uses it).
|
||||
*
|
||||
* The entities describe the migrated SQLite schema and are pinned to it
|
||||
* (`check:entities`), so the baseline is MikroORM's schema generator over the
|
||||
* entities, for the Postgres platform, after the adjustments every Postgres
|
||||
* schema in this repository shares (src/db/engine/postgres/schema-adjust.ts):
|
||||
* TEXT timestamps, portable defaults, the `nocase` collation. Unlike the
|
||||
* probe's schema it keeps the foreign keys. No database is needed: the
|
||||
* generator reads only the metadata and the platform.
|
||||
*
|
||||
* The baseline stands for every migration up to the newest one when it was
|
||||
* generated (`through`). Once a migration lands after it, the baseline is
|
||||
* frozen like a shipped migration: the newer migration runs on Postgres
|
||||
* through the migrator, and `--check` only verifies the manifest.
|
||||
*
|
||||
* Usage:
|
||||
* node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/
|
||||
* node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI)
|
||||
*/
|
||||
import { baselineMatches, readPostgresBaseline, writePostgresBaseline, type PostgresBaseline } from '../src/db/engine/postgres/baseline';
|
||||
import {
|
||||
adjustColumns,
|
||||
NOCASE_COLLATION_SQL,
|
||||
POSTGRES_TRIGGERS,
|
||||
splitStatements,
|
||||
type ColumnMeta,
|
||||
} from '../src/db/engine/postgres/schema-adjust';
|
||||
import { ALL_ENTITIES } from '../src/db/entities';
|
||||
import { knownMigrationNames } from '../src/db/known-migrations';
|
||||
import type { EntityProperty } from '@mikro-orm/core';
|
||||
import { MikroORM } from '@mikro-orm/postgresql';
|
||||
import { migrateTempDb, tempDbPath } from './generate-entities';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/** The baseline the entities and the migrations on disk describe now. */
|
||||
export async function generatePostgresBaseline(known: Set<string> = knownMigrationNames()): Promise<PostgresBaseline> {
|
||||
const migrations = [...known].sort();
|
||||
const through = migrations.at(-1);
|
||||
if (!through) throw new Error('no migrations found');
|
||||
const orm = await MikroORM.init({
|
||||
// Never contacted: init() only discovers metadata, and the schema generator needs nothing else.
|
||||
clientUrl: 'postgresql://generator@localhost/trek',
|
||||
entities: [...ALL_ENTITIES],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
});
|
||||
try {
|
||||
for (const meta of orm.getMetadata().getAll().values()) adjustColumns(meta.props as EntityProperty[] as ColumnMeta[]);
|
||||
const migrated = await readMigratedSqlite();
|
||||
const statements = [
|
||||
NOCASE_COLLATION_SQL,
|
||||
...splitStatements(await orm.schema.getCreateSchemaSQL({ wrap: false })),
|
||||
...migrated.tables.flatMap((table) => rowStatements(table.name, table.rows)),
|
||||
...Object.values(POSTGRES_TRIGGERS).flat(),
|
||||
];
|
||||
return { through, migrations, statements };
|
||||
} finally {
|
||||
await orm.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What `--check` holds. While no migration is newer than the committed
|
||||
* baseline, the baseline must be exactly what the entities generate. Once one
|
||||
* is, the baseline is frozen and only its manifest is checked: every migration
|
||||
* it names exists, and it names every migration up to `through`.
|
||||
*/
|
||||
export function checkBaseline(committed: PostgresBaseline, generated: PostgresBaseline, matches: boolean): string[] {
|
||||
const known = new Set(generated.migrations);
|
||||
const problems: string[] = [];
|
||||
const missing = committed.migrations.filter((name) => !known.has(name));
|
||||
if (missing.length > 0) problems.push(`the baseline names migrations that do not exist: ${missing.join(', ')}`);
|
||||
const unlisted = generated.migrations.filter((name) => name <= committed.through && !committed.migrations.includes(name));
|
||||
if (unlisted.length > 0) problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`);
|
||||
if (committed.through === generated.through && !matches) {
|
||||
problems.push('the entities generate a different schema; run npm run gen:pg-baseline');
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Migration bookkeeping: the baseline's manifest stands for it on Postgres. */
|
||||
const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schema_version']);
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
/** What the migrations leave in a SQLite database: its triggers, and the rows they insert, table by table in creation order. */
|
||||
async function readMigratedSqlite(): Promise<{ triggers: string[]; tables: Array<{ name: string; rows: Row[] }> }> {
|
||||
const dbPath = tempDbPath();
|
||||
try {
|
||||
await migrateTempDb(dbPath);
|
||||
const conn = new Database(dbPath, { readonly: true });
|
||||
try {
|
||||
const objects = conn
|
||||
.prepare("select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid")
|
||||
.all() as Array<{ type: string; name: string }>;
|
||||
const tables = objects
|
||||
.filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name))
|
||||
.map((object) => ({ name: object.name, rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[] }))
|
||||
.filter((table) => table.rows.length > 0);
|
||||
const triggers = objects.filter((object) => object.type === 'trigger').map((object) => object.name).sort();
|
||||
return { triggers, tables };
|
||||
} finally {
|
||||
conn.close();
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dbPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** A value as a Postgres literal the column's own type takes in: quoted text, which Postgres casts on insert. */
|
||||
function literal(value: unknown): string {
|
||||
if (value === null || value === undefined) return 'NULL';
|
||||
if (Buffer.isBuffer(value)) throw new Error('a migration inserted binary data, which the baseline cannot carry');
|
||||
return `'${String(value).replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows a migration inserted into `table` (reference data such as the tour
|
||||
* types and the provider catalogues), as the statements that put them in a
|
||||
* Postgres database the baseline creates, then move the table's id sequence,
|
||||
* if it has one, past them. A migration up to the cut never runs on Postgres,
|
||||
* so without these the database would start without what every SQLite
|
||||
* install has before its first seeder.
|
||||
*/
|
||||
export function rowStatements(table: string, rows: readonly Row[]): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
const columns = Object.keys(rows[0]);
|
||||
const values = rows.map((row) => `(${columns.map((column) => literal(row[column])).join(', ')})`);
|
||||
const statements = [`insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`];
|
||||
// A text key (addons.id) has no sequence; a numeric one may.
|
||||
if (rows.every((row) => typeof row.id === 'number')) {
|
||||
statements.push(
|
||||
`select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('"${table}"', 'id') as seq, ` +
|
||||
`max("id") as max_id from "${table}") ids where seq is not null`,
|
||||
);
|
||||
}
|
||||
return statements;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SQLite triggers without a Postgres port in `POSTGRES_TRIGGERS`, and the
|
||||
* ports whose trigger is gone. The entities carry no triggers, so this is
|
||||
* what keeps the hand-written ports level with the migrations.
|
||||
*/
|
||||
export function triggerPortProblems(sqliteTriggers: readonly string[], ported: readonly string[]): string[] {
|
||||
const problems: string[] = [];
|
||||
const missing = sqliteTriggers.filter((name) => !ported.includes(name));
|
||||
if (missing.length > 0) problems.push(`SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}`);
|
||||
const stale = ported.filter((name) => !sqliteTriggers.includes(name));
|
||||
if (stale.length > 0) problems.push(`POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}`);
|
||||
return problems;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const generated = await generatePostgresBaseline();
|
||||
if (!process.argv.slice(2).includes('--check')) {
|
||||
writePostgresBaseline(generated);
|
||||
console.log(`[generate-pg-baseline] wrote ${generated.statements.length} statements through ${generated.through}`);
|
||||
return;
|
||||
}
|
||||
const committed = readPostgresBaseline();
|
||||
const problems = checkBaseline(committed, generated, baselineMatches(generated));
|
||||
// Once frozen, a later migration's trigger reaches Postgres through the migrator, not this list.
|
||||
if (committed.through === generated.through) {
|
||||
problems.push(...triggerPortProblems((await readMigratedSqlite()).triggers, Object.keys(POSTGRES_TRIGGERS)));
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
for (const problem of problems) console.error(`[generate-pg-baseline] ${problem}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[generate-pg-baseline] --check: the Postgres baseline matches the entities and the migrations');
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
* repositories. Nothing tested that claim until this probe. Given an empty
|
||||
* Postgres database it
|
||||
*
|
||||
* 1. derives a Postgres schema from the entities (pg-probe/schema.ts);
|
||||
* 1. derives a Postgres schema from the entities (src/db/engine/postgres/schema-adjust.ts);
|
||||
* 2. runs every dialect helper on SQLite and on Postgres and checks both
|
||||
* return what the helper promises (pg-probe/helper-cases.ts); every
|
||||
* case must pass;
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
} from './helper-cases';
|
||||
import { ProbePostgresDialect, type StatementRecorder } from './recorder';
|
||||
import type { MethodResult } from './report';
|
||||
import { adjustColumns, NOCASE_COLLATION_SQL, splitStatements, type ColumnMeta } from './schema';
|
||||
import { adjustColumns, NOCASE_COLLATION_SQL, splitStatements, type ColumnMeta } from '../../src/db/engine/postgres/schema-adjust';
|
||||
import { RequestContext, type Configuration, type EntityManager, type EntityProperty } from '@mikro-orm/core';
|
||||
import { MikroORM, PostgreSqlConnection, PostgreSqlDriver, type Options } from '@mikro-orm/postgresql';
|
||||
import { SqlitePlatform } from '@mikro-orm/sql';
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
/**
|
||||
* The Postgres schema the probe runs against, derived from the entities.
|
||||
*
|
||||
* The entities are generated from the migrated SQLite schema and a parity
|
||||
* test pins them to it (`npm run check:entities`), so they are the one
|
||||
* description of the tables both engines can read. MikroORM's schema
|
||||
* generator turns them into Postgres DDL after three adjustments that keep
|
||||
* the probe about the SQL the repositories send, not about the storage
|
||||
* decisions a real Postgres port still has to make:
|
||||
*
|
||||
* - timestamps stay TEXT, the storage model the services and the dialect
|
||||
* helpers are written for (server/CLAUDE.md, "Storage model");
|
||||
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`) gets
|
||||
* its Postgres equivalent, and a column declared `COLLATE NOCASE` uses
|
||||
* the `nocase` collation `collateNoCase()` names on Postgres;
|
||||
* - no foreign keys, so a statement is never refused only because the
|
||||
* sample row it writes points at nothing (every statement meets empty
|
||||
* tables).
|
||||
*
|
||||
* Unique and partial unique indexes stay, because `ON CONFLICT` needs them.
|
||||
*/
|
||||
import { pgTimestampText, PG_UTC_NOW } from '../../src/db/dialect/platform';
|
||||
|
||||
/** SQLite's timestamp text on Postgres, for a column default. */
|
||||
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
|
||||
|
||||
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
|
||||
export const NOCASE_COLLATION_SQL =
|
||||
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
|
||||
|
||||
/** The Postgres spelling of a column default the entities carry in SQLite's. */
|
||||
export function portableDefault(raw: string): string {
|
||||
const text = raw
|
||||
.trim()
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
|
||||
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
|
||||
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits generated DDL into single statements on `;` outside quotes, so one
|
||||
* statement Postgres refuses does not take the rest of the schema with it.
|
||||
*/
|
||||
export function splitStatements(sql: string): string[] {
|
||||
const statements: string[] = [];
|
||||
let current = '';
|
||||
let quote: '"' | "'" | null = null;
|
||||
for (const char of sql) {
|
||||
if (quote) {
|
||||
if (char === quote) quote = null;
|
||||
current += char;
|
||||
} else if (char === "'" || char === '"') {
|
||||
quote = char;
|
||||
current += char;
|
||||
} else if (char === ';') {
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
return statements.filter((statement) => !/^set\s/i.test(statement));
|
||||
}
|
||||
|
||||
/** The subset of MikroORM's property metadata the schema adjustments touch. */
|
||||
export interface ColumnMeta {
|
||||
name: string;
|
||||
defaultRaw?: string;
|
||||
collation?: string;
|
||||
columnTypes?: string[];
|
||||
customType?: { constructor: { name: string } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the adjustments above to one entity's properties in place:
|
||||
* timestamp columns become `text`, SQLite-only defaults and the NOCASE
|
||||
* collation their Postgres spelling. Returns the names of the properties
|
||||
* it changed, for the log.
|
||||
*/
|
||||
export function adjustColumns(properties: readonly ColumnMeta[]): string[] {
|
||||
const changed: string[] = [];
|
||||
for (const property of properties) {
|
||||
let touched = false;
|
||||
if (property.customType?.constructor.name === 'DbTimestampType' && property.columnTypes) {
|
||||
property.columnTypes = property.columnTypes.map(() => 'text');
|
||||
touched = true;
|
||||
}
|
||||
if (typeof property.defaultRaw === 'string') {
|
||||
const next = portableDefault(property.defaultRaw);
|
||||
if (next !== property.defaultRaw) {
|
||||
property.defaultRaw = next;
|
||||
touched = true;
|
||||
}
|
||||
}
|
||||
if (property.collation?.toUpperCase() === 'NOCASE') {
|
||||
property.collation = 'nocase';
|
||||
touched = true;
|
||||
}
|
||||
if (touched) changed.push(property.name);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
Reference in New Issue
Block a user