mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
feat(db): run TREK on PostgreSQL
TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before otherwise: an engine port under src/db/engine, the dialect helpers under src/db/dialect, a Postgres baseline for the migrations up to the cut, pg_dump/pg_restore backups, a probe of every repository method against a real server and a Postgres test lane in CI.
This commit is contained in:
@@ -26,9 +26,6 @@
|
||||
"src/db/entities/PluginUserErasureQueue.entity.ts": {
|
||||
"datetime": 1
|
||||
},
|
||||
"src/db/repositories/MaintenanceRepository.ts": {
|
||||
"pragma": 1
|
||||
},
|
||||
"src/nest/backup/backup-archive.ts": {
|
||||
"pragma": 1
|
||||
}
|
||||
|
||||
@@ -124,9 +124,16 @@ export function classify(file) {
|
||||
return 'source';
|
||||
}
|
||||
|
||||
/** PRAGMA is fine in src/db/*.ts (connection, durability, baseline, snapshot) and the migrations. */
|
||||
/**
|
||||
* PRAGMA is fine in src/db/*.ts (connection, durability, baseline, snapshot),
|
||||
* the migrations and the SQLite engine's own folder (src/db/engine/sqlite/).
|
||||
*/
|
||||
export function pragmaAllowed(file) {
|
||||
return /^src\/db\/[^/]+\.ts$/.test(file) || file.startsWith('src/db/migrations/');
|
||||
return (
|
||||
/^src\/db\/[^/]+\.ts$/.test(file) ||
|
||||
file.startsWith('src/db/migrations/') ||
|
||||
file.startsWith('src/db/engine/sqlite/')
|
||||
);
|
||||
}
|
||||
|
||||
function isSqlTag(node) {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"counts": {}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* lint:engine-dispatch: code outside the dialect layer and the engine port
|
||||
* never asks which database it runs on.
|
||||
*
|
||||
* What differs between SQLite and Postgres has two homes: the SQL helpers in
|
||||
* src/db/dialect/ and the engine implementations in src/db/engine/ (the
|
||||
* driver, the schema bootstrap, maintenance, backups). Anywhere else, an
|
||||
* `isSqlite(`/`isPostgres(` check, an `instanceof` on a platform class or a
|
||||
* comparison on the engine's name (`readEnv().db.engine === 'postgres'`) is a
|
||||
* branch a third engine would have to find by searching, so each file is held
|
||||
* to its entry in scripts/engine-dispatch-baseline.json and a file without one
|
||||
* holds none (scripts/lib/count-ratchet.mjs has the rules). Comments do not
|
||||
* count.
|
||||
*
|
||||
* npm run lint:engine-dispatch check against the baseline (CI)
|
||||
* npm run lint:engine-dispatch -- --update lower the counts; never raises or adds an entry
|
||||
*/
|
||||
import { cliMain, stripComments } from './lib/count-ratchet.mjs';
|
||||
|
||||
const DISPATCH = new RegExp(
|
||||
[
|
||||
/\bis(?:Sqlite|Postgres)\s*\(/.source,
|
||||
/\binstanceof\s+(?:Sqlite|BasePostgreSql|PostgreSql)Platform\b/.source,
|
||||
// A comparison on the engine's name: `db.engine === 'postgres'`, `engine.id !== 'sqlite'`.
|
||||
/\.(?:engine|id)\s*[!=]==?\s*['"](?:sqlite|postgres)['"]/.source,
|
||||
].join('|'),
|
||||
'g',
|
||||
);
|
||||
|
||||
/** Where an engine may be named: the dialect helpers and the engine port. */
|
||||
const ENGINE_HOMES = ['src/db/dialect/', 'src/db/engine/'];
|
||||
|
||||
/** The engine checks in one file's text, comments left out. */
|
||||
export function countEngineDispatch(text) {
|
||||
return stripComments(text).match(DISPATCH)?.length ?? 0;
|
||||
}
|
||||
|
||||
export const check = {
|
||||
name: 'engine-dispatch',
|
||||
script: 'lint:engine-dispatch',
|
||||
root: 'src',
|
||||
baseline: 'scripts/engine-dispatch-baseline.json',
|
||||
unit: 'engine check(s)',
|
||||
advice: 'Dispatch through a helper in src/db/dialect/ or the DatabaseEngine in src/db/engine/ instead.',
|
||||
include: (file) => file.endsWith('.ts') && !ENGINE_HOMES.some((home) => file.startsWith(home)),
|
||||
count: (text) => countEngineDispatch(text),
|
||||
};
|
||||
|
||||
cliMain(check, import.meta.url);
|
||||
@@ -0,0 +1,229 @@
|
||||
/**
|
||||
* Generates the Postgres baseline: the schema a fresh Postgres database starts
|
||||
* from (src/db/engine/postgres/baseline.ts says how a boot uses it).
|
||||
*
|
||||
* The entities describe the migrated SQLite schema and are pinned to it
|
||||
* (`check:entities`), so the baseline is MikroORM's schema generator over the
|
||||
* entities, for the Postgres platform, after the adjustments every Postgres
|
||||
* schema in this repository shares (src/db/engine/postgres/schema-adjust.ts):
|
||||
* TEXT timestamps, portable defaults, the `nocase` collation. Unlike the
|
||||
* probe's schema it keeps the foreign keys. No database is needed: the
|
||||
* generator reads only the metadata and the platform.
|
||||
*
|
||||
* The baseline stands for every migration up to the newest one when it was
|
||||
* generated (`through`). Once a migration lands after it, the baseline is
|
||||
* frozen like a shipped migration: the newer migration runs on Postgres
|
||||
* through the migrator, and `--check` only verifies the manifest.
|
||||
*
|
||||
* Usage:
|
||||
* node --import tsx scripts/generate-pg-baseline.ts # rewrite server/assets/postgres/
|
||||
* node --import tsx scripts/generate-pg-baseline.ts --check # exit 1 when it drifted (CI)
|
||||
*/
|
||||
import {
|
||||
baselineMatches,
|
||||
readPostgresBaseline,
|
||||
writePostgresBaseline,
|
||||
type PostgresBaseline,
|
||||
} from '../src/db/engine/postgres/baseline';
|
||||
import { POSTGRES_TRIGGERS } from '../src/db/engine/postgres/postgres-triggers';
|
||||
import { applyPostgresColumnAdjustments, NOCASE_COLLATION_SQL } from '../src/db/engine/postgres/schema-adjust';
|
||||
import { splitStatements } from '../src/db/engine/postgres/sql-split';
|
||||
import { ALL_ENTITIES } from '../src/db/entities';
|
||||
import { knownMigrationNames } from '../src/db/known-migrations';
|
||||
import { migrateTempDb, tempDbPath } from './generate-entities';
|
||||
import { MikroORM } from '@mikro-orm/postgresql';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/** What one pass over the migrated SQLite database yields: the baseline, and the triggers the migrations leave there. */
|
||||
interface BaselinePass {
|
||||
baseline: PostgresBaseline;
|
||||
sqliteTriggers: string[];
|
||||
}
|
||||
|
||||
/** The baseline the entities and the migrations on disk describe now, from one migration of the SQLite database. */
|
||||
async function generateBaseline(known: Set<string> = knownMigrationNames()): Promise<BaselinePass> {
|
||||
const migrations = [...known].sort();
|
||||
const through = migrations.at(-1);
|
||||
if (!through) throw new Error('no migrations found');
|
||||
const orm = await MikroORM.init({
|
||||
clientUrl: 'postgresql://generator@localhost/trek',
|
||||
entities: [...ALL_ENTITIES],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
});
|
||||
try {
|
||||
for (const meta of orm.getMetadata().getAll().values()) applyPostgresColumnAdjustments(meta.props);
|
||||
const migrated = await readMigratedSqlite();
|
||||
const statements = [
|
||||
NOCASE_COLLATION_SQL,
|
||||
...splitStatements(await orm.schema.getCreateSchemaSQL({ wrap: false })),
|
||||
...migrated.tables.flatMap((table) => rowStatements(table.name, table.rows)),
|
||||
...Object.values(POSTGRES_TRIGGERS).flat(),
|
||||
];
|
||||
return { baseline: { through, migrations, statements }, sqliteTriggers: migrated.triggers };
|
||||
} finally {
|
||||
await orm.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What `--check` holds, as the problems it finds. Every manifest must name
|
||||
* migrations that exist and every migration up to `through`. While no migration
|
||||
* is newer than the committed baseline (`through` matches), the baseline is
|
||||
* not frozen: it must be exactly what the entities generate, and each SQLite
|
||||
* trigger needs a port. Once a newer migration lands, the baseline is frozen
|
||||
* and only the manifest is checked.
|
||||
*/
|
||||
export function checkBaseline(
|
||||
committed: PostgresBaseline,
|
||||
generated: PostgresBaseline,
|
||||
matches: boolean,
|
||||
sqliteTriggers: readonly string[],
|
||||
): string[] {
|
||||
const known = new Set(generated.migrations);
|
||||
const problems: string[] = [];
|
||||
const missing = committed.migrations.filter((name) => !known.has(name));
|
||||
if (missing.length > 0) problems.push(`the baseline names migrations that do not exist: ${missing.join(', ')}`);
|
||||
const unlisted = generated.migrations.filter(
|
||||
(name) => name <= committed.through && !committed.migrations.includes(name),
|
||||
);
|
||||
if (unlisted.length > 0)
|
||||
problems.push(`migrations up to ${committed.through} are missing from it: ${unlisted.join(', ')}`);
|
||||
const frozen = committed.through !== generated.through;
|
||||
if (frozen) return problems;
|
||||
if (!matches) problems.push('the entities generate a different schema; run npm run gen:pg-baseline');
|
||||
problems.push(...triggerPortProblems(sqliteTriggers, Object.keys(POSTGRES_TRIGGERS)));
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Migration bookkeeping: the baseline's manifest stands for it on Postgres. */
|
||||
const BOOKKEEPING_TABLES = new Set(['mikro_orm_migrations', 'migrations', 'schema_version']);
|
||||
|
||||
type Row = Record<string, unknown>;
|
||||
|
||||
/** What the migrations leave in a SQLite database: its triggers, and the rows they insert, table by table in creation order. */
|
||||
interface MigratedSchema {
|
||||
triggers: string[];
|
||||
tables: Array<{ name: string; rows: Row[] }>;
|
||||
}
|
||||
|
||||
/** The tables and triggers of a SQLite database in creation order, each as its type and name. */
|
||||
function listSchemaObjects(conn: Database.Database): Array<{ type: string; name: string }> {
|
||||
return conn
|
||||
.prepare(
|
||||
"select type, name from sqlite_master where type in ('table', 'trigger') and name not like 'sqlite_%' order by rowid",
|
||||
)
|
||||
.all() as Array<{ type: string; name: string }>;
|
||||
}
|
||||
|
||||
/** Reads a migrated SQLite database, read-only. */
|
||||
function readMigratedSchema(dbPath: string): MigratedSchema {
|
||||
const conn = new Database(dbPath, { readonly: true });
|
||||
try {
|
||||
const objects = listSchemaObjects(conn);
|
||||
const tables = objects
|
||||
.filter((object) => object.type === 'table' && !BOOKKEEPING_TABLES.has(object.name))
|
||||
.map((object) => ({
|
||||
name: object.name,
|
||||
rows: conn.prepare(`select * from "${object.name}" order by rowid`).all() as Row[],
|
||||
}))
|
||||
.filter((table) => table.rows.length > 0);
|
||||
const triggers = objects
|
||||
.filter((object) => object.type === 'trigger')
|
||||
.map((object) => object.name)
|
||||
.sort();
|
||||
return { triggers, tables };
|
||||
} finally {
|
||||
conn.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Migrates a temporary SQLite database and reads what the migrations leave in it; the file is removed afterwards. */
|
||||
async function readMigratedSqlite(): Promise<MigratedSchema> {
|
||||
const dbPath = tempDbPath();
|
||||
try {
|
||||
await migrateTempDb(dbPath);
|
||||
return readMigratedSchema(dbPath);
|
||||
} finally {
|
||||
fs.rmSync(dbPath, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** A value as a Postgres literal the column's own type takes in: quoted text, which Postgres casts on insert. */
|
||||
function literal(value: unknown): string {
|
||||
if (value === null || value === undefined) return 'NULL';
|
||||
if (Buffer.isBuffer(value)) throw new Error('a migration inserted binary data, which the baseline cannot carry');
|
||||
return `'${String(value).replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows a migration inserted into `table` (reference data such as the tour
|
||||
* types and the provider catalogues), as the statements that put them in a
|
||||
* Postgres database the baseline creates, then move the table's id sequence,
|
||||
* if it has one, past them. A migration up to the cut never runs on Postgres,
|
||||
* so without these the database would start without what every SQLite
|
||||
* install has before its first seeder.
|
||||
*/
|
||||
export function rowStatements(table: string, rows: readonly Row[]): string[] {
|
||||
if (rows.length === 0) return [];
|
||||
const columns = Object.keys(rows[0]);
|
||||
const values = rows.map((row) => `(${columns.map((column) => literal(row[column])).join(', ')})`);
|
||||
const statements = [
|
||||
`insert into "${table}" (${columns.map((c) => `"${c}"`).join(', ')}) values ${values.join(', ')}`,
|
||||
];
|
||||
if (rows.every((row) => typeof row.id === 'number')) {
|
||||
statements.push(
|
||||
`select setval(seq, greatest(max_id, 1)) from (select pg_get_serial_sequence('"${table}"', 'id') as seq, ` +
|
||||
`max("id") as max_id from "${table}") ids where seq is not null`,
|
||||
);
|
||||
}
|
||||
return statements;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SQLite triggers without a Postgres port in `POSTGRES_TRIGGERS`, and the
|
||||
* ports whose trigger is gone. The entities carry no triggers, so this is
|
||||
* what keeps the hand-written ports level with the migrations.
|
||||
*/
|
||||
export function triggerPortProblems(sqliteTriggers: readonly string[], ported: readonly string[]): string[] {
|
||||
const problems: string[] = [];
|
||||
const missing = sqliteTriggers.filter((name) => !ported.includes(name));
|
||||
if (missing.length > 0)
|
||||
problems.push(`SQLite triggers without a Postgres port in POSTGRES_TRIGGERS: ${missing.join(', ')}`);
|
||||
const stale = ported.filter((name) => !sqliteTriggers.includes(name));
|
||||
if (stale.length > 0)
|
||||
problems.push(`POSTGRES_TRIGGERS ports triggers the migrations no longer create: ${stale.join(', ')}`);
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** The default mode: writes the baseline the entities and the migrations describe. */
|
||||
async function writeMode(): Promise<void> {
|
||||
const { baseline } = await generateBaseline();
|
||||
writePostgresBaseline(baseline);
|
||||
console.log(`[generate-pg-baseline] wrote ${baseline.statements.length} statements through ${baseline.through}`);
|
||||
}
|
||||
|
||||
/** `--check` (CI): exits 1, naming every problem, when the committed baseline has drifted. */
|
||||
async function checkMode(): Promise<void> {
|
||||
const { baseline: generated, sqliteTriggers } = await generateBaseline();
|
||||
const committed = readPostgresBaseline();
|
||||
const problems = checkBaseline(committed, generated, baselineMatches(generated), sqliteTriggers);
|
||||
if (problems.length > 0) {
|
||||
for (const problem of problems) console.error(`[generate-pg-baseline] ${problem}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[generate-pg-baseline] --check: the Postgres baseline matches the entities and the migrations');
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (process.argv.slice(2).includes('--check')) await checkMode();
|
||||
else await writeMode();
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,21 +1,4 @@
|
||||
{
|
||||
"failing": {
|
||||
"DayAccommodationsRepository.listForTripWithPlaceAndBooking": 1,
|
||||
"DayAccommodationsRepository.listRoadtripStays": 1,
|
||||
"GoogleApiUsageRepository.recordCall": 1,
|
||||
"MaintenanceRepository.vacuumInto": 1,
|
||||
"MaintenanceRepository.walCheckpoint": 1,
|
||||
"ReservationDayPositionsRepository.upsertScoped": 1,
|
||||
"ReservationsRepository.findWithJoins": 1,
|
||||
"ReservationsRepository.listForCalendar": 1,
|
||||
"ReservationsRepository.listForTrip": 1,
|
||||
"ReservationsRepository.listPublicStaysForCalendar": 1,
|
||||
"RouteUsageDailyRepository.record": 1
|
||||
},
|
||||
"uncovered": [
|
||||
"BucketListRepository.update",
|
||||
"BudgetItemsRepository.update",
|
||||
"PackingBagsRepository.update",
|
||||
"TodoItemsRepository.update"
|
||||
]
|
||||
"failing": {},
|
||||
"uncovered": []
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* repositories. Nothing tested that claim until this probe. Given an empty
|
||||
* Postgres database it
|
||||
*
|
||||
* 1. derives a Postgres schema from the entities (pg-probe/schema.ts);
|
||||
* 1. derives a Postgres schema from the entities (src/db/engine/postgres/schema-adjust.ts);
|
||||
* 2. runs every dialect helper on SQLite and on Postgres and checks both
|
||||
* return what the helper promises (pg-probe/helper-cases.ts); every
|
||||
* case must pass;
|
||||
@@ -15,10 +15,11 @@
|
||||
* (pg-probe/arg-samples.ts), through a MikroORM Postgres driver that
|
||||
* runs each statement in its own rolled-back transaction and records
|
||||
* whether Postgres accepted it (pg-probe/recorder.ts);
|
||||
* 4. holds the statements Postgres refuses as SQL (SQLSTATE class 42 or
|
||||
* 0A), per method, to scripts/pg-probe-baseline.json, which may only
|
||||
* shrink (pg-probe/baseline.ts); a refusal over a sample value
|
||||
* (class 22) or the empty tables is reported, not held;
|
||||
* 4. holds the statements Postgres refuses as SQL or over a value's type
|
||||
* (SQLSTATE class 42, 0A or 22), per method, to
|
||||
* scripts/pg-probe-baseline.json, which may only shrink
|
||||
* (pg-probe/baseline.ts); a constraint over a sample value (class 23) or
|
||||
* the empty tables is reported, not held;
|
||||
* 5. holds the methods whose SQL it did not see (it could not call them,
|
||||
* they sent no SQL, or they timed out) to the same file's `uncovered`
|
||||
* list, which may only shrink too, so a new method the probe cannot
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
* The probe has no fixtures: it reads every repository class with the
|
||||
* TypeScript checker and builds one sample argument list per public method
|
||||
* from the declared parameter types (a number is 1, a string is "probe" or a
|
||||
* date-shaped string when the name says so, an object gets every property
|
||||
* date-shaped string when the name says so, a boolean false unless it is the
|
||||
* `present` flag of a partial update's pair, an object gets every property
|
||||
* it can sample, an array one element). The values only have to get the
|
||||
* method to send its SQL; what the probe measures is whether Postgres
|
||||
* accepts that SQL against an empty schema. A sample the column cannot take
|
||||
@@ -24,6 +25,13 @@ export type MethodPlan =
|
||||
| { className: string; file: string; method: string; unprobeable: string };
|
||||
|
||||
const MAX_DEPTH = 4;
|
||||
/**
|
||||
* The label of the flag in the repositories' `[present: boolean, value]`
|
||||
* pairs (`presenceSet`): a patch whose pairs are all absent sends no SQL, so
|
||||
* this one boolean is sampled true. Every other boolean stays false. The
|
||||
* label is spelled in `presenceSet`'s signature, which a test pins.
|
||||
*/
|
||||
export const PRESENCE_LABEL = 'present';
|
||||
const SAMPLE_DATE = '2026-01-01';
|
||||
const SAMPLE_TIMESTAMP = '2026-01-01 00:00:00';
|
||||
|
||||
@@ -68,14 +76,14 @@ export function sampleFor(checker: ts.TypeChecker, type: ts.Type, name: string,
|
||||
if (type.isUnion()) {
|
||||
const members = type.types.filter((member) => !isNullish(member));
|
||||
if (members.length === 0) return OK(null);
|
||||
if (members.every((member) => member.flags & ts.TypeFlags.BooleanLiteral)) return OK(false);
|
||||
if (members.every((member) => member.flags & ts.TypeFlags.BooleanLiteral)) return OK(name === PRESENCE_LABEL);
|
||||
const numeric = members.find((member) => member.flags & ts.TypeFlags.NumberLike);
|
||||
return sampleFor(checker, numeric ?? members[0]!, name, depth + 1);
|
||||
}
|
||||
if (type.isStringLiteral()) return OK(type.value);
|
||||
if (type.isNumberLiteral()) return OK(type.value);
|
||||
if (flags & ts.TypeFlags.BooleanLiteral) return OK(checker.typeToString(type) === 'true');
|
||||
if (flags & ts.TypeFlags.BooleanLike) return OK(false);
|
||||
if (flags & ts.TypeFlags.BooleanLike) return OK(name === PRESENCE_LABEL);
|
||||
if (flags & ts.TypeFlags.BigIntLike) return OK(BigInt(1));
|
||||
if (flags & ts.TypeFlags.NumberLike) return OK(1);
|
||||
if (flags & (ts.TypeFlags.StringLike | ts.TypeFlags.TemplateLiteral)) return OK(sampleString(name));
|
||||
@@ -88,8 +96,15 @@ export function sampleFor(checker: ts.TypeChecker, type: ts.Type, name: string,
|
||||
function sampleObjectType(checker: ts.TypeChecker, type: ts.ObjectType, name: string, depth: number): Sample {
|
||||
if (checker.isTupleType(type)) {
|
||||
const values: unknown[] = [];
|
||||
const labels = ((type as ts.TypeReference).target as ts.TupleType).labeledElementDeclarations;
|
||||
for (const [i, element] of checker.getTypeArguments(type as ts.TypeReference).entries()) {
|
||||
const sample = sampleFor(checker, element, `${name}[${i}]`, depth + 1);
|
||||
const label = labels?.[i]?.name;
|
||||
const sample = sampleFor(
|
||||
checker,
|
||||
element,
|
||||
label && ts.isIdentifier(label) ? label.text : `${name}[${i}]`,
|
||||
depth + 1,
|
||||
);
|
||||
if (!sample.ok) return sample;
|
||||
values.push(sample.value);
|
||||
}
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
* `--update` lowers entries and drops the ones that reach zero or whose
|
||||
* method is gone; it never raises or adds one.
|
||||
*
|
||||
* Only statements Postgres refuses as SQL count here (SQLSTATE classes 42
|
||||
* and 0A, see `countsTowardRatchet`). A value the sample arguments got wrong
|
||||
* (class 22, a sample string in a date column) says nothing about the
|
||||
* Statements Postgres refuses as SQL or over a value's type count here
|
||||
* (SQLSTATE classes 42, 0A and 22, see `countsTowardRatchet`). A constraint
|
||||
* the sample values break (class 23) or a timeout says nothing about the
|
||||
* dialect and is reported without being held.
|
||||
*
|
||||
* `uncovered` lists the methods whose SQL the probe did not see: the ones it
|
||||
@@ -56,12 +56,16 @@ export interface BaselineVerdict {
|
||||
/**
|
||||
* Whether a refused statement counts toward the ratchet: SQLSTATE class 42
|
||||
* (syntax error, unknown function, operator or column, datatype mismatch)
|
||||
* and class 0A (feature not supported) are what a SQLite-only spelling gets.
|
||||
* Anything else (a data exception such as 22P02 or 22007, a constraint, a
|
||||
* timeout) comes from the sample values or the empty tables, not the SQL.
|
||||
* and class 0A (feature not supported) are what a SQLite-only spelling gets,
|
||||
* and class 22 (a data exception such as 22P02 or 22007) is a value of the
|
||||
* wrong type reaching a column, which SQLite compares where Postgres refuses
|
||||
* (a query-string id bound to an integer column). The sampler sends what
|
||||
* each parameter declares, so a class 22 refusal is the code's, not the
|
||||
* sample's. A constraint (class 23) or a timeout comes from the sample values
|
||||
* or the empty tables, not the SQL.
|
||||
*/
|
||||
export function countsTowardRatchet(code: string): boolean {
|
||||
return /^(42|0A)/.test(code);
|
||||
return /^(42|0A|22)/.test(code);
|
||||
}
|
||||
|
||||
const METHOD_KEY = /^[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*$/;
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
* SQLite database the helper cases are compared on, and the loop that calls
|
||||
* every repository method.
|
||||
*/
|
||||
import { applyPostgresColumnAdjustments, NOCASE_COLLATION_SQL } from '../../src/db/engine/postgres/schema-adjust';
|
||||
import { splitStatements } from '../../src/db/engine/postgres/sql-split';
|
||||
import { ALL_ENTITIES } from '../../src/db/entities';
|
||||
import type { MethodPlan } from './arg-samples';
|
||||
import {
|
||||
@@ -18,8 +20,7 @@ import {
|
||||
} from './helper-cases';
|
||||
import { ProbePostgresDialect, type StatementRecorder } from './recorder';
|
||||
import type { MethodResult } from './report';
|
||||
import { adjustColumns, NOCASE_COLLATION_SQL, splitStatements, type ColumnMeta } from './schema';
|
||||
import { RequestContext, type Configuration, type EntityManager, type EntityProperty } from '@mikro-orm/core';
|
||||
import { RequestContext, type Configuration, type EntityManager } from '@mikro-orm/core';
|
||||
import { MikroORM, PostgreSqlConnection, PostgreSqlDriver, type Options } from '@mikro-orm/postgresql';
|
||||
import { SqlitePlatform } from '@mikro-orm/sql';
|
||||
|
||||
@@ -129,7 +130,7 @@ export async function prepareDatabase(
|
||||
await run(NOCASE_COLLATION_SQL);
|
||||
let adjusted = 0;
|
||||
for (const meta of orm.getMetadata().getAll().values()) {
|
||||
adjusted += adjustColumns(meta.props as EntityProperty[] as ColumnMeta[]).length;
|
||||
adjusted += applyPostgresColumnAdjustments(meta.props).length;
|
||||
}
|
||||
const statements = splitStatements(await orm.schema.getCreateSchemaSQL({ wrap: false }));
|
||||
for (const statement of statements) await run(statement);
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
dateOf,
|
||||
dayDistance,
|
||||
foundAgainState,
|
||||
ilike,
|
||||
lower,
|
||||
lowerParam,
|
||||
lowerTrim,
|
||||
@@ -269,6 +270,18 @@ export const HELPER_CASES: readonly HelperCase[] = [
|
||||
raw: (p) => rawOf(dayDistance(p, 'a', '2026-09-24')),
|
||||
expect: { equals: 3 },
|
||||
},
|
||||
{
|
||||
name: 'ilike matches in other case',
|
||||
row: { a: 'Probe Place' },
|
||||
raw: (p) => ({ sql: ilike(p, ['a']), params: ['%PLACE%'] }),
|
||||
expect: { equals: true },
|
||||
},
|
||||
{
|
||||
name: 'ilike keeps the escaped wildcard literal',
|
||||
row: { a: 'Probe Place' },
|
||||
raw: (p) => ({ sql: ilike(p, ['a']), params: ['probe\\_place'] }),
|
||||
expect: { equals: false },
|
||||
},
|
||||
{
|
||||
name: 'collateNoCase',
|
||||
row: { a: 'Probe' },
|
||||
|
||||
@@ -22,7 +22,7 @@ export interface FailedStatement {
|
||||
code: string;
|
||||
message: string;
|
||||
sql: string;
|
||||
/** A dialect refusal (SQLSTATE class 42 or 0A) the ratchet holds; anything else is reported only. */
|
||||
/** A refusal the ratchet holds (SQLSTATE class 42, 0A or 22); anything else is reported only. */
|
||||
counted: boolean;
|
||||
}
|
||||
|
||||
@@ -38,8 +38,9 @@ export interface ProbeSummary {
|
||||
/** Every distinct statement Postgres refused, counted toward the ratchet or not. */
|
||||
failedStatements: FailedStatement[];
|
||||
/**
|
||||
* `Class.method` → distinct statements Postgres refused as SQL (SQLSTATE
|
||||
* class 42 or 0A); every called method appears, zeros included.
|
||||
* `Class.method` → distinct statements Postgres refused as SQL or over a
|
||||
* value's type (SQLSTATE class 42, 0A or 22); every called method appears,
|
||||
* zeros included.
|
||||
*/
|
||||
failingByMethod: Map<string, number>;
|
||||
/**
|
||||
@@ -192,7 +193,8 @@ export function formatConsole(input: ReportInput): string {
|
||||
const refusals = splitRefusals(summary);
|
||||
lines.push(
|
||||
`Statements: ${summary.statements} distinct, ${summary.failedStatements.length} refused by Postgres ` +
|
||||
`(${refusals.counted.length} as SQL, held to the baseline; ${refusals.reported.length} over a sample value or the empty tables, reported only).`,
|
||||
`(${refusals.counted.length} as SQL or over a value's type, held to the baseline; ` +
|
||||
`${refusals.reported.length} over a constraint, a timeout or the empty tables, reported only).`,
|
||||
);
|
||||
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) {
|
||||
lines.push(` ${code}: ${count}${countsTowardRatchet(code) ? '' : ' (not held)'}`);
|
||||
@@ -254,8 +256,8 @@ export function formatMarkdown(input: ReportInput): string {
|
||||
);
|
||||
out.push(`| Methods whose SQL the probe did not see | ${summary.uncovered.size} |`);
|
||||
out.push(`| Distinct statements | ${summary.statements} |`);
|
||||
out.push(`| Refused by Postgres as SQL (held) | ${refusals.counted.length} |`);
|
||||
out.push(`| Refused over a sample value or the empty tables (reported only) | ${refusals.reported.length} |`);
|
||||
out.push(`| Refused by Postgres as SQL or over a value's type (held) | ${refusals.counted.length} |`);
|
||||
out.push(`| Refused over a constraint, a timeout or the empty tables (reported only) | ${refusals.reported.length} |`);
|
||||
out.push('');
|
||||
if (probedNothing(summary))
|
||||
out.push('No repository method was called, so there is nothing to hold to the baseline.', '');
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
/**
|
||||
* The Postgres schema the probe runs against, derived from the entities.
|
||||
*
|
||||
* The entities are generated from the migrated SQLite schema and a parity
|
||||
* test pins them to it (`npm run check:entities`), so they are the one
|
||||
* description of the tables both engines can read. MikroORM's schema
|
||||
* generator turns them into Postgres DDL after three adjustments that keep
|
||||
* the probe about the SQL the repositories send, not about the storage
|
||||
* decisions a real Postgres port still has to make:
|
||||
*
|
||||
* - timestamps stay TEXT, the storage model the services and the dialect
|
||||
* helpers are written for (server/CLAUDE.md, "Storage model");
|
||||
* - a SQLite-only default (`datetime('now')`, `strftime('%s','now')`) gets
|
||||
* its Postgres equivalent, and a column declared `COLLATE NOCASE` uses
|
||||
* the `nocase` collation `collateNoCase()` names on Postgres;
|
||||
* - no foreign keys, so a statement is never refused only because the
|
||||
* sample row it writes points at nothing (every statement meets empty
|
||||
* tables).
|
||||
*
|
||||
* Unique and partial unique indexes stay, because `ON CONFLICT` needs them.
|
||||
*/
|
||||
import { pgTimestampText, PG_UTC_NOW } from '../../src/db/dialect/platform';
|
||||
|
||||
/** SQLite's timestamp text on Postgres, for a column default. */
|
||||
export const PG_NOW_TEXT_DEFAULT = pgTimestampText(PG_UTC_NOW);
|
||||
|
||||
/** The case-insensitive collation `collateNoCase()` names on Postgres. */
|
||||
export const NOCASE_COLLATION_SQL =
|
||||
"CREATE COLLATION IF NOT EXISTS nocase (provider = icu, locale = 'und-u-ks-level2', deterministic = false)";
|
||||
|
||||
/** The Postgres spelling of a column default the entities carry in SQLite's. */
|
||||
export function portableDefault(raw: string): string {
|
||||
const text = raw
|
||||
.trim()
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
|
||||
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
|
||||
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits generated DDL into single statements on `;` outside quotes, so one
|
||||
* statement Postgres refuses does not take the rest of the schema with it.
|
||||
*/
|
||||
export function splitStatements(sql: string): string[] {
|
||||
const statements: string[] = [];
|
||||
let current = '';
|
||||
let quote: '"' | "'" | null = null;
|
||||
for (const char of sql) {
|
||||
if (quote) {
|
||||
if (char === quote) quote = null;
|
||||
current += char;
|
||||
} else if (char === "'" || char === '"') {
|
||||
quote = char;
|
||||
current += char;
|
||||
} else if (char === ';') {
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
current = '';
|
||||
} else {
|
||||
current += char;
|
||||
}
|
||||
}
|
||||
if (current.trim()) statements.push(current.trim());
|
||||
return statements.filter((statement) => !/^set\s/i.test(statement));
|
||||
}
|
||||
|
||||
/** The subset of MikroORM's property metadata the schema adjustments touch. */
|
||||
export interface ColumnMeta {
|
||||
name: string;
|
||||
defaultRaw?: string;
|
||||
collation?: string;
|
||||
columnTypes?: string[];
|
||||
customType?: { constructor: { name: string } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the adjustments above to one entity's properties in place:
|
||||
* timestamp columns become `text`, SQLite-only defaults and the NOCASE
|
||||
* collation their Postgres spelling. Returns the names of the properties
|
||||
* it changed, for the log.
|
||||
*/
|
||||
export function adjustColumns(properties: readonly ColumnMeta[]): string[] {
|
||||
const changed: string[] = [];
|
||||
for (const property of properties) {
|
||||
let touched = false;
|
||||
if (property.customType?.constructor.name === 'DbTimestampType' && property.columnTypes) {
|
||||
property.columnTypes = property.columnTypes.map(() => 'text');
|
||||
touched = true;
|
||||
}
|
||||
if (typeof property.defaultRaw === 'string') {
|
||||
const next = portableDefault(property.defaultRaw);
|
||||
if (next !== property.defaultRaw) {
|
||||
property.defaultRaw = next;
|
||||
touched = true;
|
||||
}
|
||||
}
|
||||
if (property.collation?.toUpperCase() === 'NOCASE') {
|
||||
property.collation = 'nocase';
|
||||
touched = true;
|
||||
}
|
||||
if (touched) changed.push(property.name);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"scripts/db-call-graph.mjs": 1036,
|
||||
"scripts/generate-entities.ts": 2259,
|
||||
"src/db/repositories/Places.repository.ts": 1437,
|
||||
"src/db/repositories/Reservations.repository.ts": 1558,
|
||||
"src/db/repositories/Places.repository.ts": 1435,
|
||||
"src/db/repositories/Reservations.repository.ts": 1551,
|
||||
"src/db/repositories/Trips.repository.ts": 1120,
|
||||
"src/db/repositories/Users.repository.ts": 1445,
|
||||
"src/nest/atlas/atlas.service.ts": 1174,
|
||||
@@ -12,7 +12,7 @@
|
||||
"src/nest/doc-sync/doc-sync.service.ts": 1060,
|
||||
"src/nest/doc-sync/providers/synology-drive.client.ts": 1160,
|
||||
"src/nest/journey/journey-domain.service.ts": 1659,
|
||||
"src/nest/places/places.service.ts": 1549,
|
||||
"src/nest/places/places.service.ts": 1546,
|
||||
"src/nest/plugins/plugin-runtime.service.ts": 1460,
|
||||
"src/nest/plugins/runtime/plugin-sdk.ts": 1093,
|
||||
"src/nest/reservations/reservations.service.ts": 1168,
|
||||
|
||||
@@ -143,7 +143,6 @@
|
||||
"tests/unit/nest/sessions/sessions.service.test.ts": 1,
|
||||
"tests/unit/nest/settings.service.test.ts": 1,
|
||||
"tests/unit/nest/share.service.test.ts": 3,
|
||||
"tests/unit/nest/sqlite-database-backup.test.ts": 2,
|
||||
"tests/unit/nest/storage/storage-admin.service.test.ts": 5,
|
||||
"tests/unit/nest/storage/storage-jobs.service.test.ts": 15,
|
||||
"tests/unit/nest/storage/storage-registry.service.test.ts": 3,
|
||||
|
||||
Reference in New Issue
Block a user