/** * The Postgres probe: what of the server's SQL runs on Postgres today. * * TREK runs on SQLite, and the dialect layer (src/db/dialect/) is written so * a second engine is a branch there rather than a sweep through the * repositories. Nothing tested that claim until this probe. Given an empty * Postgres database it * * 1. derives a Postgres schema from the entities (pg-probe/schema.ts); * 2. runs every dialect helper on SQLite and on Postgres and checks both * return what the helper promises (pg-probe/helper-cases.ts); every * case must pass; * 3. calls every public repository method once with sample arguments the * TypeScript checker builds from its parameter types * (pg-probe/arg-samples.ts), through a MikroORM Postgres driver that * runs each statement in its own rolled-back transaction and records * whether Postgres accepted it (pg-probe/recorder.ts); * 4. holds the statements Postgres refuses as SQL (SQLSTATE class 42 or * 0A), per method, to scripts/pg-probe-baseline.json, which may only * shrink (pg-probe/baseline.ts); a refusal over a sample value * (class 22) or the empty tables is reported, not held; * 5. holds the methods whose SQL it did not see (it could not call them, * they sent no SQL, or they timed out) to the same file's `uncovered` * list, which may only shrink too, so a new method the probe cannot * reach fails the run instead of landing unmeasured. * * What it measures is the statements each method reaches with sample * arguments against empty tables, not every statement the code can send: a * branch that only runs on data the empty tables do not hold stays unseen. * * It needs a Postgres server, so CI runs it in the `postgres-probe` job * against a service container. Locally: * * docker run --rm -d -p 5432:5432 -e POSTGRES_PASSWORD=probe postgres:17 * TREK_PG_PROBE_URL=postgres://postgres:probe@127.0.0.1:5432/postgres npm run probe:pg * * Options: * --url= instead of TREK_PG_PROBE_URL * --update lower scripts/pg-probe-baseline.json to this run (never raises or adds an entry; * an unmeasured baseline gets its first measurement) * * A baseline whose `failing` or `uncovered` is `null` has never been measured and fails the * run (outside `--update`); the baseline the run writes with * `--next-baseline` (the CI artifact) is the first measurement to commit. A * run that called no repository method fails as well. * --next-baseline= write the baseline this run would leave (the CI artifact) * --report= write everything the run saw as JSON * --summary= append the Markdown summary (defaults to $GITHUB_STEP_SUMMARY) * * Exit codes: 0 passed, 1 a helper case or the ratchet failed, 2 the probe * could not run (no database, a database that is not empty, a crash). */ import { planRepositories, readCompilerOptions, type MethodPlan } from './pg-probe/arg-samples'; import { compareWithBaseline, formatBaseline, lowerBaseline, readBaseline } from './pg-probe/baseline'; import { connectProbeOrm, connectSetupOrm, postgresHelperEngine, prepareDatabase, probeRepositories, sqliteHelperEngine, } from './pg-probe/engines'; import { runHelperCases, type HelperResult } from './pg-probe/helper-cases'; import { StatementRecorder } from './pg-probe/recorder'; import { formatConsole, formatMarkdown, passed, probedNothing, summarize, type MethodResult, type ReportInput, } from './pg-probe/report'; import { appendFileSync, readdirSync, writeFileSync } from 'node:fs'; import path from 'node:path'; const SERVER_ROOT = path.join(__dirname, '..'); export const BASELINE_PATH = path.join(SERVER_ROOT, 'scripts', 'pg-probe-baseline.json'); const REPOSITORIES_DIR = path.join(SERVER_ROOT, 'src', 'db', 'repositories'); /** A statement that runs longer than this on empty tables is stuck. */ const STATEMENT_TIMEOUT_MS = 5_000; /** A method still running after this has hung (a pool wait, a loop over a result it never gets). */ const METHOD_TIMEOUT_MS = 20_000; export interface ProbeArgs { url: string; update: boolean; nextBaseline: string | null; report: string | null; summary: string | null; } export function parseArgs(argv: readonly string[], env: Readonly>): ProbeArgs { const args: ProbeArgs = { url: env.TREK_PG_PROBE_URL ?? '', update: false, nextBaseline: null, report: null, summary: env.GITHUB_STEP_SUMMARY || null, }; for (const arg of argv) { const [flag, value] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, '']; if (flag === '--update' && value === '') args.update = true; else if (flag === '--url' && value) args.url = value; else if (flag === '--next-baseline' && value) args.nextBaseline = value; else if (flag === '--report' && value) args.report = value; else if (flag === '--summary' && value) args.summary = value; else throw new Error(`pg-probe: unknown argument ${arg}`); } if (!/^postgres(ql)?:\/\//.test(args.url)) { throw new Error( 'pg-probe: no Postgres URL. Set TREK_PG_PROBE_URL or pass --url=postgres://user:password@host:port/database.', ); } return args; } /** The repository files the probe calls into: every `src/db/repositories/*.ts`, the `_shared/` helpers excepted. */ export function repositoryFiles(dir: string): string[] { return readdirSync(dir) .filter((name) => name.endsWith('.ts') && !name.endsWith('.d.ts')) .sort() .map((name) => path.join(dir, name)); } /** * 0 when every helper case passed, the probe called at least one repository * method, the baseline is measured, no method fails more statements than * its entry allows and every method the probe did not measure is listed as * uncovered. An entry above what the run saw (a count above what its method * fails, an uncovered method that is measured now) fails the run too, and so * does an unmeasured baseline, unless this run is the `--update` that lowers * or seeds it. */ export function exitCode(input: Pick, update: boolean): number { if (passed(input)) return 0; const helpersOk = input.helpers.every((result) => result.failure === null); const held = input.verdict.grown.length === 0 && input.verdict.newlyUncovered.length === 0; return helpersOk && !probedNothing(input.summary) && held && update ? 0 : 1; } /** GitHub annotations when running in Actions, plain lines otherwise. */ function annotate(level: 'error' | 'warning' | 'notice', message: string): void { console.log(process.env.GITHUB_ACTIONS === 'true' ? `::${level}::${message}` : `${level.toUpperCase()}: ${message}`); } function timed(label: string, fn: () => Promise): Promise { const start = Date.now(); console.log(`-- ${label}`); return fn().finally(() => console.log(` ${label}: ${((Date.now() - start) / 1000).toFixed(1)}s`)); } async function main(): Promise { const args = parseArgs(process.argv.slice(2), process.env); const baseline = readBaseline(BASELINE_PATH); const setup = await timed('connect', () => connectSetupOrm(args.url)); let helpers: HelperResult[] = []; let schemaFailures: { statement: string; message: string }[] = []; let results: MethodResult[] = []; try { const schema = await timed('derive the schema from the entities', () => prepareDatabase(setup)); schemaFailures = schema.failures; console.log( ` ${schema.statements} DDL statements, ${schema.adjusted} columns adjusted, ${schema.failures.length} refused`, ); helpers = await timed('dialect helpers on SQLite and Postgres', async () => { const sqlite = sqliteHelperEngine(); try { return [...(await runHelperCases(sqlite)), ...(await runHelperCases(postgresHelperEngine(setup)))]; } finally { sqlite.close(); } }); const plans: MethodPlan[] = await timed('plan the repository calls', async () => planRepositories( repositoryFiles(REPOSITORIES_DIR), readCompilerOptions(path.join(SERVER_ROOT, 'tsconfig.json')), SERVER_ROOT, ), ); console.log(` ${plans.length} public repository methods`); const recorder = new StatementRecorder(); const probe = await connectProbeOrm(args.url, recorder, STATEMENT_TIMEOUT_MS); try { results = await timed('call every repository method', () => probeRepositories(probe, recorder, plans, SERVER_ROOT, METHOD_TIMEOUT_MS), ); } finally { await probe.close(true); } } finally { await setup.close(true); } const summary = summarize(results); const uncoveredNow = new Set(summary.uncovered.keys()); const verdict = compareWithBaseline(baseline, summary.failingByMethod, uncoveredNow); const input: ReportInput = { summary, verdict, helpers, schemaFailures, results }; console.log(formatConsole(input)); const next = lowerBaseline(baseline, summary.failingByMethod, uncoveredNow); if (args.nextBaseline) writeFileSync(args.nextBaseline, formatBaseline(next)); if (args.update) { writeFileSync(BASELINE_PATH, formatBaseline(next)); console.log(`Wrote ${path.relative(SERVER_ROOT, BASELINE_PATH)}.`); } if (args.report) { const counts = { failingByMethod: Object.fromEntries(summary.failingByMethod), uncovered: Object.fromEntries(summary.uncovered), failuresByCode: Object.fromEntries(summary.failuresByCode), }; const report = { summary: { ...summary, ...counts }, verdict, helpers, schemaFailures, results }; writeFileSync(args.report, `${JSON.stringify(report, null, 2)}\n`); } if (args.summary) appendFileSync(args.summary, formatMarkdown(input)); for (const failure of helpers.filter((result) => result.failure !== null)) { annotate('error', `pg-probe helper case [${failure.engine}] ${failure.name}: ${failure.failure}`); } for (const entry of verdict.grown) { annotate( 'error', `pg-probe: ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`, ); } for (const entry of verdict.stale) { annotate( 'error', `pg-probe: ${entry.method} fails ${entry.now} statement(s), the baseline allows ${entry.allowed}. ` + 'Lower it (--update, or the pg-probe-baseline artifact).', ); } for (const method of verdict.newlyUncovered) { annotate( 'error', `pg-probe: ${method} ${summary.uncovered.get(method) ?? 'went unmeasured'}, so its SQL was not checked. ` + 'Give it arguments the probe can sample, or make it reach its SQL with them.', ); } for (const method of verdict.nowCovered) { annotate( 'error', `pg-probe: ${method} is listed as uncovered but is measured now (or gone). Drop it (--update, or the pg-probe-baseline artifact).`, ); } if (probedNothing(summary)) annotate('error', `pg-probe: no repository method was called (${summary.methods} planned).`); if (verdict.unseeded) { annotate( 'error', 'pg-probe: the baseline has not been measured yet. Commit pg-probe-baseline.json from the pg-probe-baseline artifact ' + 'as server/scripts/pg-probe-baseline.json.', ); } for (const failure of schemaFailures) annotate('warning', `pg-probe schema: ${failure.message}`); return exitCode(input, args.update); } if (require.main === module) { // A method abandoned after its timeout may still reject later; that is // already counted as a timeout and must not take the whole run down. process.on('unhandledRejection', (reason: unknown) => { annotate( 'warning', `pg-probe: a rejection nobody awaited: ${reason instanceof Error ? reason.message.split('\n')[0] : String(reason)}`, ); }); main().then( (code) => process.exit(code), (error: unknown) => { console.error( `pg-probe could not run: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`, ); process.exit(2); }, ); }