TREK/server/scripts/pg-probe.ts

/**
 * The Postgres probe: what of the server's SQL runs on Postgres today.
 *
 * TREK runs on SQLite, and the dialect layer (src/db/dialect/) is written so
 * a second engine is a branch there rather than a sweep through the
 * repositories. Nothing tested that claim until this probe. Given an empty
 * Postgres database it
 *
 *   1. derives a Postgres schema from the entities (pg-probe/schema.ts);
 *   2. runs every dialect helper on SQLite and on Postgres and checks both
 *      return what the helper promises (pg-probe/helper-cases.ts); every
 *      case must pass;
 *   3. calls every public repository method once with sample arguments the
 *      TypeScript checker builds from its parameter types
 *      (pg-probe/arg-samples.ts), through a MikroORM Postgres driver that
 *      runs each statement in its own rolled-back transaction and records
 *      whether Postgres accepted it (pg-probe/recorder.ts);
 *   4. holds the statements Postgres refuses, per method, to
 *      scripts/pg-probe-baseline.json, which may only shrink
 *      (pg-probe/baseline.ts).
 *
 * It needs a Postgres server, so CI runs it in the `postgres-probe` job
 * against a service container. Locally:
 *
 *   docker run --rm -d -p 5432:5432 -e POSTGRES_PASSWORD=probe postgres:17
 *   TREK_PG_PROBE_URL=postgres://postgres:probe@127.0.0.1:5432/postgres npm run probe:pg
 *
 * Options:
 *   --url=<postgres url>       instead of TREK_PG_PROBE_URL
 *   --update                   lower scripts/pg-probe-baseline.json to this run (never raises or adds an entry;
 *                              an unmeasured baseline gets its first measurement)
 *
 * A baseline whose `failing` is `null` has never been measured and fails the
 * run (outside `--update`); the baseline the run writes with
 * `--next-baseline` (the CI artifact) is the first measurement to commit. A
 * run that called no repository method fails as well.
 *   --next-baseline=<path>     write the baseline this run would leave (the CI artifact)
 *   --report=<path>            write everything the run saw as JSON
 *   --summary=<path>           append the Markdown summary (defaults to $GITHUB_STEP_SUMMARY)
 *
 * Exit codes: 0 passed, 1 a helper case or the ratchet failed, 2 the probe
 * could not run (no database, a database that is not empty, a crash).
 */

import { appendFileSync, readdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';

import { planRepositories, readCompilerOptions, type MethodPlan } from './pg-probe/arg-samples';
import { compareWithBaseline, formatBaseline, lowerBaseline, readBaseline } from './pg-probe/baseline';
import { connectProbeOrm, connectSetupOrm, postgresHelperEngine, prepareDatabase, probeRepositories, sqliteHelperEngine } from './pg-probe/engines';
import { runHelperCases, type HelperResult } from './pg-probe/helper-cases';
import { StatementRecorder } from './pg-probe/recorder';
import { formatConsole, formatMarkdown, passed, probedNothing, summarize, type MethodResult, type ReportInput } from './pg-probe/report';

const SERVER_ROOT = path.join(__dirname, '..');
export const BASELINE_PATH = path.join(SERVER_ROOT, 'scripts', 'pg-probe-baseline.json');
const REPOSITORIES_DIR = path.join(SERVER_ROOT, 'src', 'db', 'repositories');

/** A statement that runs longer than this on empty tables is stuck. */
const STATEMENT_TIMEOUT_MS = 5_000;
/** A method still running after this has hung (a pool wait, a loop over a result it never gets). */
const METHOD_TIMEOUT_MS = 20_000;

export interface ProbeArgs {
  url: string;
  update: boolean;
  nextBaseline: string | null;
  report: string | null;
  summary: string | null;
}

export function parseArgs(argv: readonly string[], env: Readonly<Record<string, string | undefined>>): ProbeArgs {
  const args: ProbeArgs = {
    url: env.TREK_PG_PROBE_URL ?? '',
    update: false,
    nextBaseline: null,
    report: null,
    summary: env.GITHUB_STEP_SUMMARY || null,
  };
  for (const arg of argv) {
    const [flag, value] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, ''];
    if (flag === '--update' && value === '') args.update = true;
    else if (flag === '--url' && value) args.url = value;
    else if (flag === '--next-baseline' && value) args.nextBaseline = value;
    else if (flag === '--report' && value) args.report = value;
    else if (flag === '--summary' && value) args.summary = value;
    else throw new Error(`pg-probe: unknown argument ${arg}`);
  }
  if (!/^postgres(ql)?:\/\//.test(args.url)) {
    throw new Error('pg-probe: no Postgres URL. Set TREK_PG_PROBE_URL or pass --url=postgres://user:password@host:port/database.');
  }
  return args;
}

/** The repository files the probe calls into: every `src/db/repositories/*.ts`, the `_shared/` helpers excepted. */
export function repositoryFiles(dir: string): string[] {
  return readdirSync(dir)
    .filter((name) => name.endsWith('.ts') && !name.endsWith('.d.ts'))
    .sort()
    .map((name) => path.join(dir, name));
}

/**
 * 0 when every helper case passed, the probe called at least one repository
 * method, the baseline is measured and no method fails more statements than
 * its entry allows. An entry above what its method fails now fails the run
 * too, and so does an unmeasured baseline, unless this run is the `--update`
 * that lowers or seeds it.
 */
export function exitCode(input: Pick<ReportInput, 'verdict' | 'helpers' | 'summary'>, update: boolean): number {
  if (passed(input)) return 0;
  const helpersOk = input.helpers.every((result) => result.failure === null);
  return helpersOk && !probedNothing(input.summary) && input.verdict.grown.length === 0 && update ? 0 : 1;
}

/** GitHub annotations when running in Actions, plain lines otherwise. */
function annotate(level: 'error' | 'warning' | 'notice', message: string): void {
  console.log(process.env.GITHUB_ACTIONS === 'true' ? `::${level}::${message}` : `${level.toUpperCase()}: ${message}`);
}

function timed<T>(label: string, fn: () => Promise<T>): Promise<T> {
  const start = Date.now();
  console.log(`-- ${label}`);
  return fn().finally(() => console.log(`   ${label}: ${((Date.now() - start) / 1000).toFixed(1)}s`));
}

async function main(): Promise<number> {
  const args = parseArgs(process.argv.slice(2), process.env);
  const baseline = readBaseline(BASELINE_PATH);

  const setup = await timed('connect', () => connectSetupOrm(args.url));
  let helpers: HelperResult[] = [];
  let schemaFailures: { statement: string; message: string }[] = [];
  let results: MethodResult[] = [];
  try {
    const schema = await timed('derive the schema from the entities', () => prepareDatabase(setup));
    schemaFailures = schema.failures;
    console.log(`   ${schema.statements} DDL statements, ${schema.adjusted} columns adjusted, ${schema.failures.length} refused`);

    helpers = await timed('dialect helpers on SQLite and Postgres', async () => {
      const sqlite = sqliteHelperEngine();
      try {
        return [...(await runHelperCases(sqlite)), ...(await runHelperCases(postgresHelperEngine(setup)))];
      } finally {
        sqlite.close();
      }
    });

    const plans: MethodPlan[] = await timed('plan the repository calls', async () =>
      planRepositories(repositoryFiles(REPOSITORIES_DIR), readCompilerOptions(path.join(SERVER_ROOT, 'tsconfig.json')), SERVER_ROOT),
    );
    console.log(`   ${plans.length} public repository methods`);

    const recorder = new StatementRecorder();
    const probe = await connectProbeOrm(args.url, recorder, STATEMENT_TIMEOUT_MS);
    try {
      results = await timed('call every repository method', () => probeRepositories(probe, recorder, plans, SERVER_ROOT, METHOD_TIMEOUT_MS));
    } finally {
      await probe.close(true);
    }
  } finally {
    await setup.close(true);
  }

  const summary = summarize(results);
  const verdict = compareWithBaseline(baseline, summary.failingByMethod);
  const input: ReportInput = { summary, verdict, helpers, schemaFailures, results };
  console.log(formatConsole(input));

  const next = lowerBaseline(baseline, summary.failingByMethod);
  if (args.nextBaseline) writeFileSync(args.nextBaseline, formatBaseline(next));
  if (args.update) {
    writeFileSync(BASELINE_PATH, formatBaseline(next));
    console.log(`Wrote ${path.relative(SERVER_ROOT, BASELINE_PATH)}.`);
  }
  if (args.report) {
    const counts = { failingByMethod: Object.fromEntries(summary.failingByMethod), failuresByCode: Object.fromEntries(summary.failuresByCode) };
    const report = { summary: { ...summary, ...counts }, verdict, helpers, schemaFailures, results };
    writeFileSync(args.report, `${JSON.stringify(report, null, 2)}\n`);
  }
  if (args.summary) appendFileSync(args.summary, formatMarkdown(input));

  for (const failure of helpers.filter((result) => result.failure !== null)) {
    annotate('error', `pg-probe helper case [${failure.engine}] ${failure.name}: ${failure.failure}`);
  }
  for (const entry of verdict.grown) {
    annotate('error', `pg-probe: ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`);
  }
  for (const entry of verdict.stale) {
    annotate(
      'error',
      `pg-probe: ${entry.method} fails ${entry.now} statement(s), the baseline allows ${entry.allowed}. ` +
        'Lower it (--update, or the pg-probe-baseline artifact).',
    );
  }
  if (probedNothing(summary)) annotate('error', `pg-probe: no repository method was called (${summary.methods} planned).`);
  if (verdict.unseeded) {
    annotate(
      'error',
      'pg-probe: the baseline has not been measured yet. Commit pg-probe-baseline.json from the pg-probe-baseline artifact ' +
        'as server/scripts/pg-probe-baseline.json.',
    );
  }
  for (const failure of schemaFailures) annotate('warning', `pg-probe schema: ${failure.message}`);
  return exitCode(input, args.update);
}

if (require.main === module) {
  // A method abandoned after its timeout may still reject later; that is
  // already counted as a timeout and must not take the whole run down.
  process.on('unhandledRejection', (reason: unknown) => {
    annotate('warning', `pg-probe: a rejection nobody awaited: ${reason instanceof Error ? reason.message.split('\n')[0] : String(reason)}`);
  });
  main().then(
    (code) => process.exit(code),
    (error: unknown) => {
      console.error(`pg-probe could not run: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`);
      process.exit(2);
    },
  );
}