mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
Maurice
c84d0efb1a
The Postgres probe only ratcheted the methods it managed to call. A method it could not build arguments for, one that threw before its first statement and one that timed out were counted and logged but held to nothing, so a new repository method with a callback parameter could land SQLite-only SQL unseen, and the part of the code the probe reaches could shrink without anyone noticing. pg-probe-baseline.json gains an `uncovered` list of those methods. A method outside it that goes unmeasured fails the run, and so does an entry whose method is measured now or gone, so the list only shrinks. It is seeded from the same first measurement as `failing`; a baseline without the key reads as unmeasured. The failing count also counted every refusal, including data errors the sample values cause (a "probe" string in a date column, SQLSTATE 22007), so renaming a parameter could fail the gate on a change that never touched the SQL. Only classes 42 and 0A count now; the rest is reported in the log and the step summary without being held. The docs said the probe measures every repository statement. They now say it measures what each method reaches with sample arguments against empty tables, and that the methods it cannot reach are held to the list.
275 lines
12 KiB
TypeScript
275 lines
12 KiB
TypeScript
/**
|
|
* Turns what the Postgres probe saw into counts, a console log and the
|
|
* Markdown the CI job writes to its step summary.
|
|
*/
|
|
|
|
import { countsTowardRatchet, type BaselineVerdict } from './baseline';
|
|
import type { HelperResult } from './helper-cases';
|
|
import type { StatementRecord } from './recorder';
|
|
|
|
export interface MethodResult {
|
|
/** `Class.method`. */
|
|
key: string;
|
|
statements: readonly StatementRecord[];
|
|
/** Why the method was not called at all. */
|
|
unprobeable?: string;
|
|
/** The method threw (in JS, after or before its SQL); the statements it sent still count. */
|
|
threw?: string;
|
|
timedOut?: boolean;
|
|
}
|
|
|
|
export interface FailedStatement {
|
|
method: string;
|
|
code: string;
|
|
message: string;
|
|
sql: string;
|
|
/** A dialect refusal (SQLSTATE class 42 or 0A) the ratchet holds; anything else is reported only. */
|
|
counted: boolean;
|
|
}
|
|
|
|
export interface ProbeSummary {
|
|
methods: number;
|
|
called: number;
|
|
unprobeable: number;
|
|
/** Called, but sent no SQL (a pure helper, or it threw before reaching the database). */
|
|
withoutSql: number;
|
|
timedOut: number;
|
|
/** Distinct statements per method, summed. */
|
|
statements: number;
|
|
/** Every distinct statement Postgres refused, counted toward the ratchet or not. */
|
|
failedStatements: FailedStatement[];
|
|
/**
|
|
* `Class.method` → distinct statements Postgres refused as SQL (SQLSTATE
|
|
* class 42 or 0A); every called method appears, zeros included.
|
|
*/
|
|
failingByMethod: Map<string, number>;
|
|
/**
|
|
* `Class.method` → why the probe saw none or not all of its SQL: it could
|
|
* not call it, it sent no SQL (it threw first, or never reaches the
|
|
* database), or it timed out. Held to the baseline's `uncovered` list.
|
|
*/
|
|
uncovered: Map<string, string>;
|
|
/** SQLSTATE → distinct failing statements. */
|
|
failuresByCode: Map<string, number>;
|
|
}
|
|
|
|
export function summarize(results: readonly MethodResult[]): ProbeSummary {
|
|
const summary: ProbeSummary = {
|
|
methods: results.length,
|
|
called: 0,
|
|
unprobeable: 0,
|
|
withoutSql: 0,
|
|
timedOut: 0,
|
|
statements: 0,
|
|
failedStatements: [],
|
|
failingByMethod: new Map(),
|
|
uncovered: new Map(),
|
|
failuresByCode: new Map(),
|
|
};
|
|
for (const result of results) {
|
|
if (result.unprobeable !== undefined) {
|
|
summary.unprobeable += 1;
|
|
summary.uncovered.set(result.key, `could not be called: ${result.unprobeable}`);
|
|
continue;
|
|
}
|
|
summary.called += 1;
|
|
if (result.timedOut) {
|
|
summary.timedOut += 1;
|
|
summary.uncovered.set(result.key, 'timed out');
|
|
} else if (result.statements.length === 0) {
|
|
summary.uncovered.set(result.key, result.threw === undefined ? 'sent no SQL' : `threw before any statement: ${result.threw}`);
|
|
}
|
|
if (result.statements.length === 0) summary.withoutSql += 1;
|
|
const seen = new Set<string>();
|
|
const failed = new Map<string, StatementRecord>();
|
|
for (const statement of result.statements) {
|
|
seen.add(statement.sql);
|
|
if (!statement.outcome.ok && !failed.has(statement.sql)) failed.set(statement.sql, statement);
|
|
}
|
|
summary.statements += seen.size;
|
|
let counted = 0;
|
|
for (const statement of failed.values()) {
|
|
const outcome = statement.outcome;
|
|
if (!('code' in outcome)) continue;
|
|
const dialect = countsTowardRatchet(outcome.code);
|
|
if (dialect) counted += 1;
|
|
summary.failedStatements.push({ method: result.key, code: outcome.code, message: outcome.message, sql: statement.sql, counted: dialect });
|
|
summary.failuresByCode.set(outcome.code, (summary.failuresByCode.get(outcome.code) ?? 0) + 1);
|
|
}
|
|
summary.failingByMethod.set(result.key, counted);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
/** Refused statements the ratchet holds (dialect errors) and the ones it only reports. */
|
|
export function splitRefusals(summary: Pick<ProbeSummary, 'failedStatements'>): { counted: FailedStatement[]; reported: FailedStatement[] } {
|
|
return {
|
|
counted: summary.failedStatements.filter((failed) => failed.counted),
|
|
reported: summary.failedStatements.filter((failed) => !failed.counted),
|
|
};
|
|
}
|
|
|
|
/** SQL on one line, cut to `max` characters, for a log or a table cell. */
|
|
export function oneLine(sql: string, max = 160): string {
|
|
const flat = sql.replace(/\s+/g, ' ').trim();
|
|
return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
|
|
}
|
|
|
|
function cell(text: string): string {
|
|
return text.replace(/\|/g, '\\|').replace(/`/g, "'");
|
|
}
|
|
|
|
export interface ReportInput {
|
|
summary: ProbeSummary;
|
|
verdict: BaselineVerdict;
|
|
helpers: readonly HelperResult[];
|
|
schemaFailures: readonly { statement: string; message: string }[];
|
|
results: readonly MethodResult[];
|
|
}
|
|
|
|
export function helperFailures(helpers: readonly HelperResult[]): HelperResult[] {
|
|
return helpers.filter((result) => result.failure !== null);
|
|
}
|
|
|
|
/**
|
|
* True when the probe called no repository method at all. A broken plan (an
|
|
* unreadable tsconfig, a moved repositories directory) would otherwise pass
|
|
* with nothing failing, and its empty measurement could seed the baseline.
|
|
*/
|
|
export function probedNothing(summary: Pick<ProbeSummary, 'called'>): boolean {
|
|
return summary.called === 0;
|
|
}
|
|
|
|
/**
|
|
* True when the run passes: every helper case on both engines, at least one
|
|
* repository method called, a measured baseline, and the ratchet held (no
|
|
* method failing more statements than allowed, no new unmeasured method, no
|
|
* entry left above what the run saw). An unmeasured baseline fails, so the
|
|
* gate cannot merge before it holds anything.
|
|
*/
|
|
export function passed(input: Pick<ReportInput, 'verdict' | 'helpers' | 'summary'>): boolean {
|
|
return (
|
|
helperFailures(input.helpers).length === 0 &&
|
|
!probedNothing(input.summary) &&
|
|
!input.verdict.unseeded &&
|
|
input.verdict.grown.length === 0 &&
|
|
input.verdict.stale.length === 0 &&
|
|
input.verdict.newlyUncovered.length === 0 &&
|
|
input.verdict.nowCovered.length === 0
|
|
);
|
|
}
|
|
|
|
export function formatConsole(input: ReportInput): string {
|
|
const { summary, verdict } = input;
|
|
const lines: string[] = [];
|
|
const failures = helperFailures(input.helpers);
|
|
lines.push(`Dialect helpers: ${input.helpers.length - failures.length}/${input.helpers.length} cases pass on SQLite and Postgres.`);
|
|
for (const failure of failures) lines.push(` FAIL [${failure.engine}] ${failure.name}: ${failure.failure}`);
|
|
if (input.schemaFailures.length > 0) {
|
|
lines.push(`Schema: ${input.schemaFailures.length} DDL statement(s) Postgres refused (their tables are missing below):`);
|
|
for (const failure of input.schemaFailures) lines.push(` ${failure.message} <- ${oneLine(failure.statement, 120)}`);
|
|
}
|
|
lines.push(
|
|
`Repositories: ${summary.methods} methods, ${summary.called} called, ${summary.unprobeable} unprobeable, ` +
|
|
`${summary.withoutSql} sent no SQL, ${summary.timedOut} timed out.`,
|
|
);
|
|
const refusals = splitRefusals(summary);
|
|
lines.push(
|
|
`Statements: ${summary.statements} distinct, ${summary.failedStatements.length} refused by Postgres ` +
|
|
`(${refusals.counted.length} as SQL, held to the baseline; ${refusals.reported.length} over a sample value or the empty tables, reported only).`,
|
|
);
|
|
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) {
|
|
lines.push(` ${code}: ${count}${countsTowardRatchet(code) ? '' : ' (not held)'}`);
|
|
}
|
|
for (const failed of summary.failedStatements) {
|
|
lines.push(` ${failed.method} ${failed.code} ${failed.message}${failed.counted ? '' : ' (not held)'}`);
|
|
lines.push(` ${oneLine(failed.sql)}`);
|
|
}
|
|
if (summary.uncovered.size > 0) {
|
|
lines.push(`Unmeasured: ${summary.uncovered.size} method(s) whose SQL the probe did not see:`);
|
|
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) lines.push(` ${method}: ${why}`);
|
|
}
|
|
if (probedNothing(summary)) {
|
|
lines.push(`FAIL No repository method was called (${summary.methods} planned); there is nothing to hold to the baseline.`);
|
|
}
|
|
if (verdict.unseeded) {
|
|
lines.push(
|
|
'FAIL Baseline: not measured yet (failing is null). Commit the next baseline this run wrote ' +
|
|
'(the pg-probe-baseline artifact in CI) as scripts/pg-probe-baseline.json.',
|
|
);
|
|
}
|
|
for (const entry of verdict.grown) {
|
|
lines.push(`FAIL ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`);
|
|
}
|
|
for (const entry of verdict.stale) {
|
|
lines.push(`FAIL ${entry.method} is held at ${entry.allowed} failing statement(s) but fails ${entry.now} now; lower the baseline.`);
|
|
}
|
|
for (const method of verdict.newlyUncovered) {
|
|
lines.push(
|
|
`FAIL ${method} ${summary.uncovered.get(method) ?? 'went unmeasured'}, and the baseline does not list it as uncovered. ` +
|
|
'Give it arguments the probe can sample, or make it reach its SQL with them.',
|
|
);
|
|
}
|
|
for (const method of verdict.nowCovered) {
|
|
lines.push(`FAIL ${method} is listed as uncovered but is measured now (or gone); drop it from the baseline.`);
|
|
}
|
|
lines.push(passed(input) ? 'Postgres probe passed.' : 'Postgres probe failed.');
|
|
return lines.join('\n');
|
|
}
|
|
|
|
export function formatMarkdown(input: ReportInput): string {
|
|
const { summary, verdict } = input;
|
|
const failures = helperFailures(input.helpers);
|
|
const out: string[] = ['## Postgres probe', ''];
|
|
out.push(passed(input) ? 'Result: **passed**' : 'Result: **failed**', '');
|
|
out.push('| | |', '|---|---|');
|
|
out.push(`| Dialect helper cases (SQLite and Postgres) | ${input.helpers.length - failures.length} of ${input.helpers.length} pass |`);
|
|
out.push(`| Schema statements refused | ${input.schemaFailures.length} |`);
|
|
const refusals = splitRefusals(summary);
|
|
out.push(`| Repository methods | ${summary.methods} (${summary.called} called, ${summary.unprobeable} unprobeable) |`);
|
|
out.push(`| Methods whose SQL the probe did not see | ${summary.uncovered.size} |`);
|
|
out.push(`| Distinct statements | ${summary.statements} |`);
|
|
out.push(`| Refused by Postgres as SQL (held) | ${refusals.counted.length} |`);
|
|
out.push(`| Refused over a sample value or the empty tables (reported only) | ${refusals.reported.length} |`);
|
|
out.push('');
|
|
if (probedNothing(summary)) out.push('No repository method was called, so there is nothing to hold to the baseline.', '');
|
|
if (verdict.unseeded) {
|
|
out.push(
|
|
'The baseline has not been measured yet, which fails the run. Commit `pg-probe-baseline.json` from the `pg-probe-baseline` artifact as `server/scripts/pg-probe-baseline.json`.',
|
|
'',
|
|
);
|
|
}
|
|
if (verdict.grown.length > 0 || verdict.stale.length > 0) {
|
|
out.push('### Ratchet', '', '| Method | Allowed | Now |', '|---|---|---|');
|
|
for (const entry of [...verdict.grown, ...verdict.stale]) out.push(`| ${cell(entry.method)} | ${entry.allowed} | ${entry.now} |`);
|
|
out.push('');
|
|
}
|
|
if (verdict.newlyUncovered.length > 0 || verdict.nowCovered.length > 0) {
|
|
out.push('### Coverage ratchet', '', '| Method | Baseline | Now |', '|---|---|---|');
|
|
for (const method of verdict.newlyUncovered) out.push(`| ${cell(method)} | measured | ${cell(summary.uncovered.get(method) ?? 'unmeasured')} |`);
|
|
for (const method of verdict.nowCovered) out.push(`| ${cell(method)} | uncovered | measured or gone |`);
|
|
out.push('');
|
|
}
|
|
if (failures.length > 0) {
|
|
out.push('### Helper cases that failed', '', '| Engine | Case | Why |', '|---|---|---|');
|
|
for (const failure of failures) out.push(`| ${failure.engine} | ${cell(failure.name)} | ${cell(failure.failure ?? '')} |`);
|
|
out.push('');
|
|
}
|
|
if (summary.failuresByCode.size > 0) {
|
|
out.push('### Refused statements by SQLSTATE', '', '| SQLSTATE | Statements |', '|---|---|');
|
|
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) out.push(`| ${code} | ${count} |`);
|
|
out.push('');
|
|
out.push('<details><summary>Every refused statement</summary>', '', '| Method | SQLSTATE | Held | Message |', '|---|---|---|---|');
|
|
for (const failed of summary.failedStatements) {
|
|
out.push(`| ${cell(failed.method)} | ${failed.code} | ${failed.counted ? 'yes' : 'no'} | ${cell(failed.message)} |`);
|
|
}
|
|
out.push('', '</details>', '');
|
|
}
|
|
if (summary.uncovered.size > 0) {
|
|
out.push('<details><summary>Methods whose SQL the probe did not see</summary>', '', '| Method | Why |', '|---|---|');
|
|
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) out.push(`| ${cell(method)} | ${cell(why)} |`);
|
|
out.push('', '</details>', '');
|
|
}
|
|
return `${out.join('\n')}\n`;
|
|
}
|