mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-20 14:06:27 +00:00
ci(server): hold unmeasured methods and only SQL errors to the probe baseline
The Postgres probe only ratcheted the methods it managed to call. A method it could not build arguments for, one that threw before its first statement and one that timed out were counted and logged but held to nothing, so a new repository method with a callback parameter could land SQLite-only SQL unseen, and the part of the code the probe reaches could shrink without anyone noticing. pg-probe-baseline.json gains an `uncovered` list of those methods. A method outside it that goes unmeasured fails the run, and so does an entry whose method is measured now or gone, so the list only shrinks. It is seeded from the same first measurement as `failing`; a baseline without the key reads as unmeasured. The failing count also counted every refusal, including data errors the sample values cause (a "probe" string in a date column, SQLSTATE 22007), so renaming a parameter could fail the gate on a change that never touched the SQL. Only classes 42 and 0A count now; the rest is reported in the log and the step summary without being held. The docs said the probe measures every repository statement. They now say it measures what each method reaches with sample arguments against empty tables, and that the methods it cannot reach are held to the list.
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
{
|
||||
"failing": null
|
||||
"failing": null,
|
||||
"uncovered": null
|
||||
}
|
||||
|
||||
+39
-12
@@ -15,9 +15,18 @@
|
||||
* (pg-probe/arg-samples.ts), through a MikroORM Postgres driver that
|
||||
* runs each statement in its own rolled-back transaction and records
|
||||
* whether Postgres accepted it (pg-probe/recorder.ts);
|
||||
* 4. holds the statements Postgres refuses, per method, to
|
||||
* scripts/pg-probe-baseline.json, which may only shrink
|
||||
* (pg-probe/baseline.ts).
|
||||
* 4. holds the statements Postgres refuses as SQL (SQLSTATE class 42 or
|
||||
* 0A), per method, to scripts/pg-probe-baseline.json, which may only
|
||||
* shrink (pg-probe/baseline.ts); a refusal over a sample value
|
||||
* (class 22) or the empty tables is reported, not held;
|
||||
* 5. holds the methods whose SQL it did not see (it could not call them,
|
||||
* they sent no SQL, or they timed out) to the same file's `uncovered`
|
||||
* list, which may only shrink too, so a new method the probe cannot
|
||||
* reach fails the run instead of landing unmeasured.
|
||||
*
|
||||
* What it measures is the statements each method reaches with sample
|
||||
* arguments against empty tables, not every statement the code can send: a
|
||||
* branch that only runs on data the empty tables do not hold stays unseen.
|
||||
*
|
||||
* It needs a Postgres server, so CI runs it in the `postgres-probe` job
|
||||
* against a service container. Locally:
|
||||
@@ -30,7 +39,7 @@
|
||||
* --update lower scripts/pg-probe-baseline.json to this run (never raises or adds an entry;
|
||||
* an unmeasured baseline gets its first measurement)
|
||||
*
|
||||
* A baseline whose `failing` is `null` has never been measured and fails the
|
||||
* A baseline whose `failing` or `uncovered` is `null` has never been measured and fails the
|
||||
* run (outside `--update`); the baseline the run writes with
|
||||
* `--next-baseline` (the CI artifact) is the first measurement to commit. A
|
||||
* run that called no repository method fails as well.
|
||||
@@ -102,15 +111,18 @@ export function repositoryFiles(dir: string): string[] {
|
||||
|
||||
/**
|
||||
* 0 when every helper case passed, the probe called at least one repository
|
||||
* method, the baseline is measured and no method fails more statements than
|
||||
* its entry allows. An entry above what its method fails now fails the run
|
||||
* too, and so does an unmeasured baseline, unless this run is the `--update`
|
||||
* that lowers or seeds it.
|
||||
* method, the baseline is measured, no method fails more statements than
|
||||
* its entry allows and every method the probe did not measure is listed as
|
||||
* uncovered. An entry above what the run saw (a count above what its method
|
||||
* fails, an uncovered method that is measured now) fails the run too, and so
|
||||
* does an unmeasured baseline, unless this run is the `--update` that lowers
|
||||
* or seeds it.
|
||||
*/
|
||||
export function exitCode(input: Pick<ReportInput, 'verdict' | 'helpers' | 'summary'>, update: boolean): number {
|
||||
if (passed(input)) return 0;
|
||||
const helpersOk = input.helpers.every((result) => result.failure === null);
|
||||
return helpersOk && !probedNothing(input.summary) && input.verdict.grown.length === 0 && update ? 0 : 1;
|
||||
const held = input.verdict.grown.length === 0 && input.verdict.newlyUncovered.length === 0;
|
||||
return helpersOk && !probedNothing(input.summary) && held && update ? 0 : 1;
|
||||
}
|
||||
|
||||
/** GitHub annotations when running in Actions, plain lines otherwise. */
|
||||
@@ -163,18 +175,23 @@ async function main(): Promise<number> {
|
||||
}
|
||||
|
||||
const summary = summarize(results);
|
||||
const verdict = compareWithBaseline(baseline, summary.failingByMethod);
|
||||
const uncoveredNow = new Set(summary.uncovered.keys());
|
||||
const verdict = compareWithBaseline(baseline, summary.failingByMethod, uncoveredNow);
|
||||
const input: ReportInput = { summary, verdict, helpers, schemaFailures, results };
|
||||
console.log(formatConsole(input));
|
||||
|
||||
const next = lowerBaseline(baseline, summary.failingByMethod);
|
||||
const next = lowerBaseline(baseline, summary.failingByMethod, uncoveredNow);
|
||||
if (args.nextBaseline) writeFileSync(args.nextBaseline, formatBaseline(next));
|
||||
if (args.update) {
|
||||
writeFileSync(BASELINE_PATH, formatBaseline(next));
|
||||
console.log(`Wrote ${path.relative(SERVER_ROOT, BASELINE_PATH)}.`);
|
||||
}
|
||||
if (args.report) {
|
||||
const counts = { failingByMethod: Object.fromEntries(summary.failingByMethod), failuresByCode: Object.fromEntries(summary.failuresByCode) };
|
||||
const counts = {
|
||||
failingByMethod: Object.fromEntries(summary.failingByMethod),
|
||||
uncovered: Object.fromEntries(summary.uncovered),
|
||||
failuresByCode: Object.fromEntries(summary.failuresByCode),
|
||||
};
|
||||
const report = { summary: { ...summary, ...counts }, verdict, helpers, schemaFailures, results };
|
||||
writeFileSync(args.report, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
@@ -193,6 +210,16 @@ async function main(): Promise<number> {
|
||||
'Lower it (--update, or the pg-probe-baseline artifact).',
|
||||
);
|
||||
}
|
||||
for (const method of verdict.newlyUncovered) {
|
||||
annotate(
|
||||
'error',
|
||||
`pg-probe: ${method} ${summary.uncovered.get(method) ?? 'went unmeasured'}, so its SQL was not checked. ` +
|
||||
'Give it arguments the probe can sample, or make it reach its SQL with them.',
|
||||
);
|
||||
}
|
||||
for (const method of verdict.nowCovered) {
|
||||
annotate('error', `pg-probe: ${method} is listed as uncovered but is measured now (or gone). Drop it (--update, or the pg-probe-baseline artifact).`);
|
||||
}
|
||||
if (probedNothing(summary)) annotate('error', `pg-probe: no repository method was called (${summary.methods} planned).`);
|
||||
if (verdict.unseeded) {
|
||||
annotate(
|
||||
|
||||
@@ -7,9 +7,12 @@
|
||||
* date-shaped string when the name says so, an object gets every property
|
||||
* it can sample, an array one element). The values only have to get the
|
||||
* method to send its SQL; what the probe measures is whether Postgres
|
||||
* accepts that SQL against an empty schema. A parameter the sampler cannot
|
||||
* build (a callback, an EntityManager, an entity instance) makes the method
|
||||
* unprobeable, which the report lists instead of guessing.
|
||||
* accepts that SQL against an empty schema. A sample the column cannot take
|
||||
* (a name-shaped string in a date column) is a data error, which the
|
||||
* ratchet does not hold (pg-probe/baseline.ts). A parameter the sampler
|
||||
* cannot build (a callback, an EntityManager, an entity instance) makes the
|
||||
* method unprobeable: the report lists it instead of guessing, and the
|
||||
* baseline's `uncovered` list must name it, so a new one fails the run.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/**
|
||||
* The Postgres probe's ratchet: how many statements of each repository method
|
||||
* Postgres may still refuse.
|
||||
* Postgres may still refuse, and which methods the probe may still leave
|
||||
* unmeasured.
|
||||
*
|
||||
* `scripts/pg-probe-baseline.json` maps `Class.method` to the number of
|
||||
* distinct statements of that method that failed on Postgres when the entry
|
||||
@@ -8,11 +9,27 @@
|
||||
* without one), and an entry above what its method fails now fails the check
|
||||
* too, so the change that fixes a statement lowers the baseline with it.
|
||||
* `--update` lowers entries and drops the ones that reach zero or whose
|
||||
* method is gone; it never raises or adds one. The only exception is the
|
||||
* first run: a baseline whose `failing` is `null` has never been measured,
|
||||
* so there is nothing to compare and the run fails (scripts/pg-probe.ts)
|
||||
* until the first measurement, written by `--update` or taken from the CI
|
||||
* artifact, is committed.
|
||||
* method is gone; it never raises or adds one.
|
||||
*
|
||||
* Only statements Postgres refuses as SQL count here (SQLSTATE classes 42
|
||||
* and 0A, see `countsTowardRatchet`). A value the sample arguments got wrong
|
||||
* (class 22, a sample string in a date column) says nothing about the
|
||||
* dialect and is reported without being held.
|
||||
*
|
||||
* `uncovered` lists the methods whose SQL the probe did not see: the ones it
|
||||
* could not call, the ones that sent no SQL (they threw first, or never
|
||||
* reach the database) and the ones that timed out. A method outside that
|
||||
* list that ends up uncovered (a new method the sampler cannot build
|
||||
* arguments for, or an old one that starts throwing before its first
|
||||
* statement) fails the run, and so does an entry whose method is measured
|
||||
* now or gone, so the list only shrinks and the part of the code the probe
|
||||
* reaches cannot quietly shrink with it.
|
||||
*
|
||||
* The only exception is the first run: a baseline whose `failing` or
|
||||
* `uncovered` is `null` has never been measured, so there is nothing to
|
||||
* compare and the run fails (scripts/pg-probe.ts) until the first
|
||||
* measurement, written by `--update` or taken from the CI artifact, is
|
||||
* committed.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'node:fs';
|
||||
@@ -20,6 +37,8 @@ import { readFileSync } from 'node:fs';
|
||||
export interface ProbeBaseline {
|
||||
/** `Class.method` → failing statements allowed; `null` until the first measurement is written. */
|
||||
failing: Record<string, number> | null;
|
||||
/** `Class.method` names the probe may leave unmeasured; `null` until the first measurement is written. */
|
||||
uncovered: string[] | null;
|
||||
}
|
||||
|
||||
export interface BaselineVerdict {
|
||||
@@ -29,8 +48,25 @@ export interface BaselineVerdict {
|
||||
grown: { method: string; allowed: number; now: number }[];
|
||||
/** Entries above what their method fails now (the method may be gone). */
|
||||
stale: { method: string; allowed: number; now: number }[];
|
||||
/** Methods the probe did not measure that the baseline does not list. */
|
||||
newlyUncovered: string[];
|
||||
/** Entries in `uncovered` whose method is measured now, or gone. */
|
||||
nowCovered: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a refused statement counts toward the ratchet: SQLSTATE class 42
|
||||
* (syntax error, unknown function, operator or column, datatype mismatch)
|
||||
* and class 0A (feature not supported) are what a SQLite-only spelling gets.
|
||||
* Anything else (a data exception such as 22P02 or 22007, a constraint, a
|
||||
* timeout) comes from the sample values or the empty tables, not the SQL.
|
||||
*/
|
||||
export function countsTowardRatchet(code: string): boolean {
|
||||
return /^(42|0A)/.test(code);
|
||||
}
|
||||
|
||||
const METHOD_KEY = /^[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*$/;
|
||||
|
||||
export function parseBaseline(text: string, source: string): ProbeBaseline {
|
||||
let data: unknown;
|
||||
try {
|
||||
@@ -41,14 +77,20 @@ export function parseBaseline(text: string, source: string): ProbeBaseline {
|
||||
if (data === null || typeof data !== 'object' || Array.isArray(data) || !('failing' in data)) {
|
||||
throw new Error(`${source} must be an object with a "failing" key`);
|
||||
}
|
||||
const failing = (data as { failing: unknown }).failing;
|
||||
if (failing === null) return { failing: null };
|
||||
return {
|
||||
failing: parseFailing((data as { failing: unknown }).failing, source),
|
||||
uncovered: parseUncovered((data as { uncovered?: unknown }).uncovered, source),
|
||||
};
|
||||
}
|
||||
|
||||
function parseFailing(failing: unknown, source: string): Record<string, number> | null {
|
||||
if (failing === null) return null;
|
||||
if (typeof failing !== 'object' || Array.isArray(failing)) {
|
||||
throw new Error(`${source}: "failing" must be null or an object of Class.method to counts`);
|
||||
}
|
||||
const entries: Record<string, number> = {};
|
||||
for (const [method, count] of Object.entries(failing as Record<string, unknown>)) {
|
||||
if (!/^[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*$/.test(method)) {
|
||||
if (!METHOD_KEY.test(method)) {
|
||||
throw new Error(`${source}: "${method}" is not a Class.method key`);
|
||||
}
|
||||
if (typeof count !== 'number' || !Number.isInteger(count) || count < 1) {
|
||||
@@ -56,16 +98,40 @@ export function parseBaseline(text: string, source: string): ProbeBaseline {
|
||||
}
|
||||
entries[method] = count;
|
||||
}
|
||||
return { failing: entries };
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** A missing `uncovered` key reads as unmeasured, so a baseline written before the list existed gets it seeded. */
|
||||
function parseUncovered(uncovered: unknown, source: string): string[] | null {
|
||||
if (uncovered === undefined || uncovered === null) return null;
|
||||
if (!Array.isArray(uncovered)) throw new Error(`${source}: "uncovered" must be null or an array of Class.method names`);
|
||||
const seen = new Set<string>();
|
||||
for (const method of uncovered as unknown[]) {
|
||||
if (typeof method !== 'string' || !METHOD_KEY.test(method)) {
|
||||
throw new Error(`${source}: "uncovered" holds ${JSON.stringify(method)}, expected a Class.method name`);
|
||||
}
|
||||
if (seen.has(method)) throw new Error(`${source}: "uncovered" lists ${method} twice`);
|
||||
seen.add(method);
|
||||
}
|
||||
return [...seen];
|
||||
}
|
||||
|
||||
export function readBaseline(path: string): ProbeBaseline {
|
||||
return parseBaseline(readFileSync(path, 'utf8'), path);
|
||||
}
|
||||
|
||||
/** Compares what failed now (`Class.method` → failing statements, zero counts allowed) with the baseline. */
|
||||
export function compareWithBaseline(baseline: ProbeBaseline, failingNow: ReadonlyMap<string, number>): BaselineVerdict {
|
||||
if (baseline.failing === null) return { unseeded: true, grown: [], stale: [] };
|
||||
/**
|
||||
* Compares what failed now (`Class.method` → failing statements, zero counts
|
||||
* allowed) and which methods went unmeasured now with the baseline.
|
||||
*/
|
||||
export function compareWithBaseline(
|
||||
baseline: ProbeBaseline,
|
||||
failingNow: ReadonlyMap<string, number>,
|
||||
uncoveredNow: ReadonlySet<string>,
|
||||
): BaselineVerdict {
|
||||
if (baseline.failing === null || baseline.uncovered === null) {
|
||||
return { unseeded: true, grown: [], stale: [], newlyUncovered: [], nowCovered: [] };
|
||||
}
|
||||
const allowedFor = baseline.failing;
|
||||
const grown: BaselineVerdict['grown'] = [];
|
||||
const stale: BaselineVerdict['stale'] = [];
|
||||
@@ -78,15 +144,22 @@ export function compareWithBaseline(baseline: ProbeBaseline, failingNow: Readonl
|
||||
const now = failingNow.get(method) ?? 0;
|
||||
if (now < allowed) stale.push({ method, allowed, now });
|
||||
}
|
||||
return { unseeded: false, grown, stale };
|
||||
const listed = new Set(baseline.uncovered);
|
||||
const newlyUncovered = [...uncoveredNow].filter((method) => !listed.has(method)).sort((a, b) => a.localeCompare(b));
|
||||
const nowCovered = [...listed].filter((method) => !uncoveredNow.has(method)).sort((a, b) => a.localeCompare(b));
|
||||
return { unseeded: false, grown, stale, newlyUncovered, nowCovered };
|
||||
}
|
||||
|
||||
/**
|
||||
* The baseline `--update` writes: every entry lowered to what its method
|
||||
* fails now, zeros dropped, nothing added. An unseeded baseline is seeded
|
||||
* with everything that fails now.
|
||||
* fails now, zeros dropped, `uncovered` cut to the methods still unmeasured,
|
||||
* nothing added. An unseeded part is seeded with what the run saw.
|
||||
*/
|
||||
export function lowerBaseline(baseline: ProbeBaseline, failingNow: ReadonlyMap<string, number>): ProbeBaseline {
|
||||
export function lowerBaseline(
|
||||
baseline: ProbeBaseline,
|
||||
failingNow: ReadonlyMap<string, number>,
|
||||
uncoveredNow: ReadonlySet<string>,
|
||||
): ProbeBaseline {
|
||||
const next: Record<string, number> = {};
|
||||
if (baseline.failing === null) {
|
||||
for (const [method, now] of failingNow) if (now > 0) next[method] = now;
|
||||
@@ -96,7 +169,11 @@ export function lowerBaseline(baseline: ProbeBaseline, failingNow: ReadonlyMap<s
|
||||
if (now > 0) next[method] = now;
|
||||
}
|
||||
}
|
||||
return { failing: Object.fromEntries(Object.entries(next).sort(([a], [b]) => a.localeCompare(b))) };
|
||||
const uncovered = baseline.uncovered === null ? [...uncoveredNow] : baseline.uncovered.filter((method) => uncoveredNow.has(method));
|
||||
return {
|
||||
failing: Object.fromEntries(Object.entries(next).sort(([a], [b]) => a.localeCompare(b))),
|
||||
uncovered: [...new Set(uncovered)].sort((a, b) => a.localeCompare(b)),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatBaseline(baseline: ProbeBaseline): string {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* Markdown the CI job writes to its step summary.
|
||||
*/
|
||||
|
||||
import type { BaselineVerdict } from './baseline';
|
||||
import { countsTowardRatchet, type BaselineVerdict } from './baseline';
|
||||
import type { HelperResult } from './helper-cases';
|
||||
import type { StatementRecord } from './recorder';
|
||||
|
||||
@@ -23,6 +23,8 @@ export interface FailedStatement {
|
||||
code: string;
|
||||
message: string;
|
||||
sql: string;
|
||||
/** A dialect refusal (SQLSTATE class 42 or 0A) the ratchet holds; anything else is reported only. */
|
||||
counted: boolean;
|
||||
}
|
||||
|
||||
export interface ProbeSummary {
|
||||
@@ -34,9 +36,19 @@ export interface ProbeSummary {
|
||||
timedOut: number;
|
||||
/** Distinct statements per method, summed. */
|
||||
statements: number;
|
||||
/** Every distinct statement Postgres refused, counted toward the ratchet or not. */
|
||||
failedStatements: FailedStatement[];
|
||||
/** `Class.method` → distinct statements Postgres refused; every called method appears, zeros included. */
|
||||
/**
|
||||
* `Class.method` → distinct statements Postgres refused as SQL (SQLSTATE
|
||||
* class 42 or 0A); every called method appears, zeros included.
|
||||
*/
|
||||
failingByMethod: Map<string, number>;
|
||||
/**
|
||||
* `Class.method` → why the probe saw none or not all of its SQL: it could
|
||||
* not call it, it sent no SQL (it threw first, or never reaches the
|
||||
* database), or it timed out. Held to the baseline's `uncovered` list.
|
||||
*/
|
||||
uncovered: Map<string, string>;
|
||||
/** SQLSTATE → distinct failing statements. */
|
||||
failuresByCode: Map<string, number>;
|
||||
}
|
||||
@@ -51,15 +63,22 @@ export function summarize(results: readonly MethodResult[]): ProbeSummary {
|
||||
statements: 0,
|
||||
failedStatements: [],
|
||||
failingByMethod: new Map(),
|
||||
uncovered: new Map(),
|
||||
failuresByCode: new Map(),
|
||||
};
|
||||
for (const result of results) {
|
||||
if (result.unprobeable !== undefined) {
|
||||
summary.unprobeable += 1;
|
||||
summary.uncovered.set(result.key, `could not be called: ${result.unprobeable}`);
|
||||
continue;
|
||||
}
|
||||
summary.called += 1;
|
||||
if (result.timedOut) summary.timedOut += 1;
|
||||
if (result.timedOut) {
|
||||
summary.timedOut += 1;
|
||||
summary.uncovered.set(result.key, 'timed out');
|
||||
} else if (result.statements.length === 0) {
|
||||
summary.uncovered.set(result.key, result.threw === undefined ? 'sent no SQL' : `threw before any statement: ${result.threw}`);
|
||||
}
|
||||
if (result.statements.length === 0) summary.withoutSql += 1;
|
||||
const seen = new Set<string>();
|
||||
const failed = new Map<string, StatementRecord>();
|
||||
@@ -68,17 +87,28 @@ export function summarize(results: readonly MethodResult[]): ProbeSummary {
|
||||
if (!statement.outcome.ok && !failed.has(statement.sql)) failed.set(statement.sql, statement);
|
||||
}
|
||||
summary.statements += seen.size;
|
||||
summary.failingByMethod.set(result.key, failed.size);
|
||||
let counted = 0;
|
||||
for (const statement of failed.values()) {
|
||||
const outcome = statement.outcome;
|
||||
if (!('code' in outcome)) continue;
|
||||
summary.failedStatements.push({ method: result.key, code: outcome.code, message: outcome.message, sql: statement.sql });
|
||||
const dialect = countsTowardRatchet(outcome.code);
|
||||
if (dialect) counted += 1;
|
||||
summary.failedStatements.push({ method: result.key, code: outcome.code, message: outcome.message, sql: statement.sql, counted: dialect });
|
||||
summary.failuresByCode.set(outcome.code, (summary.failuresByCode.get(outcome.code) ?? 0) + 1);
|
||||
}
|
||||
summary.failingByMethod.set(result.key, counted);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
/** Refused statements the ratchet holds (dialect errors) and the ones it only reports. */
|
||||
export function splitRefusals(summary: Pick<ProbeSummary, 'failedStatements'>): { counted: FailedStatement[]; reported: FailedStatement[] } {
|
||||
return {
|
||||
counted: summary.failedStatements.filter((failed) => failed.counted),
|
||||
reported: summary.failedStatements.filter((failed) => !failed.counted),
|
||||
};
|
||||
}
|
||||
|
||||
/** SQL on one line, cut to `max` characters, for a log or a table cell. */
|
||||
export function oneLine(sql: string, max = 160): string {
|
||||
const flat = sql.replace(/\s+/g, ' ').trim();
|
||||
@@ -112,8 +142,10 @@ export function probedNothing(summary: Pick<ProbeSummary, 'called'>): boolean {
|
||||
|
||||
/**
|
||||
* True when the run passes: every helper case on both engines, at least one
|
||||
* repository method called, a measured baseline, and the ratchet held. An
|
||||
* unmeasured baseline fails, so the gate cannot merge before it holds anything.
|
||||
* repository method called, a measured baseline, and the ratchet held (no
|
||||
* method failing more statements than allowed, no new unmeasured method, no
|
||||
* entry left above what the run saw). An unmeasured baseline fails, so the
|
||||
* gate cannot merge before it holds anything.
|
||||
*/
|
||||
export function passed(input: Pick<ReportInput, 'verdict' | 'helpers' | 'summary'>): boolean {
|
||||
return (
|
||||
@@ -121,7 +153,9 @@ export function passed(input: Pick<ReportInput, 'verdict' | 'helpers' | 'summary
|
||||
!probedNothing(input.summary) &&
|
||||
!input.verdict.unseeded &&
|
||||
input.verdict.grown.length === 0 &&
|
||||
input.verdict.stale.length === 0
|
||||
input.verdict.stale.length === 0 &&
|
||||
input.verdict.newlyUncovered.length === 0 &&
|
||||
input.verdict.nowCovered.length === 0
|
||||
);
|
||||
}
|
||||
|
||||
@@ -139,14 +173,22 @@ export function formatConsole(input: ReportInput): string {
|
||||
`Repositories: ${summary.methods} methods, ${summary.called} called, ${summary.unprobeable} unprobeable, ` +
|
||||
`${summary.withoutSql} sent no SQL, ${summary.timedOut} timed out.`,
|
||||
);
|
||||
lines.push(`Statements: ${summary.statements} distinct, ${summary.failedStatements.length} refused by Postgres.`);
|
||||
const refusals = splitRefusals(summary);
|
||||
lines.push(
|
||||
`Statements: ${summary.statements} distinct, ${summary.failedStatements.length} refused by Postgres ` +
|
||||
`(${refusals.counted.length} as SQL, held to the baseline; ${refusals.reported.length} over a sample value or the empty tables, reported only).`,
|
||||
);
|
||||
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) {
|
||||
lines.push(` ${code}: ${count}`);
|
||||
lines.push(` ${code}: ${count}${countsTowardRatchet(code) ? '' : ' (not held)'}`);
|
||||
}
|
||||
for (const failed of summary.failedStatements) {
|
||||
lines.push(` ${failed.method} ${failed.code} ${failed.message}`);
|
||||
lines.push(` ${failed.method} ${failed.code} ${failed.message}${failed.counted ? '' : ' (not held)'}`);
|
||||
lines.push(` ${oneLine(failed.sql)}`);
|
||||
}
|
||||
if (summary.uncovered.size > 0) {
|
||||
lines.push(`Unmeasured: ${summary.uncovered.size} method(s) whose SQL the probe did not see:`);
|
||||
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) lines.push(` ${method}: ${why}`);
|
||||
}
|
||||
if (probedNothing(summary)) {
|
||||
lines.push(`FAIL No repository method was called (${summary.methods} planned); there is nothing to hold to the baseline.`);
|
||||
}
|
||||
@@ -162,6 +204,15 @@ export function formatConsole(input: ReportInput): string {
|
||||
for (const entry of verdict.stale) {
|
||||
lines.push(`FAIL ${entry.method} is held at ${entry.allowed} failing statement(s) but fails ${entry.now} now; lower the baseline.`);
|
||||
}
|
||||
for (const method of verdict.newlyUncovered) {
|
||||
lines.push(
|
||||
`FAIL ${method} ${summary.uncovered.get(method) ?? 'went unmeasured'}, and the baseline does not list it as uncovered. ` +
|
||||
'Give it arguments the probe can sample, or make it reach its SQL with them.',
|
||||
);
|
||||
}
|
||||
for (const method of verdict.nowCovered) {
|
||||
lines.push(`FAIL ${method} is listed as uncovered but is measured now (or gone); drop it from the baseline.`);
|
||||
}
|
||||
lines.push(passed(input) ? 'Postgres probe passed.' : 'Postgres probe failed.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -174,9 +225,12 @@ export function formatMarkdown(input: ReportInput): string {
|
||||
out.push('| | |', '|---|---|');
|
||||
out.push(`| Dialect helper cases (SQLite and Postgres) | ${input.helpers.length - failures.length} of ${input.helpers.length} pass |`);
|
||||
out.push(`| Schema statements refused | ${input.schemaFailures.length} |`);
|
||||
const refusals = splitRefusals(summary);
|
||||
out.push(`| Repository methods | ${summary.methods} (${summary.called} called, ${summary.unprobeable} unprobeable) |`);
|
||||
out.push(`| Methods whose SQL the probe did not see | ${summary.uncovered.size} |`);
|
||||
out.push(`| Distinct statements | ${summary.statements} |`);
|
||||
out.push(`| Refused by Postgres | ${summary.failedStatements.length} |`);
|
||||
out.push(`| Refused by Postgres as SQL (held) | ${refusals.counted.length} |`);
|
||||
out.push(`| Refused over a sample value or the empty tables (reported only) | ${refusals.reported.length} |`);
|
||||
out.push('');
|
||||
if (probedNothing(summary)) out.push('No repository method was called, so there is nothing to hold to the baseline.', '');
|
||||
if (verdict.unseeded) {
|
||||
@@ -190,6 +244,12 @@ export function formatMarkdown(input: ReportInput): string {
|
||||
for (const entry of [...verdict.grown, ...verdict.stale]) out.push(`| ${cell(entry.method)} | ${entry.allowed} | ${entry.now} |`);
|
||||
out.push('');
|
||||
}
|
||||
if (verdict.newlyUncovered.length > 0 || verdict.nowCovered.length > 0) {
|
||||
out.push('### Coverage ratchet', '', '| Method | Baseline | Now |', '|---|---|---|');
|
||||
for (const method of verdict.newlyUncovered) out.push(`| ${cell(method)} | measured | ${cell(summary.uncovered.get(method) ?? 'unmeasured')} |`);
|
||||
for (const method of verdict.nowCovered) out.push(`| ${cell(method)} | uncovered | measured or gone |`);
|
||||
out.push('');
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
out.push('### Helper cases that failed', '', '| Engine | Case | Why |', '|---|---|---|');
|
||||
for (const failure of failures) out.push(`| ${failure.engine} | ${cell(failure.name)} | ${cell(failure.failure ?? '')} |`);
|
||||
@@ -199,14 +259,15 @@ export function formatMarkdown(input: ReportInput): string {
|
||||
out.push('### Refused statements by SQLSTATE', '', '| SQLSTATE | Statements |', '|---|---|');
|
||||
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) out.push(`| ${code} | ${count} |`);
|
||||
out.push('');
|
||||
out.push('<details><summary>Every refused statement</summary>', '', '| Method | SQLSTATE | Message |', '|---|---|---|');
|
||||
for (const failed of summary.failedStatements) out.push(`| ${cell(failed.method)} | ${failed.code} | ${cell(failed.message)} |`);
|
||||
out.push('<details><summary>Every refused statement</summary>', '', '| Method | SQLSTATE | Held | Message |', '|---|---|---|---|');
|
||||
for (const failed of summary.failedStatements) {
|
||||
out.push(`| ${cell(failed.method)} | ${failed.code} | ${failed.counted ? 'yes' : 'no'} | ${cell(failed.message)} |`);
|
||||
}
|
||||
out.push('', '</details>', '');
|
||||
}
|
||||
const unprobeable = input.results.filter((result) => result.unprobeable !== undefined);
|
||||
if (unprobeable.length > 0) {
|
||||
out.push('<details><summary>Methods the probe could not call</summary>', '', '| Method | Why |', '|---|---|');
|
||||
for (const result of unprobeable) out.push(`| ${cell(result.key)} | ${cell(result.unprobeable ?? '')} |`);
|
||||
if (summary.uncovered.size > 0) {
|
||||
out.push('<details><summary>Methods whose SQL the probe did not see</summary>', '', '| Method | Why |', '|---|---|');
|
||||
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) out.push(`| ${cell(method)} | ${cell(why)} |`);
|
||||
out.push('', '</details>', '');
|
||||
}
|
||||
return `${out.join('\n')}\n`;
|
||||
|
||||
Reference in New Issue
Block a user