mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-21 14:36:31 +00:00
ci(server): hold unmeasured methods and only SQL errors to the probe baseline
The Postgres probe only ratcheted the methods it managed to call. A method it could not build arguments for, one that threw before its first statement and one that timed out were counted and logged but held to nothing, so a new repository method with a callback parameter could land SQLite-only SQL unseen, and the part of the code the probe reaches could shrink without anyone noticing. pg-probe-baseline.json gains an `uncovered` list of those methods. A method outside it that goes unmeasured fails the run, and so does an entry whose method is measured now or gone, so the list only shrinks. It is seeded from the same first measurement as `failing`; a baseline without the key reads as unmeasured. The failing count also counted every refusal, including data errors the sample values cause (a "probe" string in a date column, SQLSTATE 22007), so renaming a parameter could fail the gate on a change that never touched the SQL. Only classes 42 and 0A count now; the rest is reported in the log and the step summary without being held. The docs said the probe measures every repository statement. They now say it measures what each method reaches with sample arguments against empty tables, and that the methods it cannot reach are held to the list.
This commit is contained in:
Maurice
@@ -2,21 +2,29 @@
|
||||
* The Postgres probe's ratchet (scripts/pg-probe/baseline.ts). The probe
|
||||
* itself only runs in CI against a Postgres service, so the rules that decide
|
||||
* whether that job passes are pinned here: a method may not fail more
|
||||
* statements than its entry, a stale entry fails until it is lowered, an
|
||||
* unmeasured baseline has nothing to compare (the run fails on it, see
|
||||
* report.test.ts), and --update never raises or adds.
|
||||
* statements than its entry, a stale entry fails until it is lowered, a
|
||||
* method the probe did not measure must be listed as uncovered and the list
|
||||
* only shrinks, only SQL errors count, an unmeasured baseline has nothing to
|
||||
* compare (the run fails on it, see report.test.ts), and --update never
|
||||
* raises or adds.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { compareWithBaseline, formatBaseline, lowerBaseline, parseBaseline } from '../../../../scripts/pg-probe/baseline';
|
||||
import { compareWithBaseline, countsTowardRatchet, formatBaseline, lowerBaseline, parseBaseline } from '../../../../scripts/pg-probe/baseline';
|
||||
|
||||
const now = (entries: Record<string, number>): Map<string, number> => new Map(Object.entries(entries));
|
||||
const none = new Set<string>();
|
||||
|
||||
describe('pg-probe baseline', () => {
|
||||
it('PGPROBE-001: parses a measured and an unmeasured baseline', () => {
|
||||
expect(parseBaseline('{"failing": null}', 'b.json')).toEqual({ failing: null });
|
||||
expect(parseBaseline('{"failing": {"TagsRepository.listByUser": 2}}', 'b.json')).toEqual({ failing: { 'TagsRepository.listByUser': 2 } });
|
||||
it('PGPROBE-001: parses a measured and an unmeasured baseline; a missing uncovered list reads as unmeasured', () => {
|
||||
expect(parseBaseline('{"failing": null, "uncovered": null}', 'b.json')).toEqual({ failing: null, uncovered: null });
|
||||
expect(parseBaseline('{"failing": null}', 'b.json')).toEqual({ failing: null, uncovered: null });
|
||||
expect(parseBaseline('{"failing": {"TagsRepository.listByUser": 2}, "uncovered": ["A.b"]}', 'b.json')).toEqual({
|
||||
failing: { 'TagsRepository.listByUser': 2 },
|
||||
uncovered: ['A.b'],
|
||||
});
|
||||
expect(parseBaseline('{"failing": {}}', 'b.json')).toEqual({ failing: {}, uncovered: null });
|
||||
});
|
||||
|
||||
it('PGPROBE-002: refuses a malformed baseline instead of reading it as empty', () => {
|
||||
@@ -27,14 +35,21 @@ describe('pg-probe baseline', () => {
|
||||
expect(() => parseBaseline('{"failing": {"listByUser": 1}}', 'b.json')).toThrow(/not a Class.method key/);
|
||||
expect(() => parseBaseline('{"failing": {"A.b": 0}}', 'b.json')).toThrow(/expected a positive integer/);
|
||||
expect(() => parseBaseline('{"failing": {"A.b": 1.5}}', 'b.json')).toThrow(/expected a positive integer/);
|
||||
expect(() => parseBaseline('{"failing": {}, "uncovered": {}}', 'b.json')).toThrow(/"uncovered" must be null or an array/);
|
||||
expect(() => parseBaseline('{"failing": {}, "uncovered": ["nodot"]}', 'b.json')).toThrow(/expected a Class.method name/);
|
||||
expect(() => parseBaseline('{"failing": {}, "uncovered": [1]}', 'b.json')).toThrow(/expected a Class.method name/);
|
||||
expect(() => parseBaseline('{"failing": {}, "uncovered": ["A.b", "A.b"]}', 'b.json')).toThrow(/lists A.b twice/);
|
||||
});
|
||||
|
||||
it('PGPROBE-003: an unmeasured baseline is flagged instead of compared', () => {
|
||||
expect(compareWithBaseline({ failing: null }, now({ 'A.b': 3 }))).toEqual({ unseeded: true, grown: [], stale: [] });
|
||||
it('PGPROBE-003: an unmeasured baseline (either part null) is flagged instead of compared', () => {
|
||||
const flagged = { unseeded: true, grown: [], stale: [], newlyUncovered: [], nowCovered: [] };
|
||||
expect(compareWithBaseline({ failing: null, uncovered: null }, now({ 'A.b': 3 }), new Set(['C.d']))).toEqual(flagged);
|
||||
expect(compareWithBaseline({ failing: { 'A.b': 1 }, uncovered: null }, now({ 'A.b': 3 }), none)).toEqual(flagged);
|
||||
expect(compareWithBaseline({ failing: null, uncovered: [] }, now({ 'A.b': 3 }), none)).toEqual(flagged);
|
||||
});
|
||||
|
||||
it('PGPROBE-004: a method failing more statements than its entry (or any without one) fails', () => {
|
||||
const verdict = compareWithBaseline({ failing: { 'A.b': 1 } }, now({ 'A.b': 2, 'C.d': 1, 'E.f': 0 }));
|
||||
const verdict = compareWithBaseline({ failing: { 'A.b': 1 }, uncovered: [] }, now({ 'A.b': 2, 'C.d': 1, 'E.f': 0 }), none);
|
||||
expect(verdict.grown).toEqual([
|
||||
{ method: 'A.b', allowed: 1, now: 2 },
|
||||
{ method: 'C.d', allowed: 0, now: 1 },
|
||||
@@ -43,7 +58,7 @@ describe('pg-probe baseline', () => {
|
||||
});
|
||||
|
||||
it('PGPROBE-005: an entry above what its method fails now is stale, also when the method is gone', () => {
|
||||
const verdict = compareWithBaseline({ failing: { 'A.b': 2, 'Gone.away': 1 } }, now({ 'A.b': 1 }));
|
||||
const verdict = compareWithBaseline({ failing: { 'A.b': 2, 'Gone.away': 1 }, uncovered: [] }, now({ 'A.b': 1 }), none);
|
||||
expect(verdict.grown).toEqual([]);
|
||||
expect(verdict.stale).toEqual([
|
||||
{ method: 'A.b', allowed: 2, now: 1 },
|
||||
@@ -51,13 +66,45 @@ describe('pg-probe baseline', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('PGPROBE-006: --update lowers and drops entries but never raises or adds one', () => {
|
||||
const lowered = lowerBaseline({ failing: { 'A.b': 2, 'C.d': 1, 'Gone.away': 1 } }, now({ 'A.b': 1, 'C.d': 5, 'New.one': 3 }));
|
||||
expect(lowered).toEqual({ failing: { 'A.b': 1, 'C.d': 1 } });
|
||||
it('PGPROBE-009: a method the probe did not measure must be listed, and a listed method measured now or gone is stale', () => {
|
||||
const verdict = compareWithBaseline(
|
||||
{ failing: {}, uncovered: ['Kept.out', 'Now.measured', 'Gone.away'] },
|
||||
now({ 'Now.measured': 0 }),
|
||||
new Set(['Kept.out', 'New.callback', 'Old.throws']),
|
||||
);
|
||||
expect(verdict).toEqual({
|
||||
unseeded: false,
|
||||
grown: [],
|
||||
stale: [],
|
||||
newlyUncovered: ['New.callback', 'Old.throws'],
|
||||
nowCovered: ['Gone.away', 'Now.measured'],
|
||||
});
|
||||
expect(compareWithBaseline({ failing: {}, uncovered: ['A.b'] }, now({}), new Set(['A.b']))).toMatchObject({ newlyUncovered: [], nowCovered: [] });
|
||||
});
|
||||
|
||||
it('PGPROBE-007: the first --update seeds an unmeasured baseline with what fails, sorted', () => {
|
||||
expect(lowerBaseline({ failing: null }, now({ 'Z.z': 1, 'A.a': 2, 'M.m': 0 }))).toEqual({ failing: { 'A.a': 2, 'Z.z': 1 } });
|
||||
it('PGPROBE-006: --update lowers and drops entries but never raises or adds one, in either part', () => {
|
||||
const lowered = lowerBaseline(
|
||||
{ failing: { 'A.b': 2, 'C.d': 1, 'Gone.away': 1 }, uncovered: ['Kept.out', 'Now.measured'] },
|
||||
now({ 'A.b': 1, 'C.d': 5, 'New.one': 3 }),
|
||||
new Set(['Kept.out', 'New.callback']),
|
||||
);
|
||||
expect(lowered).toEqual({ failing: { 'A.b': 1, 'C.d': 1 }, uncovered: ['Kept.out'] });
|
||||
});
|
||||
|
||||
it('PGPROBE-007: the first --update seeds an unmeasured baseline with what fails and what went unmeasured, sorted', () => {
|
||||
expect(lowerBaseline({ failing: null, uncovered: null }, now({ 'Z.z': 1, 'A.a': 2, 'M.m': 0 }), new Set(['Y.y', 'B.b']))).toEqual({
|
||||
failing: { 'A.a': 2, 'Z.z': 1 },
|
||||
uncovered: ['B.b', 'Y.y'],
|
||||
});
|
||||
expect(lowerBaseline({ failing: { 'A.a': 3 }, uncovered: null }, now({ 'A.a': 2, 'N.n': 1 }), new Set(['B.b']))).toEqual({
|
||||
failing: { 'A.a': 2 },
|
||||
uncovered: ['B.b'],
|
||||
});
|
||||
});
|
||||
|
||||
it('PGPROBE-010: only syntax and feature errors count toward the ratchet, not data errors from the sample values', () => {
|
||||
for (const code of ['42601', '42883', '42804', '42P01', '42703', '0A000']) expect(countsTowardRatchet(code)).toBe(true);
|
||||
for (const code of ['22P02', '22007', '22008', '23505', '57014', 'NOSQLSTATE']) expect(countsTowardRatchet(code)).toBe(false);
|
||||
});
|
||||
|
||||
it('PGPROBE-008: the checked-in baseline parses and formats back to itself', () => {
|
||||
|
||||
@@ -11,7 +11,17 @@ import { describe, expect, it } from 'vitest';
|
||||
import { exitCode, parseArgs, repositoryFiles } from '../../../../scripts/pg-probe';
|
||||
import type { BaselineVerdict } from '../../../../scripts/pg-probe/baseline';
|
||||
import type { HelperResult } from '../../../../scripts/pg-probe/helper-cases';
|
||||
import { formatConsole, formatMarkdown, oneLine, passed, probedNothing, summarize, type MethodResult, type ReportInput } from '../../../../scripts/pg-probe/report';
|
||||
import {
|
||||
formatConsole,
|
||||
formatMarkdown,
|
||||
oneLine,
|
||||
passed,
|
||||
probedNothing,
|
||||
splitRefusals,
|
||||
summarize,
|
||||
type MethodResult,
|
||||
type ReportInput,
|
||||
} from '../../../../scripts/pg-probe/report';
|
||||
|
||||
const ok = (sql: string) => ({ sql, outcome: { ok: true as const } });
|
||||
const refused = (sql: string, code = '42883', message = 'function datetime(unknown) does not exist') => ({
|
||||
@@ -25,9 +35,12 @@ const RESULTS: MethodResult[] = [
|
||||
{ key: 'C.silent', statements: [] },
|
||||
{ key: 'D.hung', statements: [ok('select 4')], timedOut: true },
|
||||
{ key: 'E.skipped', statements: [], unprobeable: 'fn: a function parameter' },
|
||||
{ key: 'F.thrower', statements: [], threw: 'no trip 1' },
|
||||
{ key: 'G.sampled', statements: [refused('select $1::date', '22007', 'invalid input syntax for type date: "probe"'), ok('select 5')] },
|
||||
];
|
||||
|
||||
const CLEAN: BaselineVerdict = { unseeded: false, grown: [], stale: [] };
|
||||
const CLEAN: BaselineVerdict = { unseeded: false, grown: [], stale: [], newlyUncovered: [], nowCovered: [] };
|
||||
const UNSEEDED: BaselineVerdict = { ...CLEAN, unseeded: true };
|
||||
const helper = (failure: string | null): HelperResult => ({ engine: 'postgres', name: 'dateOf', failure });
|
||||
|
||||
function input(overrides: Partial<ReportInput> = {}): ReportInput {
|
||||
@@ -37,10 +50,34 @@ function input(overrides: Partial<ReportInput> = {}): ReportInput {
|
||||
describe('pg-probe report', () => {
|
||||
it('PGPROBE-060: counts methods and distinct statements, and what Postgres refused per method and SQLSTATE', () => {
|
||||
const summary = summarize(RESULTS);
|
||||
expect(summary).toMatchObject({ methods: 5, called: 4, unprobeable: 1, withoutSql: 1, timedOut: 1, statements: 6 });
|
||||
expect(summary.failingByMethod).toEqual(new Map([['A.clean', 0], ['B.broken', 2], ['C.silent', 0], ['D.hung', 0]]));
|
||||
expect(summary.failuresByCode).toEqual(new Map([['42883', 1], ['42601', 1]]));
|
||||
expect(summary.failedStatements.map((f) => `${f.method} ${f.code}`)).toEqual(['B.broken 42883', 'B.broken 42601']);
|
||||
expect(summary).toMatchObject({ methods: 7, called: 6, unprobeable: 1, withoutSql: 2, timedOut: 1, statements: 8 });
|
||||
expect(summary.failingByMethod).toEqual(
|
||||
new Map([['A.clean', 0], ['B.broken', 2], ['C.silent', 0], ['D.hung', 0], ['F.thrower', 0], ['G.sampled', 0]]),
|
||||
);
|
||||
expect(summary.failuresByCode).toEqual(new Map([['42883', 1], ['42601', 1], ['22007', 1]]));
|
||||
expect(summary.failedStatements.map((f) => `${f.method} ${f.code} ${f.counted}`)).toEqual([
|
||||
'B.broken 42883 true',
|
||||
'B.broken 42601 true',
|
||||
'G.sampled 22007 false',
|
||||
]);
|
||||
});
|
||||
|
||||
it('PGPROBE-068: a data error from a sample value is reported but not held to the baseline', () => {
|
||||
const summary = summarize([{ key: 'G.sampled', statements: [refused('select $1::date', '22P02', 'bad'), refused('select x', '0A000', 'nope')] }]);
|
||||
expect(summary.failingByMethod.get('G.sampled')).toBe(1);
|
||||
expect(splitRefusals(summary).counted.map((f) => f.code)).toEqual(['0A000']);
|
||||
expect(splitRefusals(summary).reported.map((f) => f.code)).toEqual(['22P02']);
|
||||
});
|
||||
|
||||
it('PGPROBE-069: names every method whose SQL the probe did not see, and why', () => {
|
||||
expect(summarize(RESULTS).uncovered).toEqual(
|
||||
new Map([
|
||||
['C.silent', 'sent no SQL'],
|
||||
['D.hung', 'timed out'],
|
||||
['E.skipped', 'could not be called: fn: a function parameter'],
|
||||
['F.thrower', 'threw before any statement: no trip 1'],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('PGPROBE-061: passes only with every helper case green and the ratchet held', () => {
|
||||
@@ -48,14 +85,16 @@ describe('pg-probe report', () => {
|
||||
expect(passed(input({ helpers: [helper('expected 1, got 2')] }))).toBe(false);
|
||||
expect(passed(input({ verdict: { ...CLEAN, grown: [{ method: 'B.broken', allowed: 1, now: 2 }] } }))).toBe(false);
|
||||
expect(passed(input({ verdict: { ...CLEAN, stale: [{ method: 'B.broken', allowed: 3, now: 2 }] } }))).toBe(false);
|
||||
expect(passed(input({ verdict: { ...CLEAN, newlyUncovered: ['E.skipped'] } }))).toBe(false);
|
||||
expect(passed(input({ verdict: { ...CLEAN, nowCovered: ['Z.gone'] } }))).toBe(false);
|
||||
});
|
||||
|
||||
it('PGPROBE-066: an unmeasured baseline fails the run, and only --update (which seeds it) passes', () => {
|
||||
const unseeded = input({ verdict: { unseeded: true, grown: [], stale: [] } });
|
||||
const unseeded = input({ verdict: UNSEEDED });
|
||||
expect(passed(unseeded)).toBe(false);
|
||||
expect(exitCode(unseeded, false)).toBe(1);
|
||||
expect(exitCode(unseeded, true)).toBe(0);
|
||||
expect(exitCode(input({ verdict: { unseeded: true, grown: [], stale: [] }, helpers: [helper('threw: boom')] }), true)).toBe(1);
|
||||
expect(exitCode(input({ verdict: UNSEEDED, helpers: [helper('threw: boom')] }), true)).toBe(1);
|
||||
});
|
||||
|
||||
it('PGPROBE-067: a run that called no repository method fails, also under --update', () => {
|
||||
@@ -67,7 +106,7 @@ describe('pg-probe report', () => {
|
||||
expect(passed(empty)).toBe(false);
|
||||
expect(exitCode(empty, false)).toBe(1);
|
||||
expect(exitCode(empty, true)).toBe(1);
|
||||
expect(exitCode(input({ summary, results, verdict: { unseeded: true, grown: [], stale: [] } }), true)).toBe(1);
|
||||
expect(exitCode(input({ summary, results, verdict: UNSEEDED }), true)).toBe(1);
|
||||
}
|
||||
expect(probedNothing(summarize(RESULTS))).toBe(false);
|
||||
});
|
||||
@@ -81,22 +120,41 @@ describe('pg-probe report', () => {
|
||||
expect(exitCode(input(), false)).toBe(0);
|
||||
});
|
||||
|
||||
it('PGPROBE-072: --update forgives an uncovered entry that is measured now, never a newly unmeasured method', () => {
|
||||
const covered = input({ verdict: { ...CLEAN, nowCovered: ['Z.gone'] } });
|
||||
expect(exitCode(covered, false)).toBe(1);
|
||||
expect(exitCode(covered, true)).toBe(0);
|
||||
const uncovered = input({ verdict: { ...CLEAN, newlyUncovered: ['E.skipped'] } });
|
||||
expect(exitCode(uncovered, false)).toBe(1);
|
||||
expect(exitCode(uncovered, true)).toBe(1);
|
||||
});
|
||||
|
||||
it('PGPROBE-063: the console log names every refused statement and every ratchet failure', () => {
|
||||
const text = formatConsole(
|
||||
input({
|
||||
helpers: [helper('expected 1, got 2')],
|
||||
verdict: { unseeded: false, grown: [{ method: 'B.broken', allowed: 1, now: 2 }], stale: [{ method: 'Z.gone', allowed: 1, now: 0 }] },
|
||||
verdict: {
|
||||
...CLEAN,
|
||||
grown: [{ method: 'B.broken', allowed: 1, now: 2 }],
|
||||
stale: [{ method: 'Z.gone', allowed: 1, now: 0 }],
|
||||
newlyUncovered: ['F.thrower'],
|
||||
nowCovered: ['Y.measured'],
|
||||
},
|
||||
schemaFailures: [{ statement: 'create table "x" (a text collate "NOCASE")', message: 'collation "NOCASE" does not exist' }],
|
||||
}),
|
||||
);
|
||||
expect(text).toContain('FAIL [postgres] dateOf: expected 1, got 2');
|
||||
expect(text).toContain('collation "NOCASE" does not exist');
|
||||
expect(text).toContain('Statements: 6 distinct, 2 refused by Postgres.');
|
||||
expect(text).toContain('Statements: 8 distinct, 3 refused by Postgres (2 as SQL, held to the baseline; 1 over a sample value');
|
||||
expect(text).toContain(' 22007: 1 (not held)');
|
||||
expect(text).toContain(' F.thrower: threw before any statement: no trip 1');
|
||||
expect(text).toContain('FAIL F.thrower threw before any statement: no trip 1, and the baseline does not list it as uncovered.');
|
||||
expect(text).toContain('FAIL Y.measured is listed as uncovered but is measured now (or gone); drop it from the baseline.');
|
||||
expect(text).toContain('B.broken 42883 function datetime(unknown) does not exist');
|
||||
expect(text).toContain('FAIL B.broken sends 2 statement(s) Postgres refuses, 1 allowed.');
|
||||
expect(text).toContain('FAIL Z.gone is held at 1 failing statement(s) but fails 0 now; lower the baseline.');
|
||||
expect(text.trim().endsWith('Postgres probe failed.')).toBe(true);
|
||||
const unseeded = formatConsole(input({ verdict: { unseeded: true, grown: [], stale: [] } }));
|
||||
const unseeded = formatConsole(input({ verdict: UNSEEDED }));
|
||||
expect(unseeded).toContain('FAIL Baseline: not measured yet');
|
||||
expect(unseeded).toContain('as scripts/pg-probe-baseline.json');
|
||||
expect(unseeded.trim().endsWith('Postgres probe failed.')).toBe(true);
|
||||
@@ -109,8 +167,15 @@ describe('pg-probe report', () => {
|
||||
expect(md).toContain('Result: **failed**');
|
||||
expect(md).toContain('| postgres | a\\|b | got \'x\' |');
|
||||
expect(md).toContain('| 42883 | 1 |');
|
||||
expect(md).toContain('| E.skipped | fn: a function parameter |');
|
||||
const unseeded = formatMarkdown(input({ verdict: { unseeded: true, grown: [], stale: [] } }));
|
||||
expect(md).toContain('| E.skipped | could not be called: fn: a function parameter |');
|
||||
expect(md).toContain('| Refused by Postgres as SQL (held) | 2 |');
|
||||
expect(md).toContain('| Refused over a sample value or the empty tables (reported only) | 1 |');
|
||||
expect(md).toContain('| G.sampled | 22007 | no | invalid input syntax for type date: "probe" |');
|
||||
const ratchet = formatMarkdown(input({ verdict: { ...CLEAN, newlyUncovered: ['C.silent'], nowCovered: ['Z.gone'] } }));
|
||||
expect(ratchet).toContain('### Coverage ratchet');
|
||||
expect(ratchet).toContain('| C.silent | measured | sent no SQL |');
|
||||
expect(ratchet).toContain('| Z.gone | uncovered | measured or gone |');
|
||||
const unseeded = formatMarkdown(input({ verdict: UNSEEDED }));
|
||||
expect(unseeded).toContain('Result: **failed**');
|
||||
expect(unseeded).toContain('has not been measured yet, which fails the run');
|
||||
expect(formatMarkdown(input({ summary: summarize([]), results: [] }))).toContain('No repository method was called');
|
||||
|
||||
Reference in New Issue
Block a user