test(ci): fail every sliced job whose slice count the gate cannot hold to one

A sliced job states its slice count three times: the matrix shard axis,
the i/n in its name and the --shard=i/n it hands vitest. Changing one and
not the others still passes and leaves part of the suite in no slice or
reports the wrong count. workflow-jobs.test.mjs now treats every job that
names matrix.shard or runs --shard as sliced and fails, naming the exact
fix, until the three agree. A job whose axis is not the one-line
"shard: [1, ..., n]" form fails too, instead of going unchecked, and so
does a fixed --shard. Tests on small inline workflows hold the reader
itself to these cases.
This commit is contained in:
Maurice
2026-11-18 13:20:08 +01:00
parent 81628d78ee
commit 911bd825c1
+119 -2
View File
@@ -9,10 +9,16 @@ import { fileURLToPath } from 'node:url';
// layout notes at the top of test.yml).
const workflow = readFileSync(fileURLToPath(new URL('../../.github/workflows/test.yml', import.meta.url)), 'utf8');
const NOT_REQUIRED = new Set(['ci-ok', 'scan']);
const SHARD = '${{ matrix.shard }}';
function jobs(source = workflow) {
const body = source.slice(source.indexOf('\njobs:\n'));
const starts = [...body.matchAll(/^ {2}([a-z0-9-]+):\s*$/gm)];
return starts.map((match, i) => ({ id: match[1], text: body.slice(match.index, starts[i + 1]?.index) }));
}
function jobIds() {
const body = workflow.slice(workflow.indexOf('\njobs:\n'));
return [...body.matchAll(/^ {2}([a-z0-9-]+):\s*$/gm)].map((match) => match[1]);
return jobs().map((job) => job.id);
}
function ciOkNeeds() {
@@ -24,6 +30,62 @@ function ciOkNeeds() {
.map((line) => line.replace(/^\s*- /, ''));
}
// A sliced job states its slice count three times: the shard axis of its
// matrix, the "<i>/<n>" in its name and the --shard=<i>/<n> it hands vitest.
// Changing one and not the others passes and still goes wrong: a divisor
// above the axis leaves part of the suite in no slice, one below it runs a
// slice vitest refuses, and a stale name reports the wrong count. Every job
// that names matrix.shard or runs --shard counts as sliced, so a job whose
// axis is written in a form this reader does not know fails instead of
// going unchecked.
function slicedJobs(source = workflow) {
return jobs(source)
.map(({ id, text }) => ({ id, text: text.replace(/^\s*#.*\n/gm, '') }))
.filter(({ text }) => /matrix\.shard|--shard/.test(text))
.map(({ id, text }) => {
const axis = /^ {8}shard: \[([^\]]*)\]\s*$/m.exec(text);
return {
id,
shards: axis ? axis[1].split(',').map((value) => Number(value.trim())) : null,
name: /^ {4}name: (.+)$/m.exec(text)?.[1] ?? '',
shardArgs: [...text.matchAll(/--shard(?:=|\s+)(\$\{\{ matrix\.shard \}\}\/\d+|\S+)/g)].map((match) => match[1]),
};
});
}
/** What is wrong with one sliced job, each said as the fix to make; empty when the three counts agree. */
function sliceProblems(job) {
if (!job.shards) {
return [
`${job.id}: it uses matrix.shard or --shard, but its matrix has no one-line axis "shard: [1, ..., n]". Write the axis in that form, or teach slicedJobs() in scripts/ci/workflow-jobs.test.mjs the new form (a matrix include with a count per entry, say) in the same change`,
];
}
const n = job.shards.length;
const oneToN = Array.from({ length: n }, (_, i) => i + 1);
const expected = `${SHARD}/${n}`;
const problems = [];
if (job.shards.join() !== oneToN.join()) problems.push(`${job.id}: write the shard axis as [${oneToN.join(', ')}]`);
if (
!job.name
.split(expected)
.slice(1)
.some((rest) => !/^\d/.test(rest))
) {
problems.push(
`${job.id}: the shard axis has ${n} slices, so the job name must say "${expected}" (it reads ${job.name})`,
);
}
if (job.shardArgs.length === 0) problems.push(`${job.id}: no step runs vitest with --shard=${expected}`);
for (const arg of job.shardArgs) {
if (arg !== expected) {
problems.push(
`${job.id}: the shard axis has ${n} slices but a step runs --shard=${arg}; write --shard=${expected}, and change the axis, the name and every --shard together`,
);
}
}
return problems;
}
describe('test.yml', () => {
it('lists every job but the scan in ci-ok needs', () => {
const needs = new Set(ciOkNeeds());
@@ -36,4 +98,59 @@ describe('test.yml', () => {
const unknown = ciOkNeeds().filter((id) => !ids.has(id));
assert.deepEqual(unknown, [], `ci-ok needs jobs test.yml does not define: ${unknown.join(', ')}`);
});
it('finds the sliced jobs', () => {
const ids = slicedJobs().map((job) => job.id);
for (const id of ['server-postgres', 'server-tests', 'client-tests']) {
assert.ok(
ids.includes(id),
`${id} no longer names matrix.shard or --shard; when it stopped being sliced, drop it from this list in scripts/ci/workflow-jobs.test.mjs`,
);
}
});
for (const job of slicedJobs()) {
it(`slices ${job.id} alike in its matrix, its name and its --shard`, () => {
assert.deepEqual(sliceProblems(job), []);
});
}
});
describe('the slice reader', () => {
const run = (arg) => ` steps:\n - run: npx vitest run --shard=${arg}\n`;
const workflowWith = (job) => `name: t\njobs:\n sliced:\n${job}`;
const matrixJob = (axis, name, arg) =>
workflowWith(` name: '${name}'\n strategy:\n matrix:\n${axis}${run(arg)}`);
const problems = (source) => {
const found = slicedJobs(source);
assert.equal(found.length, 1, 'the reader must see the one sliced job');
return sliceProblems(found[0]).join('\n');
};
it('passes a job whose axis, name and --shard agree', () => {
assert.equal(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, `${SHARD}/2`)), '');
});
it('fails a job whose axis is a block list it cannot read', () => {
const blockList = ' shard:\n - 1\n - 2\n';
assert.match(problems(matrixJob(blockList, `T ${SHARD}/2`, `${SHARD}/2`)), /has no one-line axis/);
});
it('fails a job that runs --shard without a matrix', () => {
assert.match(problems(workflowWith(` name: 'T'\n${run('1/2')}`)), /has no one-line axis/);
});
it('fails a fixed --shard and a stale divisor', () => {
assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, '1/2')), /runs --shard=1\/2; write/);
assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, `${SHARD}/3`)), /\}\}\/3; write/);
});
it('reads a comment about --shard above the next job as no one job', () => {
const source = `${workflowWith(` name: 'Plain'\n`)} # runs in two slices (--shard=<i>/2)\n other:\n name: 'Other'\n`;
assert.deepEqual(slicedJobs(source), []);
});
it('fails a name whose count only starts like the axis', () => {
assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/20`, `${SHARD}/2`)), /job name must say/);
});
});