diff --git a/scripts/ci/workflow-jobs.test.mjs b/scripts/ci/workflow-jobs.test.mjs index 691d142f7..904ac418c 100644 --- a/scripts/ci/workflow-jobs.test.mjs +++ b/scripts/ci/workflow-jobs.test.mjs @@ -9,10 +9,16 @@ import { fileURLToPath } from 'node:url'; // layout notes at the top of test.yml). const workflow = readFileSync(fileURLToPath(new URL('../../.github/workflows/test.yml', import.meta.url)), 'utf8'); const NOT_REQUIRED = new Set(['ci-ok', 'scan']); +const SHARD = '${{ matrix.shard }}'; + +function jobs(source = workflow) { + const body = source.slice(source.indexOf('\njobs:\n')); + const starts = [...body.matchAll(/^ {2}([a-z0-9-]+):\s*$/gm)]; + return starts.map((match, i) => ({ id: match[1], text: body.slice(match.index, starts[i + 1]?.index) })); +} function jobIds() { - const body = workflow.slice(workflow.indexOf('\njobs:\n')); - return [...body.matchAll(/^ {2}([a-z0-9-]+):\s*$/gm)].map((match) => match[1]); + return jobs().map((job) => job.id); } function ciOkNeeds() { @@ -24,6 +30,62 @@ function ciOkNeeds() { .map((line) => line.replace(/^\s*- /, '')); } +// A sliced job states its slice count three times: the shard axis of its +// matrix, the "/" in its name and the --shard=/ it hands vitest. +// Changing one and not the others passes and still goes wrong: a divisor +// above the axis leaves part of the suite in no slice, one below it runs a +// slice vitest refuses, and a stale name reports the wrong count. Every job +// that names matrix.shard or runs --shard counts as sliced, so a job whose +// axis is written in a form this reader does not know fails instead of +// going unchecked. +function slicedJobs(source = workflow) { + return jobs(source) + .map(({ id, text }) => ({ id, text: text.replace(/^\s*#.*\n/gm, '') })) + .filter(({ text }) => /matrix\.shard|--shard/.test(text)) + .map(({ id, text }) => { + const axis = /^ {8}shard: \[([^\]]*)\]\s*$/m.exec(text); + return { + id, + shards: axis ? axis[1].split(',').map((value) => Number(value.trim())) : null, + name: /^ {4}name: (.+)$/m.exec(text)?.[1] ?? '', + shardArgs: [...text.matchAll(/--shard(?:=|\s+)(\$\{\{ matrix\.shard \}\}\/\d+|\S+)/g)].map((match) => match[1]), + }; + }); +} + +/** What is wrong with one sliced job, each said as the fix to make; empty when the three counts agree. */ +function sliceProblems(job) { + if (!job.shards) { + return [ + `${job.id}: it uses matrix.shard or --shard, but its matrix has no one-line axis "shard: [1, ..., n]". Write the axis in that form, or teach slicedJobs() in scripts/ci/workflow-jobs.test.mjs the new form (a matrix include with a count per entry, say) in the same change`, + ]; + } + const n = job.shards.length; + const oneToN = Array.from({ length: n }, (_, i) => i + 1); + const expected = `${SHARD}/${n}`; + const problems = []; + if (job.shards.join() !== oneToN.join()) problems.push(`${job.id}: write the shard axis as [${oneToN.join(', ')}]`); + if ( + !job.name + .split(expected) + .slice(1) + .some((rest) => !/^\d/.test(rest)) + ) { + problems.push( + `${job.id}: the shard axis has ${n} slices, so the job name must say "${expected}" (it reads ${job.name})`, + ); + } + if (job.shardArgs.length === 0) problems.push(`${job.id}: no step runs vitest with --shard=${expected}`); + for (const arg of job.shardArgs) { + if (arg !== expected) { + problems.push( + `${job.id}: the shard axis has ${n} slices but a step runs --shard=${arg}; write --shard=${expected}, and change the axis, the name and every --shard together`, + ); + } + } + return problems; +} + describe('test.yml', () => { it('lists every job but the scan in ci-ok needs', () => { const needs = new Set(ciOkNeeds()); @@ -36,4 +98,59 @@ describe('test.yml', () => { const unknown = ciOkNeeds().filter((id) => !ids.has(id)); assert.deepEqual(unknown, [], `ci-ok needs jobs test.yml does not define: ${unknown.join(', ')}`); }); + + it('finds the sliced jobs', () => { + const ids = slicedJobs().map((job) => job.id); + for (const id of ['server-postgres', 'server-tests', 'client-tests']) { + assert.ok( + ids.includes(id), + `${id} no longer names matrix.shard or --shard; when it stopped being sliced, drop it from this list in scripts/ci/workflow-jobs.test.mjs`, + ); + } + }); + + for (const job of slicedJobs()) { + it(`slices ${job.id} alike in its matrix, its name and its --shard`, () => { + assert.deepEqual(sliceProblems(job), []); + }); + } +}); + +describe('the slice reader', () => { + const run = (arg) => ` steps:\n - run: npx vitest run --shard=${arg}\n`; + const workflowWith = (job) => `name: t\njobs:\n sliced:\n${job}`; + const matrixJob = (axis, name, arg) => + workflowWith(` name: '${name}'\n strategy:\n matrix:\n${axis}${run(arg)}`); + const problems = (source) => { + const found = slicedJobs(source); + assert.equal(found.length, 1, 'the reader must see the one sliced job'); + return sliceProblems(found[0]).join('\n'); + }; + + it('passes a job whose axis, name and --shard agree', () => { + assert.equal(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, `${SHARD}/2`)), ''); + }); + + it('fails a job whose axis is a block list it cannot read', () => { + const blockList = ' shard:\n - 1\n - 2\n'; + assert.match(problems(matrixJob(blockList, `T ${SHARD}/2`, `${SHARD}/2`)), /has no one-line axis/); + }); + + it('fails a job that runs --shard without a matrix', () => { + assert.match(problems(workflowWith(` name: 'T'\n${run('1/2')}`)), /has no one-line axis/); + }); + + it('fails a fixed --shard and a stale divisor', () => { + assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, '1/2')), /runs --shard=1\/2; write/); + assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/2`, `${SHARD}/3`)), /\}\}\/3; write/); + }); + + it('reads a comment about --shard above the next job as no one job', () => { + const source = `${workflowWith(` name: 'Plain'\n`)} # runs in two slices (--shard=/2)\n other:\n name: 'Other'\n`; + assert.deepEqual(slicedJobs(source), []); + }); + + it('fails a name whose count only starts like the axis', () => { + assert.match(problems(matrixJob(' shard: [1, 2]\n', `T ${SHARD}/20`, `${SHARD}/2`)), /job name must say/); + }); });