ci(test): restructure test.yml for a ten-minute ci-ok

Rename the workflow to CI and name every job "<Area> <what>[ i/n]" with
ids in kebab-case, in area order. Fold i18n parity, shared coverage and
the plugin SDK into Shared tests, split the client gates into Client
types and Client checks (with the bundle), and merge the image and chart
jobs into Docker and Helm with per-area step conditions. Server and
client tests run in ten slices, the Postgres lane in three per major,
and e2e in two slices inside the digest-pinned Playwright container.

A local setup action restores node_modules and shared/dist from exact
cache keys and falls back to npm ci and the shared build. The
node_modules key covers the lockfile and every workspace package.json,
so npm ci still refuses a package.json out of step with the lockfile.
Only pushes to dev and main save, the npm download cache included, and
only a full install saves that one. The plugin SDK installs and runs
before the root install so it resolves only its own dependencies.
Timeouts drop to 15/10/5 minutes. The workflow tests now check the name
form, the Playwright tag against the lockfile, scan's needs, timeouts,
the cache keys and saves, and that installs go through the setup action;
.github/actions/ counts like test.yml in changed-areas.
This commit is contained in:
Maurice
2026-11-18 14:48:43 +01:00
parent 23eabe1499
commit fd1942afab
12 changed files with 1209 additions and 897 deletions
+4 -4
View File
@@ -23,9 +23,9 @@
import { appendFileSync, readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
// The workflow file and this script decide what runs, so a change to either
// runs everything.
const ALWAYS = ['.github/workflows/test.yml', 'scripts/ci/'];
// The workflow file, the local actions its jobs set up with and this script
// decide what runs and how, so a change to any of them runs everything.
const ALWAYS = ['.github/workflows/test.yml', '.github/actions/', 'scripts/ci/'];
// Files the test suites read straight from disk, outside the packages:
// config-templates.test.ts pins docker-compose.yml and the Helm chart to the
@@ -49,7 +49,7 @@ export const AREAS = {
'package-lock.json',
'.nvmrc',
'sonar-project.properties',
// lint:knip in server-lint-checks reads it.
// lint:knip in server-lint reads it.
'knip.jsonc',
'Dockerfile',
...READ_BY_TESTS,
+7
View File
@@ -95,6 +95,13 @@ describe('classify', () => {
assert.deepEqual(classify(['scripts/ci/changed-areas.mjs']), every);
});
it('runs everything when a local action the jobs set up with changes', () => {
assert.deepEqual(classify(['.github/actions/setup/action.yml']), every);
assert.deepEqual(classify(['.github/actions/other/action.yaml']), every);
// Another workflow is not one of them.
assert.deepEqual(classify(['.github/workflows/security.yml']), none);
});
it('matches whole file names, not prefixes of them', () => {
assert.deepEqual(classify(['package.json.bak', 'Dockerfile.old', 'serverless/x.ts', 'wikis/x.png']), none);
});
+2 -2
View File
@@ -67,7 +67,7 @@ export function verdict(response, sha) {
`${run.status === 'completed' ? run.conclusion : run.status} ${run.html_url ?? `check run ${run.id}`}`;
if (runs.length === 0) {
return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run Tests & SonarQube on it and dispatch again.`] };
return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run CI on it and dispatch again.`] };
}
const lines = runs.map((run) => `ci-ok on ${sha}: ${describe(run)}`);
@@ -82,7 +82,7 @@ export function verdict(response, sha) {
return diff > 0 || (diff === 0 && b.id > a.id) ? b : a;
});
if (newest.conclusion !== 'success') {
lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run Tests & SonarQube on it again, then dispatch again.`);
lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run CI on it again, then dispatch again.`);
return { ok: false, lines };
}
for (const run of runs) {
+209 -2
View File
@@ -7,7 +7,10 @@ import { fileURLToPath } from 'node:url';
// is in ci-ok's needs. A new or renamed job that is left out runs, can fail,
// and merges anyway. The scan is the one job kept out on purpose (see the
// layout notes at the top of test.yml).
const workflow = readFileSync(fileURLToPath(new URL('../../.github/workflows/test.yml', import.meta.url)), 'utf8');
const read = (path) => readFileSync(fileURLToPath(new URL(path, import.meta.url)), 'utf8');
const workflow = read('../../.github/workflows/test.yml');
const setupAction = read('../../.github/actions/setup/action.yml');
const lockfile = JSON.parse(read('../../package-lock.json'));
const NOT_REQUIRED = new Set(['ci-ok', 'scan']);
const SHARD = '${{ matrix.shard }}';
@@ -17,6 +20,53 @@ function jobs(source = workflow) {
return starts.map((match, i) => ({ id: match[1], text: body.slice(match.index, starts[i + 1]?.index) }));
}
/** A job's text without its comment lines, so prose about a step never counts as the step. */
const withoutComments = (text) => text.replace(/^\s*#.*\n/gm, '');
/** The steps of a job, each from its "- " line to the next. */
const stepsOf = (text) =>
withoutComments(text)
.split(/\n(?= {6}- )/)
.slice(1);
/** The check name a job reports, without the quotes around it; null when it sets none. */
function nameOf(text) {
const raw = /^ {4}name: (.+)$/m.exec(text)?.[1];
return raw === undefined ? null : raw.trim().replace(/^'(.*)'$/, '$1').replace(/^"(.*)"$/, '$1');
}
/** The job ids under a job's needs:, one per line. */
function needsOf(text) {
const block = /^ {4}needs:\n((?: {6}- [a-z0-9-]+\n)+)/m.exec(withoutComments(text));
return block
? block[1]
.trim()
.split('\n')
.map((line) => line.replace(/^\s*- /, ''))
: [];
}
// Every check name reads "<Area> <what>[ <variant>][ i/n]": the area first,
// from a fixed set, so the checks list sorts by area; no colon, no
// parentheses and no middle dot, and a slice only at the end.
const NAME_FORM = /^(Changes|Shared|Server|Client|Docker|Sonar)( [^:·()]+)?$/;
const SLICE_AT_END = /\$\{\{ matrix\.shard \}\}\/\d+$/;
/** What is wrong with one job's check name, each said as the fix to make. */
function nameProblems(id, name) {
if (name === null) return [`${id}: give it a name: of the form "<Area> <what>[ i/n]"`];
const problems = [];
if (!NAME_FORM.test(name)) {
problems.push(
`${id}: "${name}" must start with Changes, Shared, Server, Client, Docker or Sonar and hold no colon, parentheses or middle dot`,
);
}
if (name.includes(SHARD) && !SLICE_AT_END.test(name)) {
problems.push(`${id}: put the slice "${SHARD}/n" at the end of "${name}"`);
}
return problems;
}
function jobIds() {
return jobs().map((job) => job.id);
}
@@ -101,7 +151,7 @@ describe('test.yml', () => {
it('finds the sliced jobs', () => {
const ids = slicedJobs().map((job) => job.id);
for (const id of ['server-postgres', 'server-tests', 'client-tests']) {
for (const id of ['client-e2e', 'client-tests', 'server-postgres', 'server-tests']) {
assert.ok(
ids.includes(id),
`${id} no longer names matrix.shard or --shard; when it stopped being sliced, drop it from this list in scripts/ci/workflow-jobs.test.mjs`,
@@ -109,6 +159,85 @@ describe('test.yml', () => {
}
});
it('names every check "<Area> <what>[ i/n]"', () => {
const problems = jobs()
.filter(({ id }) => id !== 'ci-ok')
.flatMap(({ id, text }) => nameProblems(id, nameOf(text)));
assert.deepEqual(problems, []);
});
it('keeps ci-ok under its own name, the one the ruleset requires', () => {
const ciOk = jobs().find(({ id }) => id === 'ci-ok');
assert.ok(ciOk, 'test.yml must define ci-ok');
assert.equal(nameOf(ciOk.text), null, 'ci-ok reports under its job id; a name: would rename the required check');
});
// A coverage artifact the scan does not wait for may be missing when it
// runs, and Sonar then reports every line of that package as uncovered.
it('lets the scan wait for every job that uploads Sonar coverage', () => {
const scan = jobs().find(({ id }) => id === 'scan');
assert.ok(scan, 'test.yml must define scan');
const needs = new Set(needsOf(scan.text));
const uploaders = jobs()
.filter(({ text }) =>
stepsOf(text).some(
(step) => /uses: actions\/upload-artifact@/.test(step) && /name: sonar-[\w-]+-coverage/.test(step),
),
)
.map(({ id }) => id);
assert.ok(uploaders.length >= 3, `expected the coverage uploads, found them in: ${uploaders.join(', ')}`);
const missing = uploaders.filter((id) => !needs.has(id));
assert.deepEqual(missing, [], `add these jobs to scan's needs: in test.yml: ${missing.join(', ')}`);
});
// A hung job holds its slot until the timeout, and the default is six hours.
it('bounds every job with a timeout of at most 20 minutes', () => {
const problems = jobs().flatMap(({ id, text }) => {
const minutes = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(text)?.[1];
if (minutes === undefined) return [`${id}: set timeout-minutes (at most 20)`];
return Number(minutes) > 20 ? [`${id}: timeout-minutes is ${minutes}, at most 20`] : [];
});
assert.deepEqual(problems, []);
});
// The setup action restores node_modules from an exact cache and runs npm
// ci only on a miss; a job that installs on its own pays the install on
// every run. plugin-sdk has its own lockfile and installs in its own
// directory, outside the workspaces the action covers.
it('installs the workspaces only through ./.github/actions/setup', () => {
const problems = jobs().flatMap(({ id, text }) =>
stepsOf(text)
.filter((step) => /\bnpm (ci|install|i)\b/.test(step) && !/^ {8}working-directory: plugin-sdk\s*$/m.test(step))
.map((step) => `${id}: "${step.split('\n')[0].trim()}" installs with npm; use ./.github/actions/setup`),
);
assert.deepEqual(problems, []);
assert.ok(
jobs().some(({ text }) => /uses: \.\/\.github\/actions\/setup\s*$/m.test(text)),
'no job uses ./.github/actions/setup',
);
});
// Dependabot bumps @playwright/test in the lockfile only. The browsers the
// flows run in come from the container, so its tag has to follow by hand,
// and the digest pins exactly that image.
it('runs e2e in the Playwright image of the version the lockfile pins', () => {
const e2e = jobs().find(({ id }) => id === 'client-e2e');
assert.ok(e2e, 'test.yml must define client-e2e');
const image = /^ {6}image: (\S+)\s*$/m.exec(withoutComments(e2e.text))?.[1] ?? '';
const pinned = /^mcr\.microsoft\.com\/playwright:v(\d+\.\d+\.\d+)-noble@sha256:[0-9a-f]{64}$/.exec(image);
assert.ok(
pinned,
`client-e2e's container must be mcr.microsoft.com/playwright:v<version>-noble@sha256:<digest>, not "${image}"`,
);
const locked = lockfile.packages['node_modules/@playwright/test']?.version;
assert.ok(locked, 'package-lock.json has no node_modules/@playwright/test');
assert.equal(
pinned[1],
locked,
`the lockfile pins @playwright/test ${locked}: set client-e2e's image to v${locked}-noble with that tag's digest (docker buildx imagetools inspect mcr.microsoft.com/playwright:v${locked}-noble)`,
);
});
for (const job of slicedJobs()) {
it(`slices ${job.id} alike in its matrix, its name and its --shard`, () => {
assert.deepEqual(sliceProblems(job), []);
@@ -116,6 +245,84 @@ describe('test.yml', () => {
}
});
describe('the setup action', () => {
const steps = setupAction.split(/\n(?= {4}- )/).slice(1);
// A restore-keys fallback would hand a job a tree built from another
// lockfile or on another image. Only the npm download cache may fall back:
// npm ci checks each tarball against the lockfile.
it('restores node_modules and shared/dist only on an exact key', () => {
const restores = steps.filter(
(step) => /actions\/cache\/restore@/.test(step) && !/^ {8}path: ~\/\.npm\s*$/m.test(step),
);
assert.ok(restores.length >= 2, 'expected a restore for node_modules and one for shared/dist');
for (const step of restores) assert.doesNotMatch(step, /restore-keys/, step);
});
// The full actions/cache action saves in its post step on every event, pull
// requests included, so every cache is restored and saved in two steps.
it('uses only the split restore and save actions', () => {
for (const step of steps) assert.doesNotMatch(step, /uses: actions\/cache@/, step);
});
// npm ci refuses a package.json that disagrees with the lockfile; a cache hit
// skips it, so a changed package.json has to miss.
it('keys node_modules on the lockfile and every workspace package.json', () => {
const deps = setupAction.match(/^ +DEPS: \$\{\{ hashFiles\((.*)\) \}\}$/m);
assert.ok(deps, 'expected a DEPS hash in the key step');
const files = deps[1].split(',').map((file) => file.trim().replace(/^'|'$/g, ''));
const wanted = ['package-lock.json', 'package.json', 'client/package.json', 'server/package.json', 'shared/package.json'];
for (const file of wanted) {
assert.ok(files.includes(file), `the node_modules key does not hash ${file}`);
}
});
// Pull requests read what their base left; a PR that saved could evict the
// base's caches and fill the quota with trees of its own.
it('saves only on a push to dev or main', () => {
assert.match(
setupAction,
/save=\$\{\{ github\.event_name == 'push' && \(github\.ref == 'refs\/heads\/dev' \|\| github\.ref == 'refs\/heads\/main'\) \}\}/,
);
const saves = steps.filter((step) => /actions\/cache\/save@/.test(step));
assert.ok(saves.length >= 3, 'expected a save for node_modules, the npm download cache and shared/dist');
for (const step of saves) assert.match(step, /steps\.keys\.outputs\.save == 'true'/, step);
});
});
describe('the name reader', () => {
it('passes names of the form', () => {
const names = [
'Changes',
'Shared tests',
`Server tests ${SHARD}/10`,
`Server Postgres \${{ matrix.postgres }} ${SHARD}/3`,
'Docker and Helm',
'Sonar',
];
for (const name of names) assert.deepEqual(nameProblems('job', name), [], name);
});
it('fails a colon, parentheses, a dot separator, an unknown area, a slice before the end and no name', () => {
for (const name of [
'Server: Types',
`Server Postgres 14 (${SHARD}/2)`,
'Client · Lint',
'Repo Sonar scan',
`Server tests ${SHARD}/10 extra`,
null,
]) {
assert.notDeepEqual(nameProblems('job', name), [], String(name));
}
});
it('reads a quoted name without its quotes', () => {
assert.equal(nameOf(" name: 'Server types'\n"), 'Server types');
assert.equal(nameOf(' name: Server types\n'), 'Server types');
assert.equal(nameOf(' runs-on: ubuntu-24.04\n'), null);
});
});
describe('the slice reader', () => {
const run = (arg) => ` steps:\n - run: npx vitest run --shard=${arg}\n`;
const workflowWith = (job) => `name: t\njobs:\n sliced:\n${job}`;