ci(test): restructure test.yml for a ten-minute ci-ok

Rename the workflow to CI and name every job "<Area> <what>[ i/n]" with
ids in kebab-case, in area order. Fold i18n parity, shared coverage and
the plugin SDK into Shared tests, split the client gates into Client
types and Client checks (with the bundle), and merge the image and chart
jobs into Docker and Helm with per-area step conditions. Server and
client tests run in ten slices, the Postgres lane in three per major,
and e2e in two slices inside the digest-pinned Playwright container.

A local setup action restores node_modules and shared/dist from exact
cache keys and falls back to npm ci and the shared build. The
node_modules key covers the lockfile and every workspace package.json,
so npm ci still refuses a package.json out of step with the lockfile.
Only pushes to dev and main save, the npm download cache included, and
only a full install saves that one. The plugin SDK installs and runs
before the root install so it resolves only its own dependencies.
Timeouts drop to 15/10/5 minutes. The workflow tests now check the name
form, the Playwright tag against the lockfile, scan's needs, timeouts,
the cache keys and saves, and that installs go through the setup action;
.github/actions/ counts like test.yml in changed-areas.
This commit is contained in:
Maurice
2026-11-18 14:48:43 +01:00
parent 23eabe1499
commit fd1942afab
12 changed files with 1209 additions and 897 deletions
+146
View File
@@ -0,0 +1,146 @@
name: Set up Node and the workspaces
description: >-
Node from .nvmrc, the workspace dependencies from an exact cache or npm ci,
and optionally shared/dist from a cache or a build. Runs after checkout.
# A local action, so it is versioned with the workflow that calls it and needs
# no SHA pin. scripts/ci/changed-areas.mjs counts .github/actions/ like
# test.yml: a change here runs everything.
#
# node_modules is restored only on an exact key: the runner image, the scope,
# the Node ABI, the lockfile and every workspace package.json. There are no
# restore-keys, so a hit is the tree npm ci would build and a miss is a plain
# npm ci. The package.json files are in the key because npm ci is also the
# check that they agree with the lockfile: an edited range without a fresh
# lockfile misses the cache and fails the install, as it did when every job
# ran npm ci. Only pushes to dev and main save, the npm download cache
# included, so pull requests read what their base left and cannot evict it or
# fill the repository's cache quota with entries of their own.
inputs:
scope:
description: >-
Which workspaces to install: full (npm ci), client (shared and client)
or shared (shared alone).
required: true
build-shared:
description: "'true' to restore shared/dist, or build it on a miss."
required: false
default: 'false'
runs:
using: composite
steps:
- uses: actions/setup-node@v6
with:
node-version-file: .nvmrc
# ImageOS and ImageVersion name the runner image, whose system libraries
# the native modules were built against. They live in the runner's
# environment, not in the env context, so the key is put together here. A
# job in a container has neither; the container's OS release stands in,
# and that job's tree is kept apart from the runner's.
- name: Work out the cache keys
id: keys
shell: bash
env:
SCOPE: ${{ inputs.scope }}
DEPS: ${{ hashFiles('.nvmrc', 'package-lock.json', 'package.json', 'client/package.json', 'server/package.json', 'shared/package.json') }}
SHARED: ${{ hashFiles('shared/src/**', 'shared/package.json', 'shared/tsconfig*.json', 'shared/tsdown.config.ts', 'package-lock.json') }}
run: |
set -euo pipefail
case "$SCOPE" in
full | client | shared) ;;
*) echo "::error::scope must be full, client or shared, not '$SCOPE'"; exit 1 ;;
esac
if [ -n "${ImageOS:-}" ]; then
image="$ImageOS-${ImageVersion:-unknown}"
else
image="container-$(. /etc/os-release && echo "$ID$VERSION_ID")"
fi
abi="$(node -p process.versions.modules)"
echo "nm=nm-$RUNNER_OS-$image-$SCOPE-node$abi-$DEPS" >> "$GITHUB_OUTPUT"
echo "sd=sd-$RUNNER_OS-$SHARED" >> "$GITHUB_OUTPUT"
echo "save=${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}" >> "$GITHUB_OUTPUT"
- name: Restore node_modules
id: nm
uses: actions/cache/restore@v6
with:
path: |
node_modules
server/node_modules
client/node_modules
shared/node_modules
key: ${{ steps.keys.outputs.nm }}
# The npm download cache only matters when npm ci runs, so it is restored
# only then. It may fall back to an older lockfile's cache: npm ci checks
# every tarball against the lockfile's integrity hash and fetches what is
# missing, so a stale entry only costs downloads.
- name: Restore the npm download cache
id: npm
if: steps.nm.outputs.cache-hit != 'true'
uses: actions/cache/restore@v6
with:
path: ~/.npm
key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
restore-keys: npm-${{ runner.os }}-
- name: Install dependencies
if: steps.nm.outputs.cache-hit != 'true'
shell: bash
env:
SCOPE: ${{ inputs.scope }}
run: |
set -euo pipefail
case "$SCOPE" in
full) npm ci ;;
# The scoped install keeps @testing-library/dom (a peer of
# @testing-library/react) in the tree.
client) npm ci --workspace shared --workspace client ;;
shared) npm ci --workspace shared ;;
esac
- name: Save node_modules
if: steps.nm.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true'
uses: actions/cache/save@v6
with:
path: |
node_modules
server/node_modules
client/node_modules
shared/node_modules
key: ${{ steps.keys.outputs.nm }}
# Only a full install saves the download cache, so the entry holds every
# tarball; a scoped job that finished first would otherwise claim the key
# with part of the tree and no later run would fill it.
- name: Save the npm download cache
if: steps.nm.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true' && steps.npm.outputs.cache-hit != 'true' && inputs.scope == 'full'
uses: actions/cache/save@v6
with:
path: ~/.npm
key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
# The key holds every input of the build: the sources, the package and
# build configs, and the lockfile that pins tsdown and zod.
- name: Restore shared/dist
id: sd
if: inputs.build-shared == 'true'
uses: actions/cache/restore@v6
with:
path: shared/dist
key: ${{ steps.keys.outputs.sd }}
- name: Build shared
if: inputs.build-shared == 'true' && steps.sd.outputs.cache-hit != 'true'
shell: bash
run: npm run build --workspace=shared
- name: Save shared/dist
if: inputs.build-shared == 'true' && steps.sd.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true'
uses: actions/cache/save@v6
with:
path: shared/dist
key: ${{ steps.keys.outputs.sd }}
+4 -1
View File
@@ -7,8 +7,11 @@
# enforce-target-branch.yml moves those onto dev.
#
# Every PR runs test.yml like any other, so ci-ok decides whether it can
# merge. Dependabot runs get no Actions secrets, so s3-contract and the Sonar
# merge. Dependabot runs get no Actions secrets, so server-s3 and the Sonar
# scan skip themselves on these PRs and ci-ok counts the skip as passed.
# A bump of @playwright/test also needs the Playwright container tag and
# digest in test.yml's client-e2e moved by hand; workflow-jobs.test.mjs
# fails the PR until they match.
version: 2
updates:
# The root lockfile covers the client, server and shared workspaces.
+8 -1
View File
@@ -18,9 +18,16 @@ jobs:
- uses: actions/setup-node@v6
with:
node-version-file: .nvmrc
cache: npm
cache-dependency-path: package-lock.json
# shared's ESLint and Prettier plugins are its own devDependencies, and
# the format ratchet in scripts/lib/ resolves prettier from the root, so
# shared plus the root is the whole tree these checks read. npm ci
# installs exactly what the lockfile says, where npm install could
# resolve anew.
- name: Install dependencies
run: npm install
run: npm ci --workspace shared --include-workspace-root
- name: Run lint & format check
# Both are check-only: lint used to run eslint --fix, which rewrote the
+1 -1
View File
@@ -13,7 +13,7 @@ name: Security Scan
# - scout scans the built image, base image packages included. It needs
# the Docker Hub login, so it runs on PRs and pushes into main only and
# skips fork PRs. Whether the image builds at all is test.yml's
# docker-image job, which a red scan cannot hide.
# docker job, which a red scan cannot hide.
on:
pull_request:
branches: [main, dev]
+816 -879
View File
File diff suppressed because it is too large Load Diff
+8 -4
View File
@@ -137,13 +137,17 @@ These principles come out of a full-repo audit and shape all new code:
## Quality gates in CI
`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`, client also the strict-mode ratchet `lint:strict`), lint (client and server through `lint:warnings`, which also holds the warning count per rule), `lint:pages`, `lint:rtl`, `lint:size` (client and server), the server's `lint:strict`, `lint:boundaries`, `lint:tx`, `lint:dialect`, `lint:engine-dispatch`, `lint:test-sql`, `lint:test-mocks`, `lint:test-new-service`, `lint:query-api`, `lint:mcp-zod`, `lint:service-http` and `lint:response-contracts`, the client's and server's `lint:format`, `lint:layers`, `lint:offline`, `lint:dup`, `lint:pairs`, `lint:skips`, `lint:i18n-keys` and `theme:lint`, shared's `contracts:open`, the repo-wide `lint:knip` (in server-lint-checks), `check:plugin-facts`, `i18n:parity:strict` (with the untranslated-strings ratchet), the S3 contract suite, TREK on Postgres (`server-postgres`, once per supported major, 14 and 18, each in two slices: first the Postgres probe (in the first slice), every dialect helper on SQLite and Postgres and the statements every repository method reaches with sample arguments, its SQL errors per method and the methods it cannot measure held to a baseline that only shrinks and fails while it is still unmeasured; then the Postgres lane, every suite that boots the app on the real database module, found by `server/tests/postgres/lane.ts` rather than listed, plus PARITY-PG and the write lock; see `server/CLAUDE.md`), the browser end-to-end flows (Chromium and WebKit), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. It also builds the Docker image and boots it until `/api/health/ready` answers, on SQLite and on Postgres 18 (`docker-image`), and lints and renders the Helm chart against the Kubernetes schemas and runs `docker compose config` on every compose file (`deploy-manifests`). `lint-prettier.yml` additionally runs `lint` + `lint:format` on `shared/`, both check-only. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags.
`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`, client also the strict-mode ratchet `lint:strict`), lint (client and server through `lint:warnings`, which also holds the warning count per rule), `lint:pages`, `lint:rtl`, `lint:size` (client and server), the server's `lint:strict`, `lint:boundaries`, `lint:tx`, `lint:dialect`, `lint:engine-dispatch`, `lint:test-sql`, `lint:test-mocks`, `lint:test-new-service`, `lint:query-api`, `lint:mcp-zod`, `lint:service-http` and `lint:response-contracts`, the client's and server's `lint:format`, `lint:layers`, `lint:offline`, `lint:dup`, `lint:pairs`, `lint:skips`, `lint:i18n-keys` and `theme:lint`, shared's `contracts:open`, the repo-wide `lint:knip` (in `server-lint`), `check:plugin-facts`, `i18n:parity:strict` (with the untranslated-strings ratchet), the S3 contract suite, TREK on Postgres (`server-postgres`, once per supported major, 14 and 18, each in three slices: first the Postgres probe (in the first slice), every dialect helper on SQLite and Postgres and the statements every repository method reaches with sample arguments, its SQL errors per method and the methods it cannot measure held to a baseline that only shrinks and fails while it is still unmeasured; then the Postgres lane, every suite that boots the app on the real database module, found by `server/tests/postgres/lane.ts` rather than listed, plus PARITY-PG and the write lock; see `server/CLAUDE.md`), the browser end-to-end flows (Chromium and WebKit, in the Playwright container, in two slices), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. It also builds the Docker image and boots it until `/api/health/ready` answers, on SQLite and on Postgres 18, and lints and renders the Helm chart against the Kubernetes schemas and runs `docker compose config` on every compose file (both in `docker`, each part when its area changed). `lint-prettier.yml` additionally runs `lint` + `lint:format` on `shared/`, both check-only. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags.
- **Job names read "package: what"** (`Server: Types`, `Client: Tests 3/7`), and the list at the top of `test.yml` says what each job holds. A new job follows the scheme and joins that list, and the job-count note there says when a long job needs a slot freed first.
- **Tests run beside the gates, in shards.** The client and server suites run in slices on parallel runners (`client-tests`, `server-tests`, through each package's `vitest.shard.config.ts`), and `client-coverage`/`server-coverage` merge their blob reports and check the thresholds against the whole. A red slice is usually a failing test, a red merge job usually a threshold. A sliced job states its slice count in its matrix's shard axis, the `i/n` in its name and its `--shard=i/n`; `scripts/ci/workflow-jobs.test.mjs` fails until all three agree, and on any job naming `matrix.shard` or `--shard` whose axis is not the one-line `shard: [1, ..., n]` it reads.
- **Job names read "<Area> <what>[ <variant>][ i/n]"** (`Server tests 3/10`, `Server Postgres 14 2/3`, `Docker and Helm`): the area first, one of Changes, Shared, Server, Client, Docker or Sonar, then lower case, with no colon, no parentheses and no middle dot, and a slice only at the end, so the checks list sorts by area. The workflow is named `CI`, so a check reads `CI / Server tests 3/10`. Job ids are the names in kebab-case (`server-tests`), the jobs stand in `test.yml` in area order, and the list at the top of `test.yml` says what each job holds. A new job follows the scheme and joins that list; `scripts/ci/workflow-jobs.test.mjs` fails on a name outside it. `ci-ok`, `lint` and `check-target` keep their names: the ruleset requires them by name.
- **What the folded jobs hold.** `Shared tests` is the plugin SDK's typecheck and coverage, run before the root install so the SDK resolves only its own dependencies, then i18n parity and shared's coverage. `Client types` is the typecheck and the strict ratchet; `Client checks` the source ratchets, then the production build with `check:gl-split`. `Docker and Helm` builds and boots the image when the image area changed and checks the chart and compose files when the deploy area changed. Inside a folded job each group of steps runs after a failure in another, so one red check shows every gate that failed; the step names say which.
- **Tests run beside the gates, in shards.** The client and server suites run in ten slices each on parallel runners (`client-tests`, `server-tests`, through each package's `vitest.shard.config.ts`), the Postgres lane in three per major, and the end-to-end flows in two, and `client-coverage`/`server-coverage` merge their blob reports and check the thresholds against the whole. A red slice is usually a failing test, a red merge job usually a threshold. A sliced job states its slice count in its matrix's shard axis, the `i/n` in its name and its `--shard=i/n`; `scripts/ci/workflow-jobs.test.mjs` fails until all three agree, and on any job naming `matrix.shard` or `--shard` whose axis is not the one-line `shard: [1, ..., n]` it reads.
- **`ci-ok` is the required check.** The branch ruleset requires `ci-ok` (plus `lint` and `check-target` from their own workflows), and `ci-ok` needs every job above except the scan. It fails if any of them failed or was cancelled and passes if they passed or were skipped. A new gate job counts only once it is listed in `ci-ok`'s `needs:`, and `scripts/ci/workflow-jobs.test.mjs` (in `test:ci-scripts`) fails while a job other than the scan is missing there.
- **Guides name paths that exist**: the `changes` job runs `scripts/ci/doc-paths.mjs`, which resolves every backtick path in the CLAUDE.md files, `README.md`, `server/src/nest/README.md` and the wiki against the tracked files; the missing ones are listed in `scripts/ci/doc-paths-baseline.json`, which only shrinks (`--update` after a fix).
- **What runs is decided by `changes`, not by `on.paths`**: `scripts/ci/changed-areas.mjs` maps the changed files to the `code`, `image` and `deploy` areas (tests: `npm run test:ci-scripts`). A path filter on the workflow would leave `ci-ok` unreported on a docs-only PR and block it. Add a new top-level file the build reads to the right area there.
- **What runs is decided by `changes`, not by `on.paths`**: `scripts/ci/changed-areas.mjs` maps the changed files to the `code`, `image` and `deploy` areas (tests: `npm run test:ci-scripts`). A path filter on the workflow would leave `ci-ok` unreported on a docs-only PR and block it. Add a new top-level file the build reads to the right area there. A change to `test.yml`, to `scripts/ci/` or to a local action under `.github/actions/` runs everything.
- **Setup goes through one local action.** Every job that installs dependencies uses `.github/actions/setup/action.yml` (`scope` full, client or shared; `build-shared`) instead of its own `npm ci`; `workflow-jobs.test.mjs` fails on a job that installs on its own (plugin-sdk's own install in its directory excepted). The action restores node_modules on an exact key (runner image, scope, Node ABI, `.nvmrc`, the lockfile and every workspace `package.json`, so npm ci still refuses a `package.json` out of step with the lockfile; no restore-keys) and shared/dist on a key over shared's sources and build config, and runs `npm ci` or the shared build on a miss. Only pushes to dev and main save, the npm download cache included; pull requests only read.
- **End-to-end runs in the Playwright container**, `mcr.microsoft.com/playwright:v<version>-noble` pinned by digest, so no step installs browsers or system libraries. Dependabot bumps only the lockfile: on every Playwright PR move the tag and its digest in `client-e2e` by hand (`docker buildx imagetools inspect mcr.microsoft.com/playwright:v<version>-noble`); `workflow-jobs.test.mjs` fails until the tag names the lockfile's `@playwright/test` version.
- **Budget: `ci-ok` within ten minutes.** One code run starts at most about 35 long jobs (gates, slices, Postgres, e2e, Docker), so two PR runs fit in the organisation's 60 concurrent jobs. Every job sets `timeout-minutes` of at most 20 (15 for the long ones, 10 for the short ones), which `workflow-jobs.test.mjs` holds. A new long job, slice or shard needs a reason that outweighs its slot; the job-count note at the top of `test.yml` has the numbers.
- **Runners and Node**: jobs run on `ubuntu-24.04`, not `ubuntu-latest`, and every `setup-node` reads `.nvmrc`. Third-party actions (anything outside `actions/*`) are pinned by commit SHA with the tag in a comment; Dependabot keeps both current.
- **Security scans are advisory.** `security.yml` runs `npm audit` (what the image installs, plus `plugin-sdk`) on every PR and push into dev and main, and Docker Scout on the built image for main. Neither is required: a newly published advisory would turn every open PR red at once. Fixes are held as floors in the root `package.json` `overrides`; npm ignores an `overrides` block in a workspace `package.json`, so never put one there.
- **Releases** are dispatched by hand and refuse a commit whose `ci-ok` is not green. `docker-dev.yml` (any branch, usually dev) is the canary: it builds the dispatched commit as the next minor or major `-pre.N` and moves only `:latest-pre` and `:<major>-pre`. `docker.yml` (main only) bumps the version, commits and tags it, builds both platforms from exactly that commit and moves `:latest`, `:<major>`, `:<major>.<minor>` and `:<version>`. Its `bump=auto` only finalizes a prerelease in flight; without one it stops and asks for patch, minor or major. Ship a release that migrates or changes a lot as a prerelease first, and as a minor, so installs pinned to `:<major>.<minor>` can wait it out.
+4 -4
View File
@@ -23,9 +23,9 @@
import { appendFileSync, readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
// The workflow file and this script decide what runs, so a change to either
// runs everything.
const ALWAYS = ['.github/workflows/test.yml', 'scripts/ci/'];
// The workflow file, the local actions its jobs set up with and this script
// decide what runs and how, so a change to any of them runs everything.
const ALWAYS = ['.github/workflows/test.yml', '.github/actions/', 'scripts/ci/'];
// Files the test suites read straight from disk, outside the packages:
// config-templates.test.ts pins docker-compose.yml and the Helm chart to the
@@ -49,7 +49,7 @@ export const AREAS = {
'package-lock.json',
'.nvmrc',
'sonar-project.properties',
// lint:knip in server-lint-checks reads it.
// lint:knip in server-lint reads it.
'knip.jsonc',
'Dockerfile',
...READ_BY_TESTS,
+7
View File
@@ -95,6 +95,13 @@ describe('classify', () => {
assert.deepEqual(classify(['scripts/ci/changed-areas.mjs']), every);
});
it('runs everything when a local action the jobs set up with changes', () => {
assert.deepEqual(classify(['.github/actions/setup/action.yml']), every);
assert.deepEqual(classify(['.github/actions/other/action.yaml']), every);
// Another workflow is not one of them.
assert.deepEqual(classify(['.github/workflows/security.yml']), none);
});
it('matches whole file names, not prefixes of them', () => {
assert.deepEqual(classify(['package.json.bak', 'Dockerfile.old', 'serverless/x.ts', 'wikis/x.png']), none);
});
+2 -2
View File
@@ -67,7 +67,7 @@ export function verdict(response, sha) {
`${run.status === 'completed' ? run.conclusion : run.status} ${run.html_url ?? `check run ${run.id}`}`;
if (runs.length === 0) {
return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run Tests & SonarQube on it and dispatch again.`] };
return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run CI on it and dispatch again.`] };
}
const lines = runs.map((run) => `ci-ok on ${sha}: ${describe(run)}`);
@@ -82,7 +82,7 @@ export function verdict(response, sha) {
return diff > 0 || (diff === 0 && b.id > a.id) ? b : a;
});
if (newest.conclusion !== 'success') {
lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run Tests & SonarQube on it again, then dispatch again.`);
lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run CI on it again, then dispatch again.`);
return { ok: false, lines };
}
for (const run of runs) {
+209 -2
View File
@@ -7,7 +7,10 @@ import { fileURLToPath } from 'node:url';
// is in ci-ok's needs. A new or renamed job that is left out runs, can fail,
// and merges anyway. The scan is the one job kept out on purpose (see the
// layout notes at the top of test.yml).
const workflow = readFileSync(fileURLToPath(new URL('../../.github/workflows/test.yml', import.meta.url)), 'utf8');
const read = (path) => readFileSync(fileURLToPath(new URL(path, import.meta.url)), 'utf8');
const workflow = read('../../.github/workflows/test.yml');
const setupAction = read('../../.github/actions/setup/action.yml');
const lockfile = JSON.parse(read('../../package-lock.json'));
const NOT_REQUIRED = new Set(['ci-ok', 'scan']);
const SHARD = '${{ matrix.shard }}';
@@ -17,6 +20,53 @@ function jobs(source = workflow) {
return starts.map((match, i) => ({ id: match[1], text: body.slice(match.index, starts[i + 1]?.index) }));
}
/** A job's text without its comment lines, so prose about a step never counts as the step. */
const withoutComments = (text) => text.replace(/^\s*#.*\n/gm, '');
/** The steps of a job, each from its "- " line to the next. */
const stepsOf = (text) =>
withoutComments(text)
.split(/\n(?= {6}- )/)
.slice(1);
/** The check name a job reports, without the quotes around it; null when it sets none. */
function nameOf(text) {
const raw = /^ {4}name: (.+)$/m.exec(text)?.[1];
return raw === undefined ? null : raw.trim().replace(/^'(.*)'$/, '$1').replace(/^"(.*)"$/, '$1');
}
/** The job ids under a job's needs:, one per line. */
function needsOf(text) {
const block = /^ {4}needs:\n((?: {6}- [a-z0-9-]+\n)+)/m.exec(withoutComments(text));
return block
? block[1]
.trim()
.split('\n')
.map((line) => line.replace(/^\s*- /, ''))
: [];
}
// Every check name reads "<Area> <what>[ <variant>][ i/n]": the area first,
// from a fixed set, so the checks list sorts by area; no colon, no
// parentheses and no middle dot, and a slice only at the end.
const NAME_FORM = /^(Changes|Shared|Server|Client|Docker|Sonar)( [^:·()]+)?$/;
const SLICE_AT_END = /\$\{\{ matrix\.shard \}\}\/\d+$/;
/** What is wrong with one job's check name, each said as the fix to make. */
function nameProblems(id, name) {
if (name === null) return [`${id}: give it a name: of the form "<Area> <what>[ i/n]"`];
const problems = [];
if (!NAME_FORM.test(name)) {
problems.push(
`${id}: "${name}" must start with Changes, Shared, Server, Client, Docker or Sonar and hold no colon, parentheses or middle dot`,
);
}
if (name.includes(SHARD) && !SLICE_AT_END.test(name)) {
problems.push(`${id}: put the slice "${SHARD}/n" at the end of "${name}"`);
}
return problems;
}
function jobIds() {
return jobs().map((job) => job.id);
}
@@ -101,7 +151,7 @@ describe('test.yml', () => {
it('finds the sliced jobs', () => {
const ids = slicedJobs().map((job) => job.id);
for (const id of ['server-postgres', 'server-tests', 'client-tests']) {
for (const id of ['client-e2e', 'client-tests', 'server-postgres', 'server-tests']) {
assert.ok(
ids.includes(id),
`${id} no longer names matrix.shard or --shard; when it stopped being sliced, drop it from this list in scripts/ci/workflow-jobs.test.mjs`,
@@ -109,6 +159,85 @@ describe('test.yml', () => {
}
});
it('names every check "<Area> <what>[ i/n]"', () => {
const problems = jobs()
.filter(({ id }) => id !== 'ci-ok')
.flatMap(({ id, text }) => nameProblems(id, nameOf(text)));
assert.deepEqual(problems, []);
});
it('keeps ci-ok under its own name, the one the ruleset requires', () => {
const ciOk = jobs().find(({ id }) => id === 'ci-ok');
assert.ok(ciOk, 'test.yml must define ci-ok');
assert.equal(nameOf(ciOk.text), null, 'ci-ok reports under its job id; a name: would rename the required check');
});
// A coverage artifact the scan does not wait for may be missing when it
// runs, and Sonar then reports every line of that package as uncovered.
it('lets the scan wait for every job that uploads Sonar coverage', () => {
const scan = jobs().find(({ id }) => id === 'scan');
assert.ok(scan, 'test.yml must define scan');
const needs = new Set(needsOf(scan.text));
const uploaders = jobs()
.filter(({ text }) =>
stepsOf(text).some(
(step) => /uses: actions\/upload-artifact@/.test(step) && /name: sonar-[\w-]+-coverage/.test(step),
),
)
.map(({ id }) => id);
assert.ok(uploaders.length >= 3, `expected the coverage uploads, found them in: ${uploaders.join(', ')}`);
const missing = uploaders.filter((id) => !needs.has(id));
assert.deepEqual(missing, [], `add these jobs to scan's needs: in test.yml: ${missing.join(', ')}`);
});
// A hung job holds its slot until the timeout, and the default is six hours.
it('bounds every job with a timeout of at most 20 minutes', () => {
const problems = jobs().flatMap(({ id, text }) => {
const minutes = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(text)?.[1];
if (minutes === undefined) return [`${id}: set timeout-minutes (at most 20)`];
return Number(minutes) > 20 ? [`${id}: timeout-minutes is ${minutes}, at most 20`] : [];
});
assert.deepEqual(problems, []);
});
// The setup action restores node_modules from an exact cache and runs npm
// ci only on a miss; a job that installs on its own pays the install on
// every run. plugin-sdk has its own lockfile and installs in its own
// directory, outside the workspaces the action covers.
it('installs the workspaces only through ./.github/actions/setup', () => {
const problems = jobs().flatMap(({ id, text }) =>
stepsOf(text)
.filter((step) => /\bnpm (ci|install|i)\b/.test(step) && !/^ {8}working-directory: plugin-sdk\s*$/m.test(step))
.map((step) => `${id}: "${step.split('\n')[0].trim()}" installs with npm; use ./.github/actions/setup`),
);
assert.deepEqual(problems, []);
assert.ok(
jobs().some(({ text }) => /uses: \.\/\.github\/actions\/setup\s*$/m.test(text)),
'no job uses ./.github/actions/setup',
);
});
// Dependabot bumps @playwright/test in the lockfile only. The browsers the
// flows run in come from the container, so its tag has to follow by hand,
// and the digest pins exactly that image.
it('runs e2e in the Playwright image of the version the lockfile pins', () => {
const e2e = jobs().find(({ id }) => id === 'client-e2e');
assert.ok(e2e, 'test.yml must define client-e2e');
const image = /^ {6}image: (\S+)\s*$/m.exec(withoutComments(e2e.text))?.[1] ?? '';
const pinned = /^mcr\.microsoft\.com\/playwright:v(\d+\.\d+\.\d+)-noble@sha256:[0-9a-f]{64}$/.exec(image);
assert.ok(
pinned,
`client-e2e's container must be mcr.microsoft.com/playwright:v<version>-noble@sha256:<digest>, not "${image}"`,
);
const locked = lockfile.packages['node_modules/@playwright/test']?.version;
assert.ok(locked, 'package-lock.json has no node_modules/@playwright/test');
assert.equal(
pinned[1],
locked,
`the lockfile pins @playwright/test ${locked}: set client-e2e's image to v${locked}-noble with that tag's digest (docker buildx imagetools inspect mcr.microsoft.com/playwright:v${locked}-noble)`,
);
});
for (const job of slicedJobs()) {
it(`slices ${job.id} alike in its matrix, its name and its --shard`, () => {
assert.deepEqual(sliceProblems(job), []);
@@ -116,6 +245,84 @@ describe('test.yml', () => {
}
});
describe('the setup action', () => {
const steps = setupAction.split(/\n(?= {4}- )/).slice(1);
// A restore-keys fallback would hand a job a tree built from another
// lockfile or on another image. Only the npm download cache may fall back:
// npm ci checks each tarball against the lockfile.
it('restores node_modules and shared/dist only on an exact key', () => {
const restores = steps.filter(
(step) => /actions\/cache\/restore@/.test(step) && !/^ {8}path: ~\/\.npm\s*$/m.test(step),
);
assert.ok(restores.length >= 2, 'expected a restore for node_modules and one for shared/dist');
for (const step of restores) assert.doesNotMatch(step, /restore-keys/, step);
});
// The full actions/cache action saves in its post step on every event, pull
// requests included, so every cache is restored and saved in two steps.
it('uses only the split restore and save actions', () => {
for (const step of steps) assert.doesNotMatch(step, /uses: actions\/cache@/, step);
});
// npm ci refuses a package.json that disagrees with the lockfile; a cache hit
// skips it, so a changed package.json has to miss.
it('keys node_modules on the lockfile and every workspace package.json', () => {
const deps = setupAction.match(/^ +DEPS: \$\{\{ hashFiles\((.*)\) \}\}$/m);
assert.ok(deps, 'expected a DEPS hash in the key step');
const files = deps[1].split(',').map((file) => file.trim().replace(/^'|'$/g, ''));
const wanted = ['package-lock.json', 'package.json', 'client/package.json', 'server/package.json', 'shared/package.json'];
for (const file of wanted) {
assert.ok(files.includes(file), `the node_modules key does not hash ${file}`);
}
});
// Pull requests read what their base left; a PR that saved could evict the
// base's caches and fill the quota with trees of its own.
it('saves only on a push to dev or main', () => {
assert.match(
setupAction,
/save=\$\{\{ github\.event_name == 'push' && \(github\.ref == 'refs\/heads\/dev' \|\| github\.ref == 'refs\/heads\/main'\) \}\}/,
);
const saves = steps.filter((step) => /actions\/cache\/save@/.test(step));
assert.ok(saves.length >= 3, 'expected a save for node_modules, the npm download cache and shared/dist');
for (const step of saves) assert.match(step, /steps\.keys\.outputs\.save == 'true'/, step);
});
});
describe('the name reader', () => {
it('passes names of the form', () => {
const names = [
'Changes',
'Shared tests',
`Server tests ${SHARD}/10`,
`Server Postgres \${{ matrix.postgres }} ${SHARD}/3`,
'Docker and Helm',
'Sonar',
];
for (const name of names) assert.deepEqual(nameProblems('job', name), [], name);
});
it('fails a colon, parentheses, a dot separator, an unknown area, a slice before the end and no name', () => {
for (const name of [
'Server: Types',
`Server Postgres 14 (${SHARD}/2)`,
'Client · Lint',
'Repo Sonar scan',
`Server tests ${SHARD}/10 extra`,
null,
]) {
assert.notDeepEqual(nameProblems('job', name), [], String(name));
}
});
it('reads a quoted name without its quotes', () => {
assert.equal(nameOf(" name: 'Server types'\n"), 'Server types');
assert.equal(nameOf(' name: Server types\n'), 'Server types');
assert.equal(nameOf(' runs-on: ubuntu-24.04\n'), null);
});
});
describe('the slice reader', () => {
const run = (arg) => ` steps:\n - run: npx vitest run --shard=${arg}\n`;
const workflowWith = (job) => `name: t\njobs:\n sliced:\n${job}`;
+1 -1
View File
File diff suppressed because one or more lines are too long
+3 -2
View File
@@ -18,8 +18,9 @@ import { defineConfig } from 'vitest/config';
* instead of the real database module.
*
* CI runs the lane once per supported Postgres major (the server-postgres job
* in test.yml), each in two slices (`--shard=i/2`) like the SQLite suite;
* every slice builds its own template in global setup.
* in test.yml), each in three slices (`--shard=i/3`) like the SQLite suite;
* every slice builds its own template in global setup. The slice count lives
* in the workflow only.
*/
export default defineConfig({
...base,