From fd1942afab1c1828e843fbf1a84a0fe77ca4ccfa Mon Sep 17 00:00:00 2001 From: Maurice Date: Wed, 18 Nov 2026 14:48:43 +0100 Subject: [PATCH] ci(test): restructure test.yml for a ten-minute ci-ok Rename the workflow to CI and name every job " [ i/n]" with ids in kebab-case, in area order. Fold i18n parity, shared coverage and the plugin SDK into Shared tests, split the client gates into Client types and Client checks (with the bundle), and merge the image and chart jobs into Docker and Helm with per-area step conditions. Server and client tests run in ten slices, the Postgres lane in three per major, and e2e in two slices inside the digest-pinned Playwright container. A local setup action restores node_modules and shared/dist from exact cache keys and falls back to npm ci and the shared build. The node_modules key covers the lockfile and every workspace package.json, so npm ci still refuses a package.json out of step with the lockfile. Only pushes to dev and main save, the npm download cache included, and only a full install saves that one. The plugin SDK installs and runs before the root install so it resolves only its own dependencies. Timeouts drop to 15/10/5 minutes. The workflow tests now check the name form, the Playwright tag against the lockfile, scan's needs, timeouts, the cache keys and saves, and that installs go through the setup action; .github/actions/ counts like test.yml in changed-areas. --- .github/actions/setup/action.yml | 146 +++ .github/dependabot.yml | 5 +- .github/workflows/lint-prettier.yml | 9 +- .github/workflows/security.yml | 2 +- .github/workflows/test.yml | 1695 +++++++++++++-------------- CLAUDE.md | 12 +- scripts/ci/changed-areas.mjs | 8 +- scripts/ci/changed-areas.test.mjs | 7 + scripts/ci/ci-ok-verdict.mjs | 4 +- scripts/ci/workflow-jobs.test.mjs | 211 +++- server/CLAUDE.md | 2 +- server/vitest.pg.config.ts | 5 +- 12 files changed, 1209 insertions(+), 897 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 000000000..e09c8015d --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,146 @@ +name: Set up Node and the workspaces +description: >- + Node from .nvmrc, the workspace dependencies from an exact cache or npm ci, + and optionally shared/dist from a cache or a build. Runs after checkout. + +# A local action, so it is versioned with the workflow that calls it and needs +# no SHA pin. scripts/ci/changed-areas.mjs counts .github/actions/ like +# test.yml: a change here runs everything. +# +# node_modules is restored only on an exact key: the runner image, the scope, +# the Node ABI, the lockfile and every workspace package.json. There are no +# restore-keys, so a hit is the tree npm ci would build and a miss is a plain +# npm ci. The package.json files are in the key because npm ci is also the +# check that they agree with the lockfile: an edited range without a fresh +# lockfile misses the cache and fails the install, as it did when every job +# ran npm ci. Only pushes to dev and main save, the npm download cache +# included, so pull requests read what their base left and cannot evict it or +# fill the repository's cache quota with entries of their own. + +inputs: + scope: + description: >- + Which workspaces to install: full (npm ci), client (shared and client) + or shared (shared alone). + required: true + build-shared: + description: "'true' to restore shared/dist, or build it on a miss." + required: false + default: 'false' + +runs: + using: composite + steps: + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + # ImageOS and ImageVersion name the runner image, whose system libraries + # the native modules were built against. They live in the runner's + # environment, not in the env context, so the key is put together here. A + # job in a container has neither; the container's OS release stands in, + # and that job's tree is kept apart from the runner's. + - name: Work out the cache keys + id: keys + shell: bash + env: + SCOPE: ${{ inputs.scope }} + DEPS: ${{ hashFiles('.nvmrc', 'package-lock.json', 'package.json', 'client/package.json', 'server/package.json', 'shared/package.json') }} + SHARED: ${{ hashFiles('shared/src/**', 'shared/package.json', 'shared/tsconfig*.json', 'shared/tsdown.config.ts', 'package-lock.json') }} + run: | + set -euo pipefail + case "$SCOPE" in + full | client | shared) ;; + *) echo "::error::scope must be full, client or shared, not '$SCOPE'"; exit 1 ;; + esac + if [ -n "${ImageOS:-}" ]; then + image="$ImageOS-${ImageVersion:-unknown}" + else + image="container-$(. /etc/os-release && echo "$ID$VERSION_ID")" + fi + abi="$(node -p process.versions.modules)" + echo "nm=nm-$RUNNER_OS-$image-$SCOPE-node$abi-$DEPS" >> "$GITHUB_OUTPUT" + echo "sd=sd-$RUNNER_OS-$SHARED" >> "$GITHUB_OUTPUT" + echo "save=${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}" >> "$GITHUB_OUTPUT" + + - name: Restore node_modules + id: nm + uses: actions/cache/restore@v6 + with: + path: | + node_modules + server/node_modules + client/node_modules + shared/node_modules + key: ${{ steps.keys.outputs.nm }} + + # The npm download cache only matters when npm ci runs, so it is restored + # only then. It may fall back to an older lockfile's cache: npm ci checks + # every tarball against the lockfile's integrity hash and fetches what is + # missing, so a stale entry only costs downloads. + - name: Restore the npm download cache + id: npm + if: steps.nm.outputs.cache-hit != 'true' + uses: actions/cache/restore@v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + restore-keys: npm-${{ runner.os }}- + + - name: Install dependencies + if: steps.nm.outputs.cache-hit != 'true' + shell: bash + env: + SCOPE: ${{ inputs.scope }} + run: | + set -euo pipefail + case "$SCOPE" in + full) npm ci ;; + # The scoped install keeps @testing-library/dom (a peer of + # @testing-library/react) in the tree. + client) npm ci --workspace shared --workspace client ;; + shared) npm ci --workspace shared ;; + esac + + - name: Save node_modules + if: steps.nm.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true' + uses: actions/cache/save@v6 + with: + path: | + node_modules + server/node_modules + client/node_modules + shared/node_modules + key: ${{ steps.keys.outputs.nm }} + + # Only a full install saves the download cache, so the entry holds every + # tarball; a scoped job that finished first would otherwise claim the key + # with part of the tree and no later run would fill it. + - name: Save the npm download cache + if: steps.nm.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true' && steps.npm.outputs.cache-hit != 'true' && inputs.scope == 'full' + uses: actions/cache/save@v6 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + + # The key holds every input of the build: the sources, the package and + # build configs, and the lockfile that pins tsdown and zod. + - name: Restore shared/dist + id: sd + if: inputs.build-shared == 'true' + uses: actions/cache/restore@v6 + with: + path: shared/dist + key: ${{ steps.keys.outputs.sd }} + + - name: Build shared + if: inputs.build-shared == 'true' && steps.sd.outputs.cache-hit != 'true' + shell: bash + run: npm run build --workspace=shared + + - name: Save shared/dist + if: inputs.build-shared == 'true' && steps.sd.outputs.cache-hit != 'true' && steps.keys.outputs.save == 'true' + uses: actions/cache/save@v6 + with: + path: shared/dist + key: ${{ steps.keys.outputs.sd }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ab2ea9a46..584aa403a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,8 +7,11 @@ # enforce-target-branch.yml moves those onto dev. # # Every PR runs test.yml like any other, so ci-ok decides whether it can -# merge. Dependabot runs get no Actions secrets, so s3-contract and the Sonar +# merge. Dependabot runs get no Actions secrets, so server-s3 and the Sonar # scan skip themselves on these PRs and ci-ok counts the skip as passed. +# A bump of @playwright/test also needs the Playwright container tag and +# digest in test.yml's client-e2e moved by hand; workflow-jobs.test.mjs +# fails the PR until they match. version: 2 updates: # The root lockfile covers the client, server and shared workspaces. diff --git a/.github/workflows/lint-prettier.yml b/.github/workflows/lint-prettier.yml index d2bce335b..05f4999ea 100644 --- a/.github/workflows/lint-prettier.yml +++ b/.github/workflows/lint-prettier.yml @@ -18,9 +18,16 @@ jobs: - uses: actions/setup-node@v6 with: node-version-file: .nvmrc + cache: npm + cache-dependency-path: package-lock.json + # shared's ESLint and Prettier plugins are its own devDependencies, and + # the format ratchet in scripts/lib/ resolves prettier from the root, so + # shared plus the root is the whole tree these checks read. npm ci + # installs exactly what the lockfile says, where npm install could + # resolve anew. - name: Install dependencies - run: npm install + run: npm ci --workspace shared --include-workspace-root - name: Run lint & format check # Both are check-only: lint used to run eslint --fix, which rewrote the diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index d1bd3a20d..f4715c7e6 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -13,7 +13,7 @@ name: Security Scan # - scout scans the built image, base image packages included. It needs # the Docker Hub login, so it runs on PRs and pushes into main only and # skips fork PRs. Whether the image builds at all is test.yml's -# docker-image job, which a red scan cannot hide. +# docker job, which a red scan cannot hide. on: pull_request: branches: [main, dev] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5104139ab..dd97aed4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Tests & SonarQube +name: CI permissions: contents: read @@ -9,90 +9,99 @@ permissions: # analysis; pull requests get PR analysis with decoration on the PR itself # (the scanner picks the context up from the Actions environment). # -# Job names read ": ", so the checks list on a PR sorts by -# package. What each job holds: +# Job names read " [ ][ i/n]", the area one of Changes, +# Shared, Server, Client, Docker or Sonar, with no colon, no parentheses and +# the slice last, so the checks list on a PR sorts by area. The job ids are +# the names in kebab-case, and the jobs below stand in the same order. +# scripts/ci/workflow-jobs.test.mjs holds the names to that form. What each +# job holds: # -# Repo: Changed Areas decides what the change needs to run (below) -# Shared: i18n Parity every locale has en's files and keys -# Shared: Coverage shared's tests with coverage -# Server: Types builds shared and the server, typechecks shared, -# the server's src, tests and scripts, smokes the +# Changes decides what the change needs to run (below) +# Shared tests every locale has en's files and keys (i18n +# parity); shared's tests with coverage; the plugin +# SDK's typecheck and tests with coverage +# Server types builds the server, typechecks shared, the +# server's src, tests and scripts, smokes the # production require chain, the strict ratchet, -# the generated entities and Kysely types -# Server: Lint & Checks ESLint (typed), errors and warning counts; then +# the generated entities and Kysely types, the +# Postgres baseline +# Server lint ESLint (typed), errors and warning counts; then # Prettier, the plugin facts, the DB call graph, # the shared ratchet code, the source ratchets -# (size, boundaries, dialect, tx, test SQL, test -# mocks, hand-built services, query API, MCP zod, -# error envelopes, response contracts), open -# request schemas, knip -# Server: Tests n/7 one slice of the server suite, with coverage -# Server: Coverage merges the slices, checks the per-domain +# (size, boundaries, dialect, engine checks, tx, +# test SQL, test mocks, hand-built services, query +# API, MCP zod, error envelopes, response +# contracts), open request schemas, knip +# Server tests i/10 one slice of the server suite, with coverage +# Server coverage merges the slices, checks the per-domain # thresholds, uploads lcov -# Server: S3 Contract (MinIO) -# the storage driver against MinIO -# Server: Postgres 14/18 per supported major, in two slices: the probe +# Server Postgres 14 i/3, Server Postgres 18 i/3 +# per supported major, in three slices: the probe # (the repository SQL against Postgres, first slice # only), then a slice of the lane (every suite that # boots the app on the real database module, plus # PARITY-PG and the write lock; vitest.pg.config.ts) -# Client: Types & Checks typecheck and the strict ratchet; then the page -# pattern and the source ratchets (RTL, size, -# Prettier, layers, offline path, duplication, -# desktop/phone pairs, skips, i18n keys, theme -# tokens) -# Client: Lint ESLint (typed), errors and warning counts -# Client: Tests n/7 one slice of the client suite, with coverage -# Client: Coverage merges the slices, checks the thresholds, +# Server S3 the storage driver against MinIO +# Client types typecheck and the strict ratchet +# Client checks the page pattern and the source ratchets (RTL, +# size, Prettier, layers, offline path, +# duplication, desktop/phone pairs, skips, i18n +# keys, theme tokens); then the production build +# and one chunk per map engine +# Client lint ESLint (typed), errors and warning counts +# Client tests i/10 one slice of the client suite, with coverage +# Client coverage merges the slices, checks the thresholds, # uploads lcov -# Client: Bundle the production build, one chunk per map engine -# Client: End-to-End the app in Chromium and WebKit -# Plugin SDK: Types & Coverage -# Repo: Docker Image builds the image and boots it until ready, on SQLite and on Postgres 18 -# Repo: Helm Chart & Compose -# Repo: Sonar Scan SonarCloud analysis, not required +# Client e2e i/2 one slice of the browser flows in Chromium and +# WebKit, in the Playwright container +# Docker and Helm builds the image and boots it until ready, on +# SQLite and on Postgres 18; lints and renders the +# Helm chart and checks the compose files +# Sonar SonarCloud analysis, not required # ci-ok the one required check (below) # # Job layout: # - Everything starts right after changes: the gates and the tests run side -# by side, so a run takes as long as its slowest job and not the gates -# plus the tests. The tests used to wait for the gates, which put ~9 -# minutes in front of every test result. The gates were one job per -# package (~8-9 minutes each); each is now two jobs of ~4-5 minutes, -# split where the time is (the server's typechecks, the client's lint), -# so neither outlasts a test shard. -# - The client and server suites run in shards (client-tests, +# by side, so a run takes as long as its slowest chain and not the gates +# plus the tests. Each gate is cut to about four to five minutes, split +# where the time is (the server's typechecks, the client's lint, the +# client's checks with the bundle), so none outlasts a test slice. +# - The client and server suites run in ten slices each (client-tests, # server-tests), each running a slice with coverage and uploading a blob # report; client-coverage and server-coverage merge the blobs, check the -# thresholds against the whole and upload lcov. A red shard is usually a +# thresholds against the whole and upload lcov. A red slice is usually a # failing test, a red merge job usually a threshold. None of the other # test jobs feeds the scan. -# - Job count: GitHub runs at most 20 jobs of the organisation at once, -# the other workflows of the same push and other PRs included, and does -# not say in which order it starts queued ones. A code change starts up -# to 30 jobs at once: 24 that run for minutes (the 14 test shards, the -# four Postgres slices, the four gates, e2e and the image) and six that -# are done in a minute or two (parity, the SDK, the bundle, S3, shared -# coverage, the chart). The long jobs alone are more than the slots, so -# at least four of them start only after a long job has ended, four to -# five minutes in, when the first gates end; more while other workflows -# hold slots (in the run below only 16 of this workflow's jobs started at -# once). Whichever jobs wait, the run grows by that wait when one of them -# is a test shard, since its coverage job waits for it: in run -# 44200029312 (first attempt, with six Postgres slices, 26 long jobs) -# nine long jobs started four to five and a half minutes in, five client -# shards among them, and Client: Coverage ended at 23:46, eight minutes -# after the last client shard that started on time. A layout with six -# gate jobs did the same. Every long job more makes that wait likelier, -# so the lane runs in as few slices as keep each one no longer than a -# test shard, and a new long job, shard or slice needs a reason that -# outweighs it. -# - Scan `needs:` the gates and the four coverage jobs. A coverage job -# failing must stop the scan — a missing lcov file would make every line -# of that package report as uncovered, and the dashboard would show a -# collapse that did not happen. +# - Setup: every job that installs dependencies goes through +# ./.github/actions/setup, which restores node_modules and shared/dist +# from an exact cache key and falls back to npm ci and the shared build. +# The node_modules key covers the lockfile and every workspace +# package.json, so a package.json out of step with the lockfile misses and +# npm ci refuses it. Only pushes to dev and main save, the npm download +# cache included, so a pull request reads its base's caches and a first +# run after a dependency change pays the install once. +# - A job that holds several gates (Shared tests, Client checks, Docker and +# Helm) runs each group of steps after a failure in another group too, so +# one red check still shows every gate that failed. The step names say +# which one it was. +# - Job count: GitHub Team runs 60 Linux jobs of the organisation at once, +# the other workflows and other PRs included. A code change starts at +# most about 35 long jobs (today 34: the five gates, the 20 test slices, +# the six Postgres slices, the two e2e slices and the Docker job) plus two +# short ones (Shared tests, Server S3), so two PR runs fit side by side +# and a third waits a minute or two for slots. The target is ci-ok within +# ten minutes of the push. A new long job, slice or shard needs a reason +# that outweighs the slot it takes; a job that waits for a slot adds its +# wait to the run when a coverage job waits for it. +# - Timeouts: 15 minutes for the gates, the slices, Postgres, e2e and the +# Docker job, 10 for the merges, Shared tests, Server S3 and changes, 5 +# for ci-ok, so a job that hangs frees its slot in a quarter of an hour. +# - Scan `needs:` the gates and the coverage jobs, Shared tests among them. +# A coverage job failing must stop the scan: a missing lcov file would +# make every line of that package report as uncovered, and the dashboard +# would show a collapse that did not happen. # - Fork PRs and runs Dependabot triggers don't receive the repo's Actions -# secrets, so the two jobs that need them skip themselves: s3-contract +# secrets, so the two jobs that need them skip themselves: server-s3 # (MINIO_LICENSE_KEY) and scan (SONAR_TOKEN). ci-ok counts a skip as # passed. Everything else runs on those PRs so outside contributors and # dependency bumps still get the full test signal. @@ -101,9 +110,10 @@ permissions: # runs nothing else. The filter lives in a job, not in `on.paths`, because # a workflow skipped by its path filter never reports a check, and a # required check that never reports blocks the PR for good. -# - docker-image builds the Dockerfile and boots the result, and -# deploy-manifests lints and renders the Helm chart and checks the -# compose files. Both run beside the gates rather than after them. +# - docker builds the Dockerfile and boots the result when the image area +# changed, and lints and renders the Helm chart and checks the compose +# files when the deploy area changed; each step carries its area's +# condition. It runs beside the gates rather than after them. # - ci-ok `needs:` every gate, test, image and chart job, and is the one # check the branch ruleset requires. It fails when any of them failed or # was cancelled and passes when they passed or were skipped (by changes, @@ -134,7 +144,7 @@ concurrency: jobs: changes: - name: 'Repo: Changed Areas' + name: 'Changes' runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: @@ -209,48 +219,95 @@ jobs: node scripts/ci/changed-areas.mjs --all fi - i18n-parity: - name: 'Shared: i18n Parity' + # i18n parity, shared's coverage and the plugin SDK in one job: each takes + # seconds to a minute, and a runner of its own would cost more in setup than + # it saves. Parity stays out of changes on purpose: a red changes skips + # every other job and would hide their failures. Each group runs after a + # failure in another, so one red run names every gate that failed. + shared: + name: 'Shared tests' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 - + timeout-minutes: 10 steps: - uses: actions/checkout@v6 + # plugin-sdk is standalone (own lockfile, not a root workspace) and has + # to work on its own once published, so it installs, typechecks and tests + # before the root install: with the workspaces' node_modules above it, a + # module or tool missing from its package.json would resolve from there + # and pass. The Node from .nvmrc is the image's Node, which runs the + # plugins and has the node:sqlite the dev-server tests exercise. - uses: actions/setup-node@v6 with: node-version-file: .nvmrc - - name: Check i18n key parity - run: node shared/scripts/i18n-parity.mjs --strict - - server-types: - name: 'Server: Types' - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies + - name: Plugin SDK install + id: sdk-install + working-directory: plugin-sdk run: npm ci - - name: Build shared - run: npm run build --workspace=shared + - name: Plugin SDK typecheck + if: ${{ !cancelled() && steps.sdk-install.outcome == 'success' }} + working-directory: plugin-sdk + run: npm run typecheck + + - name: Plugin SDK coverage + id: sdk-coverage + if: ${{ !cancelled() && steps.sdk-install.outcome == 'success' }} + working-directory: plugin-sdk + run: npm run test:coverage + + - name: Upload plugin SDK coverage + if: ${{ !cancelled() && steps.sdk-coverage.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: sonar-plugin-sdk-coverage + path: plugin-sdk/coverage/ + retention-days: 1 + + - uses: ./.github/actions/setup + id: setup + if: ${{ !cancelled() }} + with: + scope: shared + + - name: i18n parity + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: node shared/scripts/i18n-parity.mjs --strict + + - name: Shared coverage + id: shared-coverage + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: cd shared && npm run test:coverage + + - name: Upload shared coverage + if: ${{ !cancelled() && steps.shared-coverage.outcome == 'success' }} + uses: actions/upload-artifact@v6 + with: + name: sonar-shared-coverage + path: shared/coverage/ + retention-days: 1 + + server-types: + name: 'Server types' + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup + with: + scope: full + build-shared: 'true' - name: Typecheck shared - # Rides along here rather than in shared-coverage: this job already - # pays the root install and shared build, and a coverage job may only - # fail because tests failed (it gates the scan). + # Rides along here rather than in the shared job: this job already + # pays the root install and the shared build, and a coverage job may + # only fail because tests failed (it gates the scan). run: cd shared && npm run typecheck - name: Build server (tsc -> dist) @@ -306,26 +363,19 @@ jobs: # Either way every SQLite trigger needs its Postgres port. run: cd server && npm run check:pg-baseline - server-lint-checks: - name: 'Server: Lint & Checks' + server-lint: + name: 'Server lint' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 15 steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + - uses: ./.github/actions/setup with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci - - - name: Build shared - run: npm run build --workspace=shared + scope: full + build-shared: 'true' - name: Fetch the base the warnings are compared with # lint:warnings names the file behind a grown count by linting it as @@ -455,31 +505,295 @@ jobs: # knip (knip.jsonc) over client, server and shared: unused files, # exports, types and dependencies per file may not rise above # scripts/ci/knip-baseline.json (scripts/ci/knip-ratchet.mjs). Here - # rather than in client-types-checks because it needs every workspace's + # rather than in a client job because it needs every workspace's # dependencies, which the client jobs do not install. run: npm run lint:knip - client-types-checks: - name: 'Client: Types & Checks' + # The server suite in ten slices. vitest.shard.config.ts is vitest.config.ts + # without the thresholds and the coverage reports: a slice's blob carries + # its coverage to server-coverage, which checks the whole. fail-fast is off, + # so one red run shows every failing slice and not just the first. + server-tests: + name: 'Server tests ${{ matrix.shard }}/10' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup + with: + scope: full + build-shared: 'true' + + - name: Check @swc/core's native binding loads + # The server's SWC transform (server/vitest.config.ts) needs it. The + # lockfile carries the Linux binary, so npm ci installs it; this only + # names the cause if a lockfile regenerated without it ever lands, or + # if a cached node_modules ever holds a binary this runner cannot + # load. (It used to be installed here with --legacy-peer-deps, which + # pruned eight peer packages and left the tests on a tree the + # lockfile does not describe.) + run: node -e "require('@swc/core').transformSync('let a = 1')" + + - name: Run a slice of the tests with coverage + run: >- + cd server && npx vitest run --config vitest.shard.config.ts --coverage + --shard=${{ matrix.shard }}/10 + --reporter=default --reporter=github-actions --reporter=blob + + - name: Upload the blob report + uses: actions/upload-artifact@v6 + with: + name: server-blob-${{ matrix.shard }} + path: server/.vitest-reports/ + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + + server-coverage: + name: 'Server coverage' + needs: server-tests + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup + with: + scope: full + + - name: Download the blob reports + uses: actions/download-artifact@v7 + with: + pattern: server-blob-* + path: server/.vitest-reports/ + merge-multiple: true + + - name: Merge the slices and check the thresholds + # Every blob holds a whole coverage map, and the merge reads them all + # into one process. + env: + NODE_OPTIONS: --max-old-space-size=6144 + run: cd server && npx vitest run --merge-reports --coverage + + - name: Upload coverage + uses: actions/upload-artifact@v6 + with: + name: sonar-server-coverage + path: server/coverage/ + retention-days: 1 + + # TREK on Postgres, once per supported major: the oldest (14) and the newest + # (18). Both steps run against the same service, the probe on an empty + # database of its own, the lane on databases it creates and drops. + # + # The probe (server/scripts/pg-probe.ts): against an empty Postgres it + # derives a schema from the entities, runs every dialect helper on SQLite + # and Postgres (every case must pass), and calls every public repository + # method once, each statement in its own rolled-back transaction. The + # statements Postgres refuses as SQL are held per method to + # server/scripts/pg-probe-baseline.json, and so is the list of methods it + # could not measure (uncovered); both may only shrink, on every major. An + # unmeasured baseline (failing or uncovered null) or a run that called no + # method fails. The measured baseline and the full report are uploaded + # either way (pg-probe-baseline-), so a change that fixes a statement + # can lower the file without a local Postgres. + # + # The lane (server/vitest.pg.config.ts): every suite that boots the app on + # the real database module, found by server/tests/postgres/lane.ts, each file + # on its own copy of one migrated template, plus the Postgres-only suites + # (PARITY-PG, the write lock, the search semantics). It fails, never skips, + # without TREK_TEST_DB_URL, and TREK_TEST_PG_MAJOR makes it refuse a service + # of another major. It runs after a red probe too, so one run shows both. + # + # The lane runs the whole server suite that boots the app. Each file + # copies the template with CREATE DATABASE, one copy at a time, and before + # Postgres 15 a copy is two checkpoints, each waiting for fsync. With fsync + # on, the lane's test step took 20 to 23 minutes on 14 and about 12 on 18, + # summed over the slices (runs 44194202635 and 44200029312). The service + # runs with fsync off (the first step): a test database has nothing to + # keep through a crash. Measured in Docker on a workstation, 30 copies of + # a 120-table template took 24.9 s on 14 with fsync and 3.5 s without, + # and 5.3 s on 17 with it. + # + # Each major runs the lane in three slices (--shard=/3), each on a runner + # and a service of its own; each slice builds its own template in global + # setup, and a slice spends under a minute before the lane starts + # (checkout and the cached install and shared build). vitest splits the + # files by count, not by time. Three keeps a slice of either major at about + # the length of a test slice (five to six and a half minutes): every slice + # is a long job (see Job count at the top), and one slice more is one more + # job that may wait for a slot. If a slice of 14 still runs longer than a + # test slice, give 14 one slice more rather than both majors; that needs a + # count per major (a matrix include), which scripts/ci/workflow-jobs.test.mjs + # does not read yet and fails on until it is taught to. The probe runs once + # per major, in the first slice. ci-ok needs this job, and a matrix job's + # result is failure when any one slice fails or is cancelled, so every + # slice is required. The shard axis, the --shard divisor and the "i/n" at + # the end of the name must agree; scripts/ci/workflow-jobs.test.mjs fails + # when they do not. + server-postgres: + name: 'Server Postgres ${{ matrix.postgres }} ${{ matrix.shard }}/3' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + needs: changes + if: needs.changes.outputs.code == 'true' + strategy: + fail-fast: false + matrix: + postgres: [14, 18] + shard: [1, 2, 3] + services: + postgres: + image: postgres:${{ matrix.postgres }} + env: + POSTGRES_PASSWORD: lane + POSTGRES_DB: trek_probe + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d trek_probe" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + # A service container takes no server arguments, so fsync is turned off + # through ALTER SYSTEM; it is a reload setting, no restart needed. Why + # it is off: see the comment above the job. + - name: Turn fsync off on the service + env: + SERVICE: ${{ job.services.postgres.id }} + run: | + set -euo pipefail + docker exec "$SERVICE" psql -U postgres -d trek_probe -c 'ALTER SYSTEM SET fsync = off' -c 'SELECT pg_reload_conf()' + if [ "$(docker exec "$SERVICE" psql -U postgres -d trek_probe -Atc 'SHOW fsync')" != off ]; then + echo "::error::fsync is still on after the reload; the lane would pass but take about twice as long on 14." + exit 1 + fi + + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # The probe reads the repositories' parameter types, some of which come + # from @trek/shared's built declarations. + - uses: ./.github/actions/setup + with: + scope: full + build-shared: 'true' + + - name: Run the Postgres probe + if: matrix.shard == 1 + env: + TREK_PG_PROBE_URL: postgres://postgres:lane@127.0.0.1:5432/trek_probe + run: >- + cd server && npm run probe:pg -- + --next-baseline="$RUNNER_TEMP/pg-probe-baseline.json" + --report="$RUNNER_TEMP/pg-probe-report.json" + + - name: Upload the measured baseline and the report + if: ${{ !cancelled() && matrix.shard == 1 }} + uses: actions/upload-artifact@v6 + with: + name: pg-probe-baseline-${{ matrix.postgres }} + path: | + ${{ runner.temp }}/pg-probe-baseline.json + ${{ runner.temp }}/pg-probe-report.json + if-no-files-found: ignore + retention-days: 14 + + - name: Run the Postgres lane + if: ${{ !cancelled() }} + env: + TREK_TEST_DB_URL: postgres://postgres:lane@127.0.0.1:5432/postgres + TREK_TEST_PG_MAJOR: ${{ matrix.postgres }} + run: npm run test:pg --workspace=server -- --shard=${{ matrix.shard }}/3 + + server-s3: + name: 'Server S3' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + needs: changes + # Fork PRs don't get repo secrets, so MINIO_LICENSE_KEY would be empty and + # the AIStor container would fail to boot. A run Dependabot triggers gets + # only Dependabot secrets, so its PRs (same repo, not a fork) hit the same + # wall. Pushes and manual dispatches by anyone else do have secrets, so + # only those two are filtered out. + if: >- + needs.changes.outputs.code == 'true' && + github.actor != 'dependabot[bot]' && + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + - uses: ./.github/actions/setup with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json + scope: full + build-shared: 'true' - - name: Install dependencies - run: npm ci --workspace shared --workspace client + - name: Check @swc/core's native binding loads + # The server's SWC transform (server/vitest.config.ts) needs it; see + # the same step in server-tests. + run: node -e "require('@swc/core').transformSync('let a = 1')" - - name: Build shared - run: npm run build --workspace=shared + - name: Write MinIO license + run: printf '%s' "${{ secrets.MINIO_LICENSE_KEY }}" > minio-licence.key + + - name: Start MinIO + # `services:` can't pass the `server /data --license ...` command to + # the AIStor image, so it starts via docker run. 127.0.0.1 endpoint is + # load-bearing: aws-lite picks path-style only for IP/localhost hosts, + # which is all stock MinIO/AIStor speaks. + run: | + docker run -d --name minio -p 127.0.0.1:9000:9000 \ + -v "$PWD/minio-licence.key:/minio.license:ro" \ + -e MINIO_ROOT_USER=trekci -e MINIO_ROOT_PASSWORD=trekci-secret \ + quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z server /data --license /minio.license + + - name: Wait for MinIO + # curl was removed from the AIStor image — poll from the runner instead. + # 60 x 2s gives a wider margin on cold runners than the 30 x 1s the + # pre-6fbff835 job used. + run: | + for _ in $(seq 1 60); do + curl -sf http://127.0.0.1:9000/minio/health/live && exit 0 + sleep 2 + done + echo "MinIO never became healthy" >&2 + docker logs minio >&2 + exit 1 + + - name: Run the S3 driver contract suite + # No --coverage: the src/nest/storage/** 94/84/97/94 pin must keep + # holding from hermetic tests alone. The suite's beforeAll creates + # the trek-test bucket idempotently — no separate bucket-init step. + env: + TREK_TEST_S3_ENDPOINT: http://127.0.0.1:9000 + TREK_TEST_S3_BUCKET: trek-test + TREK_TEST_S3_ACCESS_KEY_ID: trekci + TREK_TEST_S3_SECRET_ACCESS_KEY: trekci-secret + run: cd server && npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts + + client-types: + name: 'Client types' + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup + with: + scope: client + build-shared: 'true' - name: Typecheck run: cd client && npm run typecheck @@ -494,6 +808,23 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 run: cd client && npm run lint:strict + # The source ratchets, then the production build. The build runs after a + # red ratchet too, so one red run names every gate that failed. + client-checks: + name: 'Client checks' + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - uses: ./.github/actions/setup + id: setup + with: + scope: client + build-shared: 'true' + - name: Page pattern check run: cd client && npm run lint:pages @@ -557,27 +888,32 @@ jobs: # follow the user's scheme, accent and text size (scripts/theme-lint.mjs). run: cd client && npm run theme:lint + - name: Build client + id: build-client + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: cd client && npm run build + + - name: Each map engine in a chunk of its own + # One static import of mapbox-gl or maplibre-gl in the map tree folds + # both engines into one chunk again, and nothing else notices: the build + # stays green and the app still works, only every map user downloads + # both. scripts/check-gl-split.mjs explains the markers it looks for. + if: ${{ !cancelled() && steps.build-client.outcome == 'success' }} + run: cd client && npm run check:gl-split + client-lint: - name: 'Client: Lint' + name: 'Client lint' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 - + timeout-minutes: 15 steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + - uses: ./.github/actions/setup with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci --workspace shared --workspace client - - - name: Build shared - run: npm run build --workspace=shared + scope: client + build-shared: 'true' - name: Fetch the base the warnings are compared with # lint:warnings names the file behind a grown count by linting it as @@ -598,431 +934,32 @@ jobs: TREK_LINT_BASE: refs/remotes/lint-base run: cd client && npm run lint:warnings - client-bundle: - name: 'Client: Bundle' + # The client suite in ten slices, the same way as server-tests. + client-tests: + name: 'Client tests ${{ matrix.shard }}/10' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci --workspace shared --workspace client - - - name: Build shared - run: npm run build --workspace=shared - - - name: Build client - run: cd client && npm run build - - - name: Each map engine in a chunk of its own - # One static import of mapbox-gl or maplibre-gl in the map tree folds - # both engines into one chunk again, and nothing else notices: the build - # stays green and the app still works, only every map user downloads - # both. scripts/check-gl-split.mjs explains the markers it looks for. - run: cd client && npm run check:gl-split - - client-e2e: - name: 'Client: End-to-End' - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - # The run boots the real server (e2e/server-launch.mjs builds it and - # starts it against a throwaway SQLite file) and the Vite dev server in - # front of it, so it needs every workspace. - - name: Install dependencies - run: npm ci - - - name: Build shared - run: npm run build --workspace=shared - - # Keyed on the lockfile, which pins @playwright/test and with it the - # browser builds. The system libraries the browsers need are not cached - # and are installed on every run. - - name: Cache browsers - id: browsers - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright - key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - - # Chromium for the desktop and phone flows, WebKit for the iPad and - # iPhone ones (every browser on iOS and iPadOS is WebKit underneath). - - name: Install browsers - if: steps.browsers.outputs.cache-hit != 'true' - run: cd client && npx playwright install --with-deps chromium webkit - - - name: Install browser dependencies - if: steps.browsers.outputs.cache-hit == 'true' - run: cd client && npx playwright install-deps chromium webkit - - # The public and app projects are the regression flows. The screenshot - # and help-media projects make pictures for the wiki and the help center - # and assert nothing, so they stay out of CI. - - name: Run the end-to-end flows - run: cd client && npx playwright test --project=public --project=app - - - name: Upload traces - if: failure() - uses: actions/upload-artifact@v6 - with: - name: e2e-results - path: client/test-results/ - retention-days: 7 - - s3-contract: - name: 'Server: S3 Contract (MinIO)' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - needs: changes - # Fork PRs don't get repo secrets, so MINIO_LICENSE_KEY would be empty and - # the AIStor container would fail to boot. A run Dependabot triggers gets - # only Dependabot secrets, so its PRs (same repo, not a fork) hit the same - # wall. Pushes and manual dispatches by anyone else do have secrets, so - # only those two are filtered out. - if: >- - needs.changes.outputs.code == 'true' && - github.actor != 'dependabot[bot]' && - (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci - - - name: Check @swc/core's native binding loads - # The server's SWC transform (server/vitest.config.ts) needs it. The - # lockfile carries the Linux binary, so npm ci installs it; this only - # names the cause if a lockfile regenerated without it ever lands. (It - # used to be installed here with --legacy-peer-deps, which pruned - # eight peer packages and left the tests on a tree the lockfile does - # not describe.) - run: node -e "require('@swc/core').transformSync('let a = 1')" - - - name: Build shared - run: npm run build --workspace=shared - - - name: Write MinIO license - run: printf '%s' "${{ secrets.MINIO_LICENSE_KEY }}" > minio-licence.key - - - name: Start MinIO - # `services:` can't pass the `server /data --license ...` command to - # the AIStor image, so it starts via docker run. 127.0.0.1 endpoint is - # load-bearing: aws-lite picks path-style only for IP/localhost hosts, - # which is all stock MinIO/AIStor speaks. - run: | - docker run -d --name minio -p 127.0.0.1:9000:9000 \ - -v "$PWD/minio-licence.key:/minio.license:ro" \ - -e MINIO_ROOT_USER=trekci -e MINIO_ROOT_PASSWORD=trekci-secret \ - quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z server /data --license /minio.license - - - name: Wait for MinIO - # curl was removed from the AIStor image — poll from the runner instead. - # 60 x 2s gives a wider margin on cold runners than the 30 x 1s the - # pre-6fbff835 job used. - run: | - for _ in $(seq 1 60); do - curl -sf http://127.0.0.1:9000/minio/health/live && exit 0 - sleep 2 - done - echo "MinIO never became healthy" >&2 - docker logs minio >&2 - exit 1 - - - name: Run the S3 driver contract suite - # No --coverage: the src/nest/storage/** 94/84/97/94 pin must keep - # holding from hermetic tests alone. The suite's beforeAll creates - # the trek-test bucket idempotently — no separate bucket-init step. - env: - TREK_TEST_S3_ENDPOINT: http://127.0.0.1:9000 - TREK_TEST_S3_BUCKET: trek-test - TREK_TEST_S3_ACCESS_KEY_ID: trekci - TREK_TEST_S3_SECRET_ACCESS_KEY: trekci-secret - run: cd server && npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts - - # TREK on Postgres, once per supported major: the oldest (14) and the newest - # (18). Both steps run against the same service, the probe on an empty - # database of its own, the lane on databases it creates and drops. - # - # The probe (server/scripts/pg-probe.ts): against an empty Postgres it - # derives a schema from the entities, runs every dialect helper on SQLite - # and Postgres (every case must pass), and calls every public repository - # method once, each statement in its own rolled-back transaction. The - # statements Postgres refuses as SQL are held per method to - # server/scripts/pg-probe-baseline.json, and so is the list of methods it - # could not measure (uncovered); both may only shrink, on every major. An - # unmeasured baseline (failing or uncovered null) or a run that called no - # method fails. The measured baseline and the full report are uploaded - # either way (pg-probe-baseline-), so a change that fixes a statement - # can lower the file without a local Postgres. - # - # The lane (server/vitest.pg.config.ts): every suite that boots the app on - # the real database module, found by server/tests/postgres/lane.ts, each file - # on its own copy of one migrated template, plus the Postgres-only suites - # (PARITY-PG, the write lock, the search semantics). It fails, never skips, - # without TREK_TEST_DB_URL, and TREK_TEST_PG_MAJOR makes it refuse a service - # of another major. It runs after a red probe too, so one run shows both. - # - # The lane runs the whole server suite that boots the app. Each file - # copies the template with CREATE DATABASE, one copy at a time, and before - # Postgres 15 a copy is two checkpoints, each waiting for fsync. With fsync - # on, the lane's test step took 20 to 23 minutes on 14 and about 12 on 18, - # summed over the slices (runs 44194202635 and 44200029312). The service - # runs with fsync off (the first step): a test database has nothing to - # keep through a crash. Measured in Docker on a workstation, 30 copies of - # a 120-table template took 24.9 s on 14 with fsync and 3.5 s without, - # and 5.3 s on 17 with it. - # - # Each major runs the lane in two slices (--shard=/2), each on a runner - # and a service of its own; each slice builds its own template in global - # setup, and a slice spends a little over a minute before the lane starts - # (checkout, install, the shared build). Two is the fewest that keeps a - # slice of 18 about as long as a test shard (six to eight minutes), and of - # 14 too once fsync off brings it to 18's time: every slice is a long job - # (see Job count at the top), and one slice more is one more job that may - # wait for a slot. If a slice of 14 still runs longer than a test shard, - # give 14 one slice more rather than both majors; that needs a count per - # major (a matrix include), which scripts/ci/workflow-jobs.test.mjs does - # not read yet and fails on until it is taught to. The probe runs once - # per major, in the first slice. ci-ok needs this job, and a matrix job's - # result is failure when any one slice fails or is cancelled, so every - # slice is required. The shard axis, the --shard divisor and the "(i/n)" - # in the name must agree; scripts/ci/workflow-jobs.test.mjs fails when - # they do not. - server-postgres: - name: 'Server: Postgres ${{ matrix.postgres }} (${{ matrix.shard }}/2)' - runs-on: ubuntu-24.04 - timeout-minutes: 20 - needs: changes - if: needs.changes.outputs.code == 'true' - strategy: - fail-fast: false - matrix: - postgres: [14, 18] - shard: [1, 2] - services: - postgres: - image: postgres:${{ matrix.postgres }} - env: - POSTGRES_PASSWORD: lane - POSTGRES_DB: trek_probe - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres -d trek_probe" - --health-interval 5s - --health-timeout 5s - --health-retries 20 - steps: - # A service container takes no server arguments, so fsync is turned off - # through ALTER SYSTEM; it is a reload setting, no restart needed. Why - # it is off: see the comment above the job. - - name: Turn fsync off on the service - env: - SERVICE: ${{ job.services.postgres.id }} - run: | - set -euo pipefail - docker exec "$SERVICE" psql -U postgres -d trek_probe -c 'ALTER SYSTEM SET fsync = off' -c 'SELECT pg_reload_conf()' - if [ "$(docker exec "$SERVICE" psql -U postgres -d trek_probe -Atc 'SHOW fsync')" != off ]; then - echo "::error::fsync is still on after the reload; the lane would pass but take about twice as long on 14." - exit 1 - fi - - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci - - # The probe reads the repositories' parameter types, some of which come - # from @trek/shared's built declarations. - - name: Build shared - run: npm run build --workspace=shared - - - name: Run the Postgres probe - if: matrix.shard == 1 - env: - TREK_PG_PROBE_URL: postgres://postgres:lane@127.0.0.1:5432/trek_probe - run: >- - cd server && npm run probe:pg -- - --next-baseline="$RUNNER_TEMP/pg-probe-baseline.json" - --report="$RUNNER_TEMP/pg-probe-report.json" - - - name: Upload the measured baseline and the report - if: ${{ !cancelled() && matrix.shard == 1 }} - uses: actions/upload-artifact@v6 - with: - name: pg-probe-baseline-${{ matrix.postgres }} - path: | - ${{ runner.temp }}/pg-probe-baseline.json - ${{ runner.temp }}/pg-probe-report.json - if-no-files-found: ignore - retention-days: 14 - - - name: Run the Postgres lane - if: ${{ !cancelled() }} - env: - TREK_TEST_DB_URL: postgres://postgres:lane@127.0.0.1:5432/postgres - TREK_TEST_PG_MAJOR: ${{ matrix.postgres }} - run: npm run test:pg --workspace=server -- --shard=${{ matrix.shard }}/2 - - # The server suite in seven slices. vitest.shard.config.ts is vitest.config.ts - # without the thresholds and the coverage reports: a slice's blob carries - # its coverage to server-coverage, which checks the whole. fail-fast is off, - # so one red run shows every failing slice and not just the first. - server-tests: - name: 'Server: Tests ${{ matrix.shard }}/7' - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7] - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci - - - name: Check @swc/core's native binding loads - # The server's SWC transform (server/vitest.config.ts) needs it. The - # lockfile carries the Linux binary, so npm ci installs it; this only - # names the cause if a lockfile regenerated without it ever lands. (It - # used to be installed here with --legacy-peer-deps, which pruned - # eight peer packages and left the tests on a tree the lockfile does - # not describe.) - run: node -e "require('@swc/core').transformSync('let a = 1')" - - - name: Build shared - run: npm run build --workspace=shared - - - name: Run a slice of the tests with coverage - run: >- - cd server && npx vitest run --config vitest.shard.config.ts --coverage - --shard=${{ matrix.shard }}/7 - --reporter=default --reporter=github-actions --reporter=blob - - - name: Upload the blob report - uses: actions/upload-artifact@v6 - with: - name: server-blob-${{ matrix.shard }} - path: server/.vitest-reports/ - include-hidden-files: true - if-no-files-found: error - retention-days: 1 - - server-coverage: - name: 'Server: Coverage' - needs: server-tests - runs-on: ubuntu-24.04 timeout-minutes: 15 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci - - - name: Download the blob reports - uses: actions/download-artifact@v7 - with: - pattern: server-blob-* - path: server/.vitest-reports/ - merge-multiple: true - - - name: Merge the slices and check the thresholds - # Every blob holds a whole coverage map, and the merge reads them all - # into one process. - env: - NODE_OPTIONS: --max-old-space-size=6144 - run: cd server && npx vitest run --merge-reports --coverage - - - name: Upload coverage - uses: actions/upload-artifact@v6 - with: - name: sonar-server-coverage - path: server/coverage/ - retention-days: 1 - - # The client suite in seven slices, the same way as server-tests. The two - # suites cost about the same per slice at seven each (~4-6 minutes). - client-tests: - name: 'Client: Tests ${{ matrix.shard }}/7' - needs: changes - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 30 strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5, 6, 7] + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - # Only the shared and client workspaces — a full root npm ci is not + # Only the shared and client workspaces: a full root npm ci is not # needed, and the scoped install keeps @testing-library/dom (a peer of - # @testing-library/react) in the tree without the re-resolution dance - # the folded coverage job used to need. - - name: Install dependencies - run: npm ci --workspace shared --workspace client - - - name: Build shared - run: npm run build --workspace=shared + # @testing-library/react) in the tree. + - uses: ./.github/actions/setup + with: + scope: client + build-shared: 'true' - name: Run a slice of the tests with coverage run: >- cd client && npx vitest run --config vitest.shard.config.ts --coverage - --shard=${{ matrix.shard }}/7 + --shard=${{ matrix.shard }}/10 --reporter=default --reporter=github-actions --reporter=blob - name: Upload the blob report @@ -1035,21 +972,16 @@ jobs: retention-days: 1 client-coverage: - name: 'Client: Coverage' + name: 'Client coverage' needs: client-tests runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 10 steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + - uses: ./.github/actions/setup with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json - - - name: Install dependencies - run: npm ci --workspace shared --workspace client + scope: client - name: Download the blob reports uses: actions/download-artifact@v7 @@ -1072,77 +1004,343 @@ jobs: path: client/coverage/ retention-days: 1 - shared-coverage: - name: 'Shared: Coverage' + # The browser flows in two slices, in the Playwright image of the version + # the lockfile pins. The image carries Chromium, WebKit and the system + # libraries they need, so no step installs them from the Ubuntu mirror + # (that apt step used to hang for 20 minutes now and then). Dependabot bumps + # only the lockfile: scripts/ci/workflow-jobs.test.mjs fails until the tag + # here names the same version, so a Playwright bump moves the tag and its + # digest by hand in the same PR (the digest of the multi-arch index, from + # `docker buildx imagetools inspect mcr.microsoft.com/playwright:v-noble`). + # + # The job runs as root in the container; git is in the image, so checkout + # clones as usual, and Chromium runs without its sandbox as Playwright does + # for root on its own. Each slice boots its own server on its own SQLite + # file (e2e/server-launch.mjs) and runs the setup project before its share + # of the app flows. + client-e2e: + name: 'Client e2e ${{ matrix.shard }}/2' needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 15 + container: + image: mcr.microsoft.com/playwright:v1.65.0-noble@sha256:9beb546099e11f52d62e505537914870e01a67e0e6b00ae41da4775324e73dbe + options: --ipc=host + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + # The run boots the real server (e2e/server-launch.mjs builds it and + # starts it against a throwaway SQLite file) and the Vite dev server in + # front of it, so it needs every workspace. + - uses: ./.github/actions/setup with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: package-lock.json + scope: full + build-shared: 'true' - - name: Install dependencies - run: npm ci --workspace shared + # The public and app projects are the regression flows. The screenshot + # and help-media projects make pictures for the wiki and the help center + # and assert nothing, so they stay out of CI. + - name: Run a slice of the end-to-end flows + run: >- + cd client && npx playwright test --project=public --project=app + --shard=${{ matrix.shard }}/2 - - name: Run tests with coverage - run: cd shared && npm run test:coverage - - - name: Upload coverage + - name: Upload traces + if: failure() uses: actions/upload-artifact@v6 with: - name: sonar-shared-coverage - path: shared/coverage/ - retention-days: 1 + name: e2e-results-${{ matrix.shard }} + path: client/test-results/ + retention-days: 7 - # Typecheck and tests in one job: each takes seconds, and a runner of its - # own would cost more in setup than it saves. - plugin-sdk: - name: 'Plugin SDK: Types & Coverage' + # The image every release ships, built the way docker.yml builds it and + # started the way docker-compose.yml starts it (read-only root, every + # capability dropped but the three the entrypoint needs, bind-mounted data + # and uploads). Nothing else in CI builds the Dockerfile: the unit tests only + # read it as text, which is how a Dockerfile that could not be parsed + # reached dev unseen. It is separate from the vulnerability scan on purpose, + # so a newly published CVE cannot hide a broken build or the other way round. + # + # The Helm chart and the compose files are what self-hosters deploy from, + # and before this check only a release ever touched them. The chart is + # linted, rendered with the defaults and with each secret and persistence + # mode, and the output checked against the Kubernetes API schemas. Both + # tools come from their release pages, pinned by version and checksum. + # + # One job for both: the image steps run when the image area changed, the + # chart and compose steps when the deploy area changed, exactly as the two + # jobs this replaced did. Each step names the step it builds on, so a red + # chart step does not skip the image steps or the other way round. + docker: + name: 'Docker and Helm' needs: changes - if: needs.changes.outputs.code == 'true' + if: needs.changes.outputs.image == 'true' || needs.changes.outputs.deploy == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 45 - defaults: - run: - working-directory: plugin-sdk - + timeout-minutes: 15 + env: + HELM_VERSION: v3.22.0 + HELM_SHA256: 1e4ab49e429626cf6c6958d914248b78c9730803c2751b87627e171dc800e7bb + KUBECONFORM_VERSION: v0.8.0 + KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 steps: - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 with: - # The image's Node runs the plugins, and it has the node:sqlite the - # dev-server tests exercise. - node-version-file: .nvmrc + persist-credentials: false - # plugin-sdk is standalone (own lockfile, not a root workspace), so it - # installs from its own directory. - - name: Install dependencies - run: npm ci + - name: Install helm and kubeconform + id: helm-install + if: needs.changes.outputs.deploy == 'true' + run: | + set -euo pipefail + bin="$RUNNER_TEMP/bin" + mkdir -p "$bin" + cd "$RUNNER_TEMP" + # GitHub's release downloads answer 504 now and then: retry, the checksum still decides. + fetch() { curl -sSfL --retry 5 --retry-delay 15 --retry-all-errors -o "$1" "$2"; } + fetch helm.tgz "https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz" + echo "$HELM_SHA256 helm.tgz" | sha256sum -c - + tar -xzf helm.tgz --strip-components=1 -C "$bin" linux-amd64/helm + fetch kubeconform.tgz "https://github.com/yannh/kubeconform/releases/download/$KUBECONFORM_VERSION/kubeconform-linux-amd64.tar.gz" + echo "$KUBECONFORM_SHA256 kubeconform.tgz" | sha256sum -c - + tar -xzf kubeconform.tgz -C "$bin" kubeconform + echo "$bin" >> "$GITHUB_PATH" - - name: Typecheck - run: npm run typecheck + - name: Lint the chart + if: ${{ !cancelled() && steps.helm-install.outcome == 'success' }} + run: helm lint --strict charts/trek - - name: Run tests with coverage - run: npm run test:coverage + - name: Render the chart + id: helm-render + if: ${{ !cancelled() && steps.helm-install.outcome == 'success' }} + run: | + set -euo pipefail + out="$RUNNER_TEMP/rendered" + mkdir -p "$out" + helm template trek charts/trek > "$out/defaults.yaml" + helm template trek charts/trek --set existingSecret=trek-secrets > "$out/existing-secret.yaml" + helm template trek charts/trek --set generateEncryptionKey=true > "$out/generated-key.yaml" + helm template trek charts/trek \ + --set persistence.enabled=false \ + --set ingress.enabled=true \ + --set 'ingress.tls[0].secretName=trek-tls' \ + --set 'ingress.tls[0].hosts[0]=chart-example.local' \ + --set 'extraEnv[0].name=SESSION_DURATION' \ + --set 'extraEnv[0].value=7d' \ + --set 'extraEnvFrom[0].secretRef.name=trek-smtp' \ + --set 'extraVolumes[0].name=kitinerary-extractors' \ + --set 'extraVolumes[0].configMap.name=trek-kitinerary-extractors' \ + --set 'extraVolumeMounts[0].name=kitinerary-extractors' \ + --set 'extraVolumeMounts[0].mountPath=/usr/local/share/kitinerary/extractors' \ + --set 'extraVolumeMounts[0].readOnly=true' \ + --set probes.liveness=null \ + > "$out/variants.yaml" + helm template trek charts/trek \ + --set httpRoute.enabled=true \ + --set 'httpRoute.parentRefs[0].name=gateway' \ + > "$out/http-route.yaml" + helm template trek charts/trek \ + --set 'persistence.data.accessModes[0]=ReadWriteMany' \ + --set 'persistence.uploads.accessModes[0]=ReadWriteMany' \ + > "$out/access-modes.yaml" + helm template trek charts/trek \ + --set podSecurityContext.runAsUser=1000 \ + --set podSecurityContext.runAsGroup=1000 \ + --set podSecurityContext.runAsNonRoot=true \ + --set podSecurityContext.seccompProfile.type=RuntimeDefault \ + --set securityContext.allowPrivilegeEscalation=false \ + --set 'securityContext.capabilities.drop[0]=ALL' \ + > "$out/non-root.yaml" + helm template trek charts/trek \ + --set ingress.enabled=true \ + --set fullnameOverride=custom-trek \ + --set service.port=8080 \ + --set-json 'httpRoute={"enabled":true,"parentRefs":[{"name":"gateway","namespace":"gateway-system","sectionName":"https"}],"hostnames":["trek.example.com"],"paths":["/","/api"],"annotations":{"example.com/note":"custom route"},"timeouts":{"request":"0s","backendRequest":"0s"}}' \ + > "$out/http-route-variants.yaml" - - name: Upload coverage - uses: actions/upload-artifact@v6 + - name: Check the manifests against the Kubernetes schemas + # The pinned Gateway API schema adds HTTPRoute validation without skipping unknown resources. + if: ${{ !cancelled() && steps.helm-render.outcome == 'success' }} + run: >- + kubeconform -strict -summary + -schema-location default + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/041baf4c1d3740ca21461cab765e7432ecd073ef/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' + "$RUNNER_TEMP"/rendered/*.yaml + + - name: Check every compose file + if: ${{ !cancelled() && needs.changes.outputs.deploy == 'true' }} + run: | + set -euo pipefail + shopt -s nullglob + files=(docker-compose*.yml docker-compose*.yaml) + [ "${#files[@]}" -gt 0 ] || { echo "::error::No compose file found."; exit 1; } + for f in "${files[@]}"; do + echo "$f" + docker compose -f "$f" config --quiet + done + + - name: Set up buildx + id: buildx + if: ${{ !cancelled() && needs.changes.outputs.image == 'true' }} + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + # Only pushes write the layer cache. Pull requests read what the last + # push to their base left, so a stream of PRs cannot evict it and the + # cache stays within the repository's quota. + - name: Build the image + id: image-build + if: ${{ !cancelled() && steps.buildx.outcome == 'success' }} + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: - name: sonar-plugin-sdk-coverage - path: plugin-sdk/coverage/ - retention-days: 1 + context: . + platforms: linux/amd64 + push: false + load: true + tags: trek:ci + build-args: | + APP_VERSION=0.0.0-ci + cache-from: type=gha,scope=trek-image + cache-to: ${{ github.event_name == 'push' && 'type=gha,mode=max,scope=trek-image' || '' }} + + - name: Start the container + id: image-start + if: ${{ !cancelled() && steps.image-build.outcome == 'success' }} + run: | + mkdir -p "$RUNNER_TEMP/trek/data" "$RUNNER_TEMP/trek/uploads" + docker run -d --name trek-ci -p 127.0.0.1:3000:3000 \ + --read-only --tmpfs /tmp:noexec,nosuid,size=128m \ + --security-opt no-new-privileges:true \ + --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add KILL \ + -v "$RUNNER_TEMP/trek/data:/app/data" \ + -v "$RUNNER_TEMP/trek/uploads:/app/uploads" \ + -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ + trek:ci + + # Ready means migrated, seeded and answering from the database. A first + # boot migrates an empty database, which takes seconds; three minutes + # is the bound, not the expectation. + - name: Wait until the server is ready + id: image-ready + if: ${{ !cancelled() && steps.image-start.outcome == 'success' }} + run: | + for _ in $(seq 1 90); do + if [ "$(docker inspect -f '{{.State.Running}}' trek-ci)" != "true" ]; then + echo "::error::The container exited before it became ready." + exit 1 + fi + if curl -sf http://127.0.0.1:3000/api/health/ready; then + echo + exit 0 + fi + sleep 2 + done + echo "::error::/api/health/ready did not answer within 180 seconds." + exit 1 + + - name: Check what the image serves + id: image-serves + if: ${{ !cancelled() && steps.image-ready.outcome == 'success' }} + run: | + set -euo pipefail + version="$(docker exec trek-ci cat /app/server/VERSION)" + [ "$version" = "0.0.0-ci" ] || { echo "::error::VERSION holds '$version'"; exit 1; } + curl -sf http://127.0.0.1:3000/ | grep -q '
' \ + || { echo "::error::/ does not serve the client"; exit 1; } + + # Same capabilities as the shipped compose file: dumb-init has to be able + # to pass the stop signal on to node, or docker stop ends in SIGKILL (137). + - name: Stop gracefully + id: image-stop + if: ${{ !cancelled() && steps.image-serves.outcome == 'success' }} + run: | + set -euo pipefail + docker stop -t 30 trek-ci >/dev/null + code="$(docker inspect -f '{{.State.ExitCode}}' trek-ci)" + [ "$code" = "0" ] || { echo "::error::docker stop ended with exit code $code, not a graceful shutdown"; exit 1; } + docker logs trek-ci 2>&1 | grep 'SIGTERM received' >/dev/null \ + || { echo "::error::the server never saw SIGTERM"; exit 1; } + + # The same image on a Postgres database (TREK_DB_URL): an empty database + # gets the baseline schema, the admin is seeded, and the server answers + # ready. The database runs beside it on a private network, never exposed. + # It is the newest supported major, the one the image's client tools are + # for; server-postgres covers the oldest. + - name: Start the container on Postgres + id: pg-start + if: ${{ !cancelled() && steps.image-stop.outcome == 'success' }} + run: | + set -euo pipefail + docker network create trek-ci-pg + docker run -d --name trek-ci-db --network trek-ci-pg \ + -e POSTGRES_PASSWORD=ci -e POSTGRES_DB=trek postgres:18 + for _ in $(seq 1 30); do + docker exec trek-ci-db pg_isready -U postgres -d trek >/dev/null 2>&1 && break + sleep 2 + done + docker exec trek-ci-db pg_isready -U postgres -d trek \ + || { echo "::error::Postgres did not accept connections within 60 seconds."; exit 1; } + mkdir -p "$RUNNER_TEMP/trek-pg/data" "$RUNNER_TEMP/trek-pg/uploads" + docker run -d --name trek-ci-on-pg --network trek-ci-pg -p 127.0.0.1:3001:3000 \ + --read-only --tmpfs /tmp:noexec,nosuid,size=128m \ + --security-opt no-new-privileges:true \ + --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add KILL \ + -v "$RUNNER_TEMP/trek-pg/data:/app/data" \ + -v "$RUNNER_TEMP/trek-pg/uploads:/app/uploads" \ + -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ + -e TREK_DB_URL=postgres://postgres:ci@trek-ci-db:5432/trek \ + trek:ci + + # Twice: the first start creates the schema in the empty database, the + # restart finds it migrated and must start without touching it. + - name: Wait until the server on Postgres is ready, then again after a restart + id: pg-ready + if: ${{ !cancelled() && steps.pg-start.outcome == 'success' }} + run: | + wait_ready() { + for _ in $(seq 1 90); do + if [ "$(docker inspect -f '{{.State.Running}}' trek-ci-on-pg)" != "true" ]; then + echo "::error::The container on Postgres exited before it became ready." + exit 1 + fi + if curl -sf http://127.0.0.1:3001/api/health/ready; then + echo + return 0 + fi + sleep 2 + done + echo "::error::/api/health/ready on Postgres did not answer within 180 seconds." + exit 1 + } + wait_ready + docker restart -t 30 trek-ci-on-pg >/dev/null + wait_ready + + - name: Stop the container on Postgres gracefully + if: ${{ !cancelled() && steps.pg-ready.outcome == 'success' }} + run: | + set -euo pipefail + docker stop -t 30 trek-ci-on-pg >/dev/null + code="$(docker inspect -f '{{.State.ExitCode}}' trek-ci-on-pg)" + [ "$code" = "0" ] || { echo "::error::docker stop ended with exit code $code, not a graceful shutdown"; exit 1; } + + - name: Container log + if: always() && needs.changes.outputs.image == 'true' + run: docker logs trek-ci 2>&1 || true + + - name: Container log on Postgres + if: always() && needs.changes.outputs.image == 'true' + run: | + docker logs trek-ci-on-pg 2>&1 || true + docker logs trek-ci-db 2>&1 | tail -50 || true scan: - name: 'Repo: Sonar Scan' + name: 'Sonar' runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 20 # Fork PRs and Dependabot runs don't receive SONAR_TOKEN, so skip the scan # rather than fail it. if: >- @@ -1150,18 +1348,19 @@ jobs: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) # Coverage jobs failing must stop the scan: a missing lcov file would make # every line of that package report as uncovered, and the dashboard would - # show a collapse that did not happen. The gates run beside the tests now, - # so they are listed here too, and a failed gate still stops the scan. + # show a collapse that did not happen. The gates run beside the tests, so + # they are listed here too, and a failed gate still stops the scan. + # scripts/ci/workflow-jobs.test.mjs fails while a job that uploads a + # sonar-*-coverage artifact is missing here. needs: - - i18n-parity + - shared - server-types - - server-lint-checks - - client-types-checks + - server-lint + - client-types + - client-checks - client-lint - server-coverage - client-coverage - - shared-coverage - - plugin-sdk steps: - uses: actions/checkout@v6 @@ -1214,288 +1413,26 @@ jobs: # reviewable in a PR rather than buried in workflow YAML. args: -Dsonar.projectVersion=4.0.0 - # The image every release ships, built the way docker.yml builds it and - # started the way docker-compose.yml starts it (read-only root, every - # capability dropped but the three the entrypoint needs, bind-mounted data - # and uploads). Nothing else in CI builds the Dockerfile: the unit tests only - # read it as text, which is how a Dockerfile that could not be parsed - # reached dev unseen. It is separate from the vulnerability scan on purpose, - # so a newly published CVE cannot hide a broken build or the other way round. - docker-image: - name: 'Repo: Docker Image' - needs: changes - if: needs.changes.outputs.image == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - # Only pushes write the layer cache. Pull requests read what the last - # push to their base left, so a stream of PRs cannot evict it and the - # cache stays within the repository's quota. - - name: Build the image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - platforms: linux/amd64 - push: false - load: true - tags: trek:ci - build-args: | - APP_VERSION=0.0.0-ci - cache-from: type=gha,scope=trek-image - cache-to: ${{ github.event_name == 'push' && 'type=gha,mode=max,scope=trek-image' || '' }} - - - name: Start the container - run: | - mkdir -p "$RUNNER_TEMP/trek/data" "$RUNNER_TEMP/trek/uploads" - docker run -d --name trek-ci -p 127.0.0.1:3000:3000 \ - --read-only --tmpfs /tmp:noexec,nosuid,size=128m \ - --security-opt no-new-privileges:true \ - --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add KILL \ - -v "$RUNNER_TEMP/trek/data:/app/data" \ - -v "$RUNNER_TEMP/trek/uploads:/app/uploads" \ - -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ - trek:ci - - # Ready means migrated, seeded and answering from the database. A first - # boot migrates an empty database, which takes seconds; three minutes - # is the bound, not the expectation. - - name: Wait until the server is ready - run: | - for _ in $(seq 1 90); do - if [ "$(docker inspect -f '{{.State.Running}}' trek-ci)" != "true" ]; then - echo "::error::The container exited before it became ready." - exit 1 - fi - if curl -sf http://127.0.0.1:3000/api/health/ready; then - echo - exit 0 - fi - sleep 2 - done - echo "::error::/api/health/ready did not answer within 180 seconds." - exit 1 - - - name: Check what the image serves - run: | - set -euo pipefail - version="$(docker exec trek-ci cat /app/server/VERSION)" - [ "$version" = "0.0.0-ci" ] || { echo "::error::VERSION holds '$version'"; exit 1; } - curl -sf http://127.0.0.1:3000/ | grep -q '
' \ - || { echo "::error::/ does not serve the client"; exit 1; } - - # Same capabilities as the shipped compose file: dumb-init has to be able - # to pass the stop signal on to node, or docker stop ends in SIGKILL (137). - - name: Stop gracefully - run: | - set -euo pipefail - docker stop -t 30 trek-ci >/dev/null - code="$(docker inspect -f '{{.State.ExitCode}}' trek-ci)" - [ "$code" = "0" ] || { echo "::error::docker stop ended with exit code $code, not a graceful shutdown"; exit 1; } - docker logs trek-ci 2>&1 | grep 'SIGTERM received' >/dev/null \ - || { echo "::error::the server never saw SIGTERM"; exit 1; } - - # The same image on a Postgres database (TREK_DB_URL): an empty database - # gets the baseline schema, the admin is seeded, and the server answers - # ready. The database runs beside it on a private network, never exposed. - # It is the newest supported major, the one the image's client tools are - # for; server-postgres covers the oldest. - - name: Start the container on Postgres - run: | - set -euo pipefail - docker network create trek-ci-pg - docker run -d --name trek-ci-db --network trek-ci-pg \ - -e POSTGRES_PASSWORD=ci -e POSTGRES_DB=trek postgres:18 - for _ in $(seq 1 30); do - docker exec trek-ci-db pg_isready -U postgres -d trek >/dev/null 2>&1 && break - sleep 2 - done - docker exec trek-ci-db pg_isready -U postgres -d trek \ - || { echo "::error::Postgres did not accept connections within 60 seconds."; exit 1; } - mkdir -p "$RUNNER_TEMP/trek-pg/data" "$RUNNER_TEMP/trek-pg/uploads" - docker run -d --name trek-ci-on-pg --network trek-ci-pg -p 127.0.0.1:3001:3000 \ - --read-only --tmpfs /tmp:noexec,nosuid,size=128m \ - --security-opt no-new-privileges:true \ - --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID --cap-add KILL \ - -v "$RUNNER_TEMP/trek-pg/data:/app/data" \ - -v "$RUNNER_TEMP/trek-pg/uploads:/app/uploads" \ - -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ - -e TREK_DB_URL=postgres://postgres:ci@trek-ci-db:5432/trek \ - trek:ci - - # Twice: the first start creates the schema in the empty database, the - # restart finds it migrated and must start without touching it. - - name: Wait until the server on Postgres is ready, then again after a restart - run: | - wait_ready() { - for _ in $(seq 1 90); do - if [ "$(docker inspect -f '{{.State.Running}}' trek-ci-on-pg)" != "true" ]; then - echo "::error::The container on Postgres exited before it became ready." - exit 1 - fi - if curl -sf http://127.0.0.1:3001/api/health/ready; then - echo - return 0 - fi - sleep 2 - done - echo "::error::/api/health/ready on Postgres did not answer within 180 seconds." - exit 1 - } - wait_ready - docker restart -t 30 trek-ci-on-pg >/dev/null - wait_ready - - - name: Stop the container on Postgres gracefully - run: | - set -euo pipefail - docker stop -t 30 trek-ci-on-pg >/dev/null - code="$(docker inspect -f '{{.State.ExitCode}}' trek-ci-on-pg)" - [ "$code" = "0" ] || { echo "::error::docker stop ended with exit code $code, not a graceful shutdown"; exit 1; } - - - name: Container log - if: always() - run: docker logs trek-ci 2>&1 || true - - - name: Container log on Postgres - if: always() - run: | - docker logs trek-ci-on-pg 2>&1 || true - docker logs trek-ci-db 2>&1 | tail -50 || true - - # The Helm chart and the compose files are what self-hosters deploy from, - # and before this job only a release ever touched them. The chart is linted, - # rendered with the defaults and with each secret and persistence mode, and - # the output checked against the Kubernetes API schemas. Both tools come - # from their release pages, pinned by version and checksum. - deploy-manifests: - name: 'Repo: Helm Chart & Compose' - needs: changes - if: needs.changes.outputs.deploy == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 15 - env: - HELM_VERSION: v3.22.0 - HELM_SHA256: 1e4ab49e429626cf6c6958d914248b78c9730803c2751b87627e171dc800e7bb - KUBECONFORM_VERSION: v0.8.0 - KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Install helm and kubeconform - run: | - set -euo pipefail - bin="$RUNNER_TEMP/bin" - mkdir -p "$bin" - cd "$RUNNER_TEMP" - # GitHub's release downloads answer 504 now and then: retry, the checksum still decides. - fetch() { curl -sSfL --retry 5 --retry-delay 15 --retry-all-errors -o "$1" "$2"; } - fetch helm.tgz "https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz" - echo "$HELM_SHA256 helm.tgz" | sha256sum -c - - tar -xzf helm.tgz --strip-components=1 -C "$bin" linux-amd64/helm - fetch kubeconform.tgz "https://github.com/yannh/kubeconform/releases/download/$KUBECONFORM_VERSION/kubeconform-linux-amd64.tar.gz" - echo "$KUBECONFORM_SHA256 kubeconform.tgz" | sha256sum -c - - tar -xzf kubeconform.tgz -C "$bin" kubeconform - echo "$bin" >> "$GITHUB_PATH" - - - name: Lint the chart - run: helm lint --strict charts/trek - - - name: Render the chart - run: | - set -euo pipefail - out="$RUNNER_TEMP/rendered" - mkdir -p "$out" - helm template trek charts/trek > "$out/defaults.yaml" - helm template trek charts/trek --set existingSecret=trek-secrets > "$out/existing-secret.yaml" - helm template trek charts/trek --set generateEncryptionKey=true > "$out/generated-key.yaml" - helm template trek charts/trek \ - --set persistence.enabled=false \ - --set ingress.enabled=true \ - --set 'ingress.tls[0].secretName=trek-tls' \ - --set 'ingress.tls[0].hosts[0]=chart-example.local' \ - --set 'extraEnv[0].name=SESSION_DURATION' \ - --set 'extraEnv[0].value=7d' \ - --set 'extraEnvFrom[0].secretRef.name=trek-smtp' \ - --set 'extraVolumes[0].name=kitinerary-extractors' \ - --set 'extraVolumes[0].configMap.name=trek-kitinerary-extractors' \ - --set 'extraVolumeMounts[0].name=kitinerary-extractors' \ - --set 'extraVolumeMounts[0].mountPath=/usr/local/share/kitinerary/extractors' \ - --set 'extraVolumeMounts[0].readOnly=true' \ - --set probes.liveness=null \ - > "$out/variants.yaml" - helm template trek charts/trek \ - --set httpRoute.enabled=true \ - --set 'httpRoute.parentRefs[0].name=gateway' \ - > "$out/http-route.yaml" - helm template trek charts/trek \ - --set 'persistence.data.accessModes[0]=ReadWriteMany' \ - --set 'persistence.uploads.accessModes[0]=ReadWriteMany' \ - > "$out/access-modes.yaml" - helm template trek charts/trek \ - --set podSecurityContext.runAsUser=1000 \ - --set podSecurityContext.runAsGroup=1000 \ - --set podSecurityContext.runAsNonRoot=true \ - --set podSecurityContext.seccompProfile.type=RuntimeDefault \ - --set securityContext.allowPrivilegeEscalation=false \ - --set 'securityContext.capabilities.drop[0]=ALL' \ - > "$out/non-root.yaml" - helm template trek charts/trek \ - --set ingress.enabled=true \ - --set fullnameOverride=custom-trek \ - --set service.port=8080 \ - --set-json 'httpRoute={"enabled":true,"parentRefs":[{"name":"gateway","namespace":"gateway-system","sectionName":"https"}],"hostnames":["trek.example.com"],"paths":["/","/api"],"annotations":{"example.com/note":"custom route"},"timeouts":{"request":"0s","backendRequest":"0s"}}' \ - > "$out/http-route-variants.yaml" - - - name: Check the manifests against the Kubernetes schemas - # The pinned Gateway API schema adds HTTPRoute validation without skipping unknown resources. - run: >- - kubeconform -strict -summary - -schema-location default - -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/041baf4c1d3740ca21461cab765e7432ecd073ef/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' - "$RUNNER_TEMP"/rendered/*.yaml - - - name: Check every compose file - run: | - set -euo pipefail - shopt -s nullglob - files=(docker-compose*.yml docker-compose*.yaml) - [ "${#files[@]}" -gt 0 ] || { echo "::error::No compose file found."; exit 1; } - for f in "${files[@]}"; do - echo "$f" - docker compose -f "$f" config --quiet - done - # The single check the branch ruleset requires. It needs every job that # gates a merge, so listing a job here is what makes it required. The scan # stays out (see the layout notes at the top). ci-ok: needs: - changes - - i18n-parity + - shared - server-types - - server-lint-checks - - client-types-checks - - client-lint - - client-bundle - - client-e2e - - s3-contract - - server-postgres + - server-lint - server-tests - server-coverage + - server-postgres + - server-s3 + - client-types + - client-checks + - client-lint - client-tests - client-coverage - - shared-coverage - - plugin-sdk - - docker-image - - deploy-manifests + - client-e2e + - docker # always(): without it a failed dependency would skip this job, and the # ruleset reads a skipped required check as passed. if: always() diff --git a/CLAUDE.md b/CLAUDE.md index 14b0b0a86..aaaf1c92d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,13 +137,17 @@ These principles come out of a full-repo audit and shape all new code: ## Quality gates in CI -`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`, client also the strict-mode ratchet `lint:strict`), lint (client and server through `lint:warnings`, which also holds the warning count per rule), `lint:pages`, `lint:rtl`, `lint:size` (client and server), the server's `lint:strict`, `lint:boundaries`, `lint:tx`, `lint:dialect`, `lint:engine-dispatch`, `lint:test-sql`, `lint:test-mocks`, `lint:test-new-service`, `lint:query-api`, `lint:mcp-zod`, `lint:service-http` and `lint:response-contracts`, the client's and server's `lint:format`, `lint:layers`, `lint:offline`, `lint:dup`, `lint:pairs`, `lint:skips`, `lint:i18n-keys` and `theme:lint`, shared's `contracts:open`, the repo-wide `lint:knip` (in server-lint-checks), `check:plugin-facts`, `i18n:parity:strict` (with the untranslated-strings ratchet), the S3 contract suite, TREK on Postgres (`server-postgres`, once per supported major, 14 and 18, each in two slices: first the Postgres probe (in the first slice), every dialect helper on SQLite and Postgres and the statements every repository method reaches with sample arguments, its SQL errors per method and the methods it cannot measure held to a baseline that only shrinks and fails while it is still unmeasured; then the Postgres lane, every suite that boots the app on the real database module, found by `server/tests/postgres/lane.ts` rather than listed, plus PARITY-PG and the write lock; see `server/CLAUDE.md`), the browser end-to-end flows (Chromium and WebKit), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. It also builds the Docker image and boots it until `/api/health/ready` answers, on SQLite and on Postgres 18 (`docker-image`), and lints and renders the Helm chart against the Kubernetes schemas and runs `docker compose config` on every compose file (`deploy-manifests`). `lint-prettier.yml` additionally runs `lint` + `lint:format` on `shared/`, both check-only. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags. +`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`, client also the strict-mode ratchet `lint:strict`), lint (client and server through `lint:warnings`, which also holds the warning count per rule), `lint:pages`, `lint:rtl`, `lint:size` (client and server), the server's `lint:strict`, `lint:boundaries`, `lint:tx`, `lint:dialect`, `lint:engine-dispatch`, `lint:test-sql`, `lint:test-mocks`, `lint:test-new-service`, `lint:query-api`, `lint:mcp-zod`, `lint:service-http` and `lint:response-contracts`, the client's and server's `lint:format`, `lint:layers`, `lint:offline`, `lint:dup`, `lint:pairs`, `lint:skips`, `lint:i18n-keys` and `theme:lint`, shared's `contracts:open`, the repo-wide `lint:knip` (in `server-lint`), `check:plugin-facts`, `i18n:parity:strict` (with the untranslated-strings ratchet), the S3 contract suite, TREK on Postgres (`server-postgres`, once per supported major, 14 and 18, each in three slices: first the Postgres probe (in the first slice), every dialect helper on SQLite and Postgres and the statements every repository method reaches with sample arguments, its SQL errors per method and the methods it cannot measure held to a baseline that only shrinks and fails while it is still unmeasured; then the Postgres lane, every suite that boots the app on the real database module, found by `server/tests/postgres/lane.ts` rather than listed, plus PARITY-PG and the write lock; see `server/CLAUDE.md`), the browser end-to-end flows (Chromium and WebKit, in the Playwright container, in two slices), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. It also builds the Docker image and boots it until `/api/health/ready` answers, on SQLite and on Postgres 18, and lints and renders the Helm chart against the Kubernetes schemas and runs `docker compose config` on every compose file (both in `docker`, each part when its area changed). `lint-prettier.yml` additionally runs `lint` + `lint:format` on `shared/`, both check-only. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags. -- **Job names read "package: what"** (`Server: Types`, `Client: Tests 3/7`), and the list at the top of `test.yml` says what each job holds. A new job follows the scheme and joins that list, and the job-count note there says when a long job needs a slot freed first. -- **Tests run beside the gates, in shards.** The client and server suites run in slices on parallel runners (`client-tests`, `server-tests`, through each package's `vitest.shard.config.ts`), and `client-coverage`/`server-coverage` merge their blob reports and check the thresholds against the whole. A red slice is usually a failing test, a red merge job usually a threshold. A sliced job states its slice count in its matrix's shard axis, the `i/n` in its name and its `--shard=i/n`; `scripts/ci/workflow-jobs.test.mjs` fails until all three agree, and on any job naming `matrix.shard` or `--shard` whose axis is not the one-line `shard: [1, ..., n]` it reads. +- **Job names read " [ ][ i/n]"** (`Server tests 3/10`, `Server Postgres 14 2/3`, `Docker and Helm`): the area first, one of Changes, Shared, Server, Client, Docker or Sonar, then lower case, with no colon, no parentheses and no middle dot, and a slice only at the end, so the checks list sorts by area. The workflow is named `CI`, so a check reads `CI / Server tests 3/10`. Job ids are the names in kebab-case (`server-tests`), the jobs stand in `test.yml` in area order, and the list at the top of `test.yml` says what each job holds. A new job follows the scheme and joins that list; `scripts/ci/workflow-jobs.test.mjs` fails on a name outside it. `ci-ok`, `lint` and `check-target` keep their names: the ruleset requires them by name. +- **What the folded jobs hold.** `Shared tests` is the plugin SDK's typecheck and coverage, run before the root install so the SDK resolves only its own dependencies, then i18n parity and shared's coverage. `Client types` is the typecheck and the strict ratchet; `Client checks` the source ratchets, then the production build with `check:gl-split`. `Docker and Helm` builds and boots the image when the image area changed and checks the chart and compose files when the deploy area changed. Inside a folded job each group of steps runs after a failure in another, so one red check shows every gate that failed; the step names say which. +- **Tests run beside the gates, in shards.** The client and server suites run in ten slices each on parallel runners (`client-tests`, `server-tests`, through each package's `vitest.shard.config.ts`), the Postgres lane in three per major, and the end-to-end flows in two, and `client-coverage`/`server-coverage` merge their blob reports and check the thresholds against the whole. A red slice is usually a failing test, a red merge job usually a threshold. A sliced job states its slice count in its matrix's shard axis, the `i/n` in its name and its `--shard=i/n`; `scripts/ci/workflow-jobs.test.mjs` fails until all three agree, and on any job naming `matrix.shard` or `--shard` whose axis is not the one-line `shard: [1, ..., n]` it reads. - **`ci-ok` is the required check.** The branch ruleset requires `ci-ok` (plus `lint` and `check-target` from their own workflows), and `ci-ok` needs every job above except the scan. It fails if any of them failed or was cancelled and passes if they passed or were skipped. A new gate job counts only once it is listed in `ci-ok`'s `needs:`, and `scripts/ci/workflow-jobs.test.mjs` (in `test:ci-scripts`) fails while a job other than the scan is missing there. - **Guides name paths that exist**: the `changes` job runs `scripts/ci/doc-paths.mjs`, which resolves every backtick path in the CLAUDE.md files, `README.md`, `server/src/nest/README.md` and the wiki against the tracked files; the missing ones are listed in `scripts/ci/doc-paths-baseline.json`, which only shrinks (`--update` after a fix). -- **What runs is decided by `changes`, not by `on.paths`**: `scripts/ci/changed-areas.mjs` maps the changed files to the `code`, `image` and `deploy` areas (tests: `npm run test:ci-scripts`). A path filter on the workflow would leave `ci-ok` unreported on a docs-only PR and block it. Add a new top-level file the build reads to the right area there. +- **What runs is decided by `changes`, not by `on.paths`**: `scripts/ci/changed-areas.mjs` maps the changed files to the `code`, `image` and `deploy` areas (tests: `npm run test:ci-scripts`). A path filter on the workflow would leave `ci-ok` unreported on a docs-only PR and block it. Add a new top-level file the build reads to the right area there. A change to `test.yml`, to `scripts/ci/` or to a local action under `.github/actions/` runs everything. +- **Setup goes through one local action.** Every job that installs dependencies uses `.github/actions/setup/action.yml` (`scope` full, client or shared; `build-shared`) instead of its own `npm ci`; `workflow-jobs.test.mjs` fails on a job that installs on its own (plugin-sdk's own install in its directory excepted). The action restores node_modules on an exact key (runner image, scope, Node ABI, `.nvmrc`, the lockfile and every workspace `package.json`, so npm ci still refuses a `package.json` out of step with the lockfile; no restore-keys) and shared/dist on a key over shared's sources and build config, and runs `npm ci` or the shared build on a miss. Only pushes to dev and main save, the npm download cache included; pull requests only read. +- **End-to-end runs in the Playwright container**, `mcr.microsoft.com/playwright:v-noble` pinned by digest, so no step installs browsers or system libraries. Dependabot bumps only the lockfile: on every Playwright PR move the tag and its digest in `client-e2e` by hand (`docker buildx imagetools inspect mcr.microsoft.com/playwright:v-noble`); `workflow-jobs.test.mjs` fails until the tag names the lockfile's `@playwright/test` version. +- **Budget: `ci-ok` within ten minutes.** One code run starts at most about 35 long jobs (gates, slices, Postgres, e2e, Docker), so two PR runs fit in the organisation's 60 concurrent jobs. Every job sets `timeout-minutes` of at most 20 (15 for the long ones, 10 for the short ones), which `workflow-jobs.test.mjs` holds. A new long job, slice or shard needs a reason that outweighs its slot; the job-count note at the top of `test.yml` has the numbers. - **Runners and Node**: jobs run on `ubuntu-24.04`, not `ubuntu-latest`, and every `setup-node` reads `.nvmrc`. Third-party actions (anything outside `actions/*`) are pinned by commit SHA with the tag in a comment; Dependabot keeps both current. - **Security scans are advisory.** `security.yml` runs `npm audit` (what the image installs, plus `plugin-sdk`) on every PR and push into dev and main, and Docker Scout on the built image for main. Neither is required: a newly published advisory would turn every open PR red at once. Fixes are held as floors in the root `package.json` `overrides`; npm ignores an `overrides` block in a workspace `package.json`, so never put one there. - **Releases** are dispatched by hand and refuse a commit whose `ci-ok` is not green. `docker-dev.yml` (any branch, usually dev) is the canary: it builds the dispatched commit as the next minor or major `-pre.N` and moves only `:latest-pre` and `:-pre`. `docker.yml` (main only) bumps the version, commits and tags it, builds both platforms from exactly that commit and moves `:latest`, `:`, `:.` and `:`. Its `bump=auto` only finalizes a prerelease in flight; without one it stops and asks for patch, minor or major. Ship a release that migrates or changes a lot as a prerelease first, and as a minor, so installs pinned to `:.` can wait it out. diff --git a/scripts/ci/changed-areas.mjs b/scripts/ci/changed-areas.mjs index ab3b5befd..6f84b4d37 100644 --- a/scripts/ci/changed-areas.mjs +++ b/scripts/ci/changed-areas.mjs @@ -23,9 +23,9 @@ import { appendFileSync, readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -// The workflow file and this script decide what runs, so a change to either -// runs everything. -const ALWAYS = ['.github/workflows/test.yml', 'scripts/ci/']; +// The workflow file, the local actions its jobs set up with and this script +// decide what runs and how, so a change to any of them runs everything. +const ALWAYS = ['.github/workflows/test.yml', '.github/actions/', 'scripts/ci/']; // Files the test suites read straight from disk, outside the packages: // config-templates.test.ts pins docker-compose.yml and the Helm chart to the @@ -49,7 +49,7 @@ export const AREAS = { 'package-lock.json', '.nvmrc', 'sonar-project.properties', - // lint:knip in server-lint-checks reads it. + // lint:knip in server-lint reads it. 'knip.jsonc', 'Dockerfile', ...READ_BY_TESTS, diff --git a/scripts/ci/changed-areas.test.mjs b/scripts/ci/changed-areas.test.mjs index c71cf4cdb..d0d371ca0 100644 --- a/scripts/ci/changed-areas.test.mjs +++ b/scripts/ci/changed-areas.test.mjs @@ -95,6 +95,13 @@ describe('classify', () => { assert.deepEqual(classify(['scripts/ci/changed-areas.mjs']), every); }); + it('runs everything when a local action the jobs set up with changes', () => { + assert.deepEqual(classify(['.github/actions/setup/action.yml']), every); + assert.deepEqual(classify(['.github/actions/other/action.yaml']), every); + // Another workflow is not one of them. + assert.deepEqual(classify(['.github/workflows/security.yml']), none); + }); + it('matches whole file names, not prefixes of them', () => { assert.deepEqual(classify(['package.json.bak', 'Dockerfile.old', 'serverless/x.ts', 'wikis/x.png']), none); }); diff --git a/scripts/ci/ci-ok-verdict.mjs b/scripts/ci/ci-ok-verdict.mjs index ab8f0776e..d2df8846d 100644 --- a/scripts/ci/ci-ok-verdict.mjs +++ b/scripts/ci/ci-ok-verdict.mjs @@ -67,7 +67,7 @@ export function verdict(response, sha) { `${run.status === 'completed' ? run.conclusion : run.status} ${run.html_url ?? `check run ${run.id}`}`; if (runs.length === 0) { - return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run Tests & SonarQube on it and dispatch again.`] }; + return { ok: false, lines: [`::error::ci-ok never ran on ${sha}. Run CI on it and dispatch again.`] }; } const lines = runs.map((run) => `ci-ok on ${sha}: ${describe(run)}`); @@ -82,7 +82,7 @@ export function verdict(response, sha) { return diff > 0 || (diff === 0 && b.id > a.id) ? b : a; }); if (newest.conclusion !== 'success') { - lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run Tests & SonarQube on it again, then dispatch again.`); + lines.push(`::error::The newest ci-ok on ${sha} is not green: ${describe(newest)}. Re-run that run, or run CI on it again, then dispatch again.`); return { ok: false, lines }; } for (const run of runs) { diff --git a/scripts/ci/workflow-jobs.test.mjs b/scripts/ci/workflow-jobs.test.mjs index 904ac418c..49e0138a2 100644 --- a/scripts/ci/workflow-jobs.test.mjs +++ b/scripts/ci/workflow-jobs.test.mjs @@ -7,7 +7,10 @@ import { fileURLToPath } from 'node:url'; // is in ci-ok's needs. A new or renamed job that is left out runs, can fail, // and merges anyway. The scan is the one job kept out on purpose (see the // layout notes at the top of test.yml). -const workflow = readFileSync(fileURLToPath(new URL('../../.github/workflows/test.yml', import.meta.url)), 'utf8'); +const read = (path) => readFileSync(fileURLToPath(new URL(path, import.meta.url)), 'utf8'); +const workflow = read('../../.github/workflows/test.yml'); +const setupAction = read('../../.github/actions/setup/action.yml'); +const lockfile = JSON.parse(read('../../package-lock.json')); const NOT_REQUIRED = new Set(['ci-ok', 'scan']); const SHARD = '${{ matrix.shard }}'; @@ -17,6 +20,53 @@ function jobs(source = workflow) { return starts.map((match, i) => ({ id: match[1], text: body.slice(match.index, starts[i + 1]?.index) })); } +/** A job's text without its comment lines, so prose about a step never counts as the step. */ +const withoutComments = (text) => text.replace(/^\s*#.*\n/gm, ''); + +/** The steps of a job, each from its "- " line to the next. */ +const stepsOf = (text) => + withoutComments(text) + .split(/\n(?= {6}- )/) + .slice(1); + +/** The check name a job reports, without the quotes around it; null when it sets none. */ +function nameOf(text) { + const raw = /^ {4}name: (.+)$/m.exec(text)?.[1]; + return raw === undefined ? null : raw.trim().replace(/^'(.*)'$/, '$1').replace(/^"(.*)"$/, '$1'); +} + +/** The job ids under a job's needs:, one per line. */ +function needsOf(text) { + const block = /^ {4}needs:\n((?: {6}- [a-z0-9-]+\n)+)/m.exec(withoutComments(text)); + return block + ? block[1] + .trim() + .split('\n') + .map((line) => line.replace(/^\s*- /, '')) + : []; +} + +// Every check name reads " [ ][ i/n]": the area first, +// from a fixed set, so the checks list sorts by area; no colon, no +// parentheses and no middle dot, and a slice only at the end. +const NAME_FORM = /^(Changes|Shared|Server|Client|Docker|Sonar)( [^:·()]+)?$/; +const SLICE_AT_END = /\$\{\{ matrix\.shard \}\}\/\d+$/; + +/** What is wrong with one job's check name, each said as the fix to make. */ +function nameProblems(id, name) { + if (name === null) return [`${id}: give it a name: of the form " [ i/n]"`]; + const problems = []; + if (!NAME_FORM.test(name)) { + problems.push( + `${id}: "${name}" must start with Changes, Shared, Server, Client, Docker or Sonar and hold no colon, parentheses or middle dot`, + ); + } + if (name.includes(SHARD) && !SLICE_AT_END.test(name)) { + problems.push(`${id}: put the slice "${SHARD}/n" at the end of "${name}"`); + } + return problems; +} + function jobIds() { return jobs().map((job) => job.id); } @@ -101,7 +151,7 @@ describe('test.yml', () => { it('finds the sliced jobs', () => { const ids = slicedJobs().map((job) => job.id); - for (const id of ['server-postgres', 'server-tests', 'client-tests']) { + for (const id of ['client-e2e', 'client-tests', 'server-postgres', 'server-tests']) { assert.ok( ids.includes(id), `${id} no longer names matrix.shard or --shard; when it stopped being sliced, drop it from this list in scripts/ci/workflow-jobs.test.mjs`, @@ -109,6 +159,85 @@ describe('test.yml', () => { } }); + it('names every check " [ i/n]"', () => { + const problems = jobs() + .filter(({ id }) => id !== 'ci-ok') + .flatMap(({ id, text }) => nameProblems(id, nameOf(text))); + assert.deepEqual(problems, []); + }); + + it('keeps ci-ok under its own name, the one the ruleset requires', () => { + const ciOk = jobs().find(({ id }) => id === 'ci-ok'); + assert.ok(ciOk, 'test.yml must define ci-ok'); + assert.equal(nameOf(ciOk.text), null, 'ci-ok reports under its job id; a name: would rename the required check'); + }); + + // A coverage artifact the scan does not wait for may be missing when it + // runs, and Sonar then reports every line of that package as uncovered. + it('lets the scan wait for every job that uploads Sonar coverage', () => { + const scan = jobs().find(({ id }) => id === 'scan'); + assert.ok(scan, 'test.yml must define scan'); + const needs = new Set(needsOf(scan.text)); + const uploaders = jobs() + .filter(({ text }) => + stepsOf(text).some( + (step) => /uses: actions\/upload-artifact@/.test(step) && /name: sonar-[\w-]+-coverage/.test(step), + ), + ) + .map(({ id }) => id); + assert.ok(uploaders.length >= 3, `expected the coverage uploads, found them in: ${uploaders.join(', ')}`); + const missing = uploaders.filter((id) => !needs.has(id)); + assert.deepEqual(missing, [], `add these jobs to scan's needs: in test.yml: ${missing.join(', ')}`); + }); + + // A hung job holds its slot until the timeout, and the default is six hours. + it('bounds every job with a timeout of at most 20 minutes', () => { + const problems = jobs().flatMap(({ id, text }) => { + const minutes = /^ {4}timeout-minutes: (\d+)\s*$/m.exec(text)?.[1]; + if (minutes === undefined) return [`${id}: set timeout-minutes (at most 20)`]; + return Number(minutes) > 20 ? [`${id}: timeout-minutes is ${minutes}, at most 20`] : []; + }); + assert.deepEqual(problems, []); + }); + + // The setup action restores node_modules from an exact cache and runs npm + // ci only on a miss; a job that installs on its own pays the install on + // every run. plugin-sdk has its own lockfile and installs in its own + // directory, outside the workspaces the action covers. + it('installs the workspaces only through ./.github/actions/setup', () => { + const problems = jobs().flatMap(({ id, text }) => + stepsOf(text) + .filter((step) => /\bnpm (ci|install|i)\b/.test(step) && !/^ {8}working-directory: plugin-sdk\s*$/m.test(step)) + .map((step) => `${id}: "${step.split('\n')[0].trim()}" installs with npm; use ./.github/actions/setup`), + ); + assert.deepEqual(problems, []); + assert.ok( + jobs().some(({ text }) => /uses: \.\/\.github\/actions\/setup\s*$/m.test(text)), + 'no job uses ./.github/actions/setup', + ); + }); + + // Dependabot bumps @playwright/test in the lockfile only. The browsers the + // flows run in come from the container, so its tag has to follow by hand, + // and the digest pins exactly that image. + it('runs e2e in the Playwright image of the version the lockfile pins', () => { + const e2e = jobs().find(({ id }) => id === 'client-e2e'); + assert.ok(e2e, 'test.yml must define client-e2e'); + const image = /^ {6}image: (\S+)\s*$/m.exec(withoutComments(e2e.text))?.[1] ?? ''; + const pinned = /^mcr\.microsoft\.com\/playwright:v(\d+\.\d+\.\d+)-noble@sha256:[0-9a-f]{64}$/.exec(image); + assert.ok( + pinned, + `client-e2e's container must be mcr.microsoft.com/playwright:v-noble@sha256:, not "${image}"`, + ); + const locked = lockfile.packages['node_modules/@playwright/test']?.version; + assert.ok(locked, 'package-lock.json has no node_modules/@playwright/test'); + assert.equal( + pinned[1], + locked, + `the lockfile pins @playwright/test ${locked}: set client-e2e's image to v${locked}-noble with that tag's digest (docker buildx imagetools inspect mcr.microsoft.com/playwright:v${locked}-noble)`, + ); + }); + for (const job of slicedJobs()) { it(`slices ${job.id} alike in its matrix, its name and its --shard`, () => { assert.deepEqual(sliceProblems(job), []); @@ -116,6 +245,84 @@ describe('test.yml', () => { } }); +describe('the setup action', () => { + const steps = setupAction.split(/\n(?= {4}- )/).slice(1); + + // A restore-keys fallback would hand a job a tree built from another + // lockfile or on another image. Only the npm download cache may fall back: + // npm ci checks each tarball against the lockfile. + it('restores node_modules and shared/dist only on an exact key', () => { + const restores = steps.filter( + (step) => /actions\/cache\/restore@/.test(step) && !/^ {8}path: ~\/\.npm\s*$/m.test(step), + ); + assert.ok(restores.length >= 2, 'expected a restore for node_modules and one for shared/dist'); + for (const step of restores) assert.doesNotMatch(step, /restore-keys/, step); + }); + + // The full actions/cache action saves in its post step on every event, pull + // requests included, so every cache is restored and saved in two steps. + it('uses only the split restore and save actions', () => { + for (const step of steps) assert.doesNotMatch(step, /uses: actions\/cache@/, step); + }); + + // npm ci refuses a package.json that disagrees with the lockfile; a cache hit + // skips it, so a changed package.json has to miss. + it('keys node_modules on the lockfile and every workspace package.json', () => { + const deps = setupAction.match(/^ +DEPS: \$\{\{ hashFiles\((.*)\) \}\}$/m); + assert.ok(deps, 'expected a DEPS hash in the key step'); + const files = deps[1].split(',').map((file) => file.trim().replace(/^'|'$/g, '')); + const wanted = ['package-lock.json', 'package.json', 'client/package.json', 'server/package.json', 'shared/package.json']; + for (const file of wanted) { + assert.ok(files.includes(file), `the node_modules key does not hash ${file}`); + } + }); + + // Pull requests read what their base left; a PR that saved could evict the + // base's caches and fill the quota with trees of its own. + it('saves only on a push to dev or main', () => { + assert.match( + setupAction, + /save=\$\{\{ github\.event_name == 'push' && \(github\.ref == 'refs\/heads\/dev' \|\| github\.ref == 'refs\/heads\/main'\) \}\}/, + ); + const saves = steps.filter((step) => /actions\/cache\/save@/.test(step)); + assert.ok(saves.length >= 3, 'expected a save for node_modules, the npm download cache and shared/dist'); + for (const step of saves) assert.match(step, /steps\.keys\.outputs\.save == 'true'/, step); + }); +}); + +describe('the name reader', () => { + it('passes names of the form', () => { + const names = [ + 'Changes', + 'Shared tests', + `Server tests ${SHARD}/10`, + `Server Postgres \${{ matrix.postgres }} ${SHARD}/3`, + 'Docker and Helm', + 'Sonar', + ]; + for (const name of names) assert.deepEqual(nameProblems('job', name), [], name); + }); + + it('fails a colon, parentheses, a dot separator, an unknown area, a slice before the end and no name', () => { + for (const name of [ + 'Server: Types', + `Server Postgres 14 (${SHARD}/2)`, + 'Client · Lint', + 'Repo Sonar scan', + `Server tests ${SHARD}/10 extra`, + null, + ]) { + assert.notDeepEqual(nameProblems('job', name), [], String(name)); + } + }); + + it('reads a quoted name without its quotes', () => { + assert.equal(nameOf(" name: 'Server types'\n"), 'Server types'); + assert.equal(nameOf(' name: Server types\n'), 'Server types'); + assert.equal(nameOf(' runs-on: ubuntu-24.04\n'), null); + }); +}); + describe('the slice reader', () => { const run = (arg) => ` steps:\n - run: npx vitest run --shard=${arg}\n`; const workflowWith = (job) => `name: t\njobs:\n sliced:\n${job}`; diff --git a/server/CLAUDE.md b/server/CLAUDE.md index 51167e56f..c77319f03 100644 --- a/server/CLAUDE.md +++ b/server/CLAUDE.md @@ -82,7 +82,7 @@ Nest owns everything. Every domain is a DI module under `src/nest//` (`c - **Plugins** (`src/nest/plugins/`): the child-process runtime (`supervisor/`, `runtime/plugin-host-entry.ts`, `protocol/envelope.ts`) is **intentionally not Nest**; it is the security boundary. `host/rpc-host.ts` registers an RPC method at spawn only if the plugin holds the unlocking permission (registration = authorization). Install-time gates live in `install/`; the manifest's `trek` version range fails closed, and `TREK_PLUGINS_IGNORE_TREK_RANGE` downgrades that to a warning. The protocol tables in `protocol/envelope.ts` (+ `src/plugin-event-sink.ts`) are **generated** into `plugin-sdk/` and `shared/`: run `gen:plugin-facts` after touching them and commit the output. What a plugin gets back is `protocol/output-contract.ts`: every wire method has an entry, and an entity result (`trips.getById`, `todos.list`, `vacay.mine`, ...) is cut down to its published fields in `rpc-host.ts` on the way out, together with every child row it carries whose field is listed in `PLUGIN_ENTITY_NESTED` (a day's `notes_items`, a reservation's `endpoints`). A new column on any whole-row table behind those entities, child tables included, fails `tests/unit/plugins/output-contract.schema.test.ts` until it is listed as published or withheld (the SDK's `PLUGIN_ENTITY_FIELDS` and `PLUGIN_ENTITY_NESTED` are generated from it); a method left as `readModel` must return a value built field by field from named columns. A plugin's own SQL runs in one store per engine behind the `PluginDataStore` port (`src/db/engine/plugin-data.ts`), each in its engine's folder: `SqlitePluginDataStore` (`src/db/engine/sqlite/plugin-data-store.ts`) and `PostgresPluginDataStore` (`src/db/engine/postgres/plugin-data-store.ts`). Both hold it to a row cap, and `query`, `tx` and an `exec` script to a time budget checked between rows and statements (`migrate` is deliberately not timed), and refuse a call that breaks the contract with the same words, taken from `PLUGIN_DATA_ERRORS` beside the caps in `src/db/engine/plugin-data-contract.ts` (`tests/unit/db/engine/plugin-data-contract.test.ts` runs every such call on both stores and fails on one answered differently; a closed store refuses every call). Transaction control is refused in every statement of every call on both engines with one text, `PLUGIN_DATA_ERRORS.txControl` (`refuseTransactionControl` on SQLite and in the SDK, the guard on Postgres). The SDK's dev harness and mock host throw the same texts and hold the same caps from a byte-identical copy of that file, held by a parity test in `plugin-sdk/`. Where that data lives is the engine's: `PLUGIN_DATA` (`src/nest/plugins/host/plugin-data.backend.ts`) binds the SQLite file per plugin, or on Postgres a schema per plugin (`trek_plugin_`) whose SQL is kept inside that schema in layers that `src/db/engine/postgres/plugin-sql-guard.ts`'s header names, the ones after the guard enforced by the database: no plugin SQL at all while TREK's role is privileged in the sense of `PRIVILEGED_ROLE_SQL` (`src/db/engine/postgres/plugin-data-checks.ts`, read once per pool connection, and the boot log says why); the guard, which refuses what it would read differently from the Postgres lexer, other schemas, the catalogs, temporary objects, `SELECT ... INTO`, the `reg*` types, the functions that look an object up by name, a read of a system column (`xmin`, `ctid`, ...; the name may still be given with AS), the `xid` types where the word stands for the type, a collation or access method beyond the ones every server has, and every function of `pg_catalog` outside the allowlist `assets/postgres/plugin-functions.txt`, read by `src/db/engine/postgres/plugin-sql-functions.ts` (its catalog list, `assets/postgres/catalog-functions.txt`, is held to `pg_proc` by PGPLUG-021, and PGPLUG-022 fails on an allowed function with an overload that takes or returns a `reg*` type or `aclitem`, which would turn a text argument into a catalog lookup, and on one that makes a transaction id out of other values); the settings that decide how values print and parse (time zone UTC, the locales `C`, the date style, ...), pinned with `SET LOCAL` in every plugin transaction (`src/db/engine/postgres/plugin-session.ts`; PGPLUG-023 fails on a client setting of those categories a new Postgres adds until it is pinned or named as harmless); the objects an ALTER, DROP, REFRESH, TRUNCATE or CREATE INDEX changes, resolved by Postgres (`to_regclass`/`to_regtype`) right before the statement and refused outside the plugin's schema, so an unqualified `int4` is `pg_catalog`'s and refused even on a superuser connection; one statement per round trip over the extended query protocol, which the store sends on a pool client it checks out for each call and runs its transaction on directly; `pg_temp` last on the plugin's and on every pool connection's `search_path`, `session-search-path.ts`, with the store dropping its client's temporary tables and other session state before the client goes back to the pool; and a quota over every relation of the schema with its TOAST table plus a cap on its catalog objects (`PLUGIN_SCHEMA_MAX_OBJECTS`). Plugin SQL, an argument or a migration id holding NUL is refused on both engines with one error (`refuseNulSql`/`refuseNulValues` in the same contract file, so the SDK refuses it too), since Postgres text cannot hold it and a plugin must answer the same on either; a NUL the plugin's SQLite SQL builds itself (`char(0)`) is not caught, and the move to Postgres refuses such a row; the plugin's SQL is never translated, and `db.engine` tells the plugin which dialect to write. Backups and restores carry the plugin schemas the role owns (`pg-tools.ts`). `runtime/egress-policy.ts` is a hand-kept mirror guarded by a parity test in `plugin-sdk/`; change both sides together. - **Storage**: `src/nest/storage/` is the upload driver abstraction (local / S3 / mirror behind one interface, with an admin surface and a usage job). Domain services never touch the filesystem or the S3 SDK directly. Where the data lives (`data/`, `uploads/`, backups, logs, key files, the DB file) is resolved in one place, `src/app-config/data-paths.ts`; a provider that reads one injects `DataPathsService`, plain helper modules shared with code outside the container call `resolveDataPaths()` (the list is in `src/app-config/README.md`), and nothing anchors a data path on its own `__dirname` or on `process.cwd()`. - **DB** (`src/db/`): `better-sqlite3`, WAL, FK on. `database.ts` only holds the connection and applies pragmas. It does not build a schema, and importing it opens nothing (the first reader does). The connection's lifecycle belongs to the `DatabaseLifecycle` provider (`nest/database/database-lifecycle.service.ts`): `buildApp()` opens it through `open()` (which binds the ORM to later swaps and runs the schema bootstrap below), the shutdown closes it through `close()`, and a restore or the demo reset swaps the file only through the database backup port, `DATABASE_BACKUP` (`nest/database/database-backup.interface.ts`; the SQLite implementation is `nest/backup/sqlite-database-backup.ts`: `VACUUM INTO` snapshot, close, atomic file swap, `reopen()`). The port works on `DatabaseLifecycle.file`, the one resolved path (`TREK_DB_FILE` or `data/travel.db`), and a snapshot that fails fails the backup rather than archiving the live file. `MaintenanceRepository` is provided only by `MaintenanceModule`'s factory and injected everywhere else; no consumer constructs it. A provider that has to follow every reopen (the demo reset job saves the first demo baseline after a restore's re-bootstrap seeded the example trips) registers with `DatabaseLifecycle.onReopened()`, which runs after the reopen's schema bootstrap; a listener contains its own failures, since one that throws fails the reopen. MikroORM's migrations (`db/migrations/`) are the only schema source, **including for tests**: the hand-kept `schema.ts`/`migrations.ts`/`seeds.ts` builders and the parity test that guarded them are gone. Seeding is the MikroORM seeders in `db/seeders/`, applied by `db/orm.ts` from `buildApp()` before anything reads. An install the retired positional runner migrated (a `schema_version` row, no `mikro_orm_migrations` rows) is baselined by `db/legacy-baseline.ts`: the migrations whose `Legacy migration step N` docstring is ≤ its version are recorded as executed (never replayed) in the same transaction as the migrator run, while the baseline migration itself still runs first, as the legacy runner's `createTables()` did on every boot. It refuses the boot when `schema_version` is not exactly one row holding an integer ≥ 1, or is above the last mapped step (258); a value from 1 to 258 is read as dev's steps of those numbers, so a pre-ORM feature-branch database whose steps differ is not detected (the Tours branch is the one case it cuts back). Recorded `mikro_orm_migrations` names this build does not ship refuse it too (`refuseNewerDatabase`), whether a newer release or a development build wrote them. 4.3.x and earlier have no such check and start on a database 5.x migrated. Before a boot applies pending migrations to a file database that holds anything, `db/pre-migrate-snapshot.ts` writes a `VACUUM INTO` copy next to it (`pre-migrate--