From 8e695cc26703825a550edf5b836f040c83d8ffe6 Mon Sep 17 00:00:00 2001 From: Maurice Date: Thu, 8 Oct 2026 11:37:12 +0200 Subject: [PATCH] ci: gate merges on one ci-ok check that covers the image and the chart The ruleset required only `lint` (shared's eslint and prettier) and `check-target`. Types, lint, coverage, end-to-end and the S3 contract were advisory, and the three health PRs merged with red checks. Nothing on a PR into dev built the Dockerfile or rendered the Helm chart either, which is how #2610 shipped a Dockerfile no build could parse. ci-ok needs every gate, test, image and chart job and is meant to be the one check the ruleset requires. It runs with always(), fails when a job it needs failed or was cancelled, and passes when they passed or were skipped. The Sonar scan stays outside: its quality gate reports on the PR by itself, and an outage there should not block merges. A required check cannot sit behind a workflow-level path filter: on a docs-only PR the workflow never starts and the check never reports. The filter moves into a `changes` job. It diffs the PR's merge commit against its base (or a push against the previous head) and scripts/ci/ changed-areas.mjs maps the files to code, image and deploy areas. It fails closed: no usable diff, an empty one or a path it cannot read cleanly runs everything. Its tests run in the same job. docker-image builds the Dockerfile with buildx (layer cache written by pushes only, read by PRs), starts it with the same hardening as docker-compose.yml, waits up to three minutes for /api/health/ready, checks the VERSION file and that / serves the client, and prints the container log. deploy-manifests lints the chart, renders it with the default values and with each secret, persistence and ingress mode, validates the output with kubeconform, and runs `docker compose config` on every compose file. helm and kubeconform are pinned by version and sha256. While the file is open: every job runs on ubuntu-24.04 instead of the moving ubuntu-latest, reads its Node version from .nvmrc (the plugin-sdk jobs ran 22 under a comment saying it matched the server, which runs 24), download-artifact moves from v6, still on Node 20, to v7, and the Sonar action is pinned by commit. --- .github/workflows/test.yml | 400 +++++++++++++++++++++++------- CLAUDE.md | 5 +- package.json | 1 + scripts/ci/changed-areas.mjs | 117 +++++++++ scripts/ci/changed-areas.test.mjs | 100 ++++++++ 5 files changed, 537 insertions(+), 86 deletions(-) create mode 100644 scripts/ci/changed-areas.mjs create mode 100644 scripts/ci/changed-areas.test.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ebeca5eb7..9b0b6e30b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,65 +29,33 @@ permissions: # themselves: s3-contract (MINIO_LICENSE_KEY) and scan (SONAR_TOKEN). # Everything else runs on fork PRs so outside contributors still get the # full test signal. +# - changes runs first and decides which of the jobs below a change needs +# (scripts/ci/changed-areas.mjs holds the rules). A docs-only push or PR +# runs nothing else. The filter lives in a job, not in `on.paths`, because +# a workflow skipped by its path filter never reports a check, and a +# required check that never reports blocks the PR for good. +# - docker-image builds the Dockerfile and boots the result, and +# deploy-manifests lints and renders the Helm chart and checks the +# compose files. Both run beside the preflight rather than after it. +# - ci-ok `needs:` every gate, test, image and chart job, and is the one +# check the branch ruleset requires. It fails when any of them failed or +# was cancelled and passes when they passed or were skipped (by changes, +# or on a fork PR without secrets). The scan is not in it: the quality +# gate reports on the PR by itself, and a SonarCloud outage should not +# block a merge. # # Automatic Analysis must stay disabled on sonarcloud.io — it is mutually # exclusive with CI-based analysis, and it cannot import the lcov coverage # this workflow produces. -# A push or PR that touches none of these paths can't change the tests or the -# analysis, so don't spend the coverage fan-out on it. For a docs-only push -# that means no run and no fresh branch analysis — harmless, since the last -# analysis is still accurate for unchanged code. GitHub Actions doesn't -# support YAML anchors, so the two lists are literal copies: keep them -# identical when editing either one. +# +# Every job runs on ubuntu-24.04 rather than ubuntu-latest, so the move of +# that label to a new Ubuntu release is a change made here, in a PR, and not +# one that lands on its own. on: push: branches: [main, dev] - paths: - - 'server/**' - - 'client/**' - - 'shared/**' - - 'plugin-sdk/**' - # npm workspaces hoist every dependency into the single root lockfile, - # so a dependency bump touches none of the four directories above — and - # a native binding like better-sqlite3 is exactly what needs the suite. - - 'package.json' - - 'package-lock.json' - - 'sonar-project.properties' - - '.github/workflows/test.yml' - # The image's start-up shape is pinned by a unit test - # (server/tests/unit/docker-entrypoint.test.ts), so a change to the - # Dockerfile or to the entrypoint script has to run the suite — that is - # exactly the change the test exists to catch. - - 'Dockerfile' - - 'server/scripts/**' - # Docs inside the workspaces (CLAUDE.md, READMEs, PATTERN.md) affect - # neither the tests nor the scan — no test reads checked-in markdown - # and SonarCloud has no markdown analyzer. Negations must stay last. - - '!**/*.md' pull_request: branches: [main, dev] - paths: - - 'server/**' - - 'client/**' - - 'shared/**' - - 'plugin-sdk/**' - # npm workspaces hoist every dependency into the single root lockfile, - # so a dependency bump touches none of the four directories above — and - # a native binding like better-sqlite3 is exactly what needs the suite. - - 'package.json' - - 'package-lock.json' - - 'sonar-project.properties' - - '.github/workflows/test.yml' - # The image's start-up shape is pinned by a unit test - # (server/tests/unit/docker-entrypoint.test.ts), so a change to the - # Dockerfile or to the entrypoint script has to run the suite — that is - # exactly the change the test exists to catch. - - 'Dockerfile' - - 'server/scripts/**' - # Docs inside the workspaces (CLAUDE.md, READMEs, PATTERN.md) affect - # neither the tests nor the scan — no test reads checked-in markdown - # and SonarCloud has no markdown analyzer. Negations must stay last. - - '!**/*.md' workflow_dispatch: # A newer push to the same branch/PR supersedes the running one — cancel it @@ -98,18 +66,84 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: Changed Areas + runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + code: ${{ steps.areas.outputs.code }} + image: ${{ steps.areas.outputs.image }} + deploy: ${{ steps.areas.outputs.deploy }} + steps: + - uses: actions/checkout@v6 + with: + # On a pull request HEAD is the merge commit GitHub builds of the PR + # into its base, and its first parent is the base, so two commits + # are enough to see exactly what the PR changes. + fetch-depth: 2 + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + + - name: Test the classifier + run: npm run test:ci-scripts + + - name: List the changed files + id: list + env: + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + run: | + set -euo pipefail + list="$RUNNER_TEMP/changed-files.txt" + changed_between() { git -c core.quotePath=false diff --name-only "$1" "$2" > "$list"; } + case "$EVENT" in + pull_request) + changed_between HEAD^1 HEAD + ;; + push) + # A new branch has no previous commit, and a force push can leave + # it unreachable. Both run everything. + if [ -n "$BEFORE" ] && [ "$BEFORE" != "0000000000000000000000000000000000000000" ] && git fetch --no-tags --depth=1 origin "$BEFORE"; then + changed_between "$BEFORE" "$AFTER" + else + echo "No previous commit to compare with, so everything runs." + list="" + fi + ;; + *) + list="" + ;; + esac + echo "list=$list" >> "$GITHUB_OUTPUT" + [ -z "$list" ] || cat "$list" + + - name: Decide what runs + id: areas + env: + LIST: ${{ steps.list.outputs.list }} + run: | + if [ -n "$LIST" ]; then + node scripts/ci/changed-areas.mjs --files "$LIST" + else + node scripts/ci/changed-areas.mjs --all + fi + plugin-facts: name: Plugin Permission Facts - runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: - # 22 matches the TREK server runtime (same choice as the plugin-sdk - # jobs and publish-plugin-sdk.yml). - node-version: 22 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -128,7 +162,9 @@ jobs: i18n-parity: name: i18n Key Parity - runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -136,21 +172,23 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc - name: Check i18n key parity run: node shared/scripts/i18n-parity.mjs --strict server-quality: name: Server Types & Lint - runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -208,7 +246,9 @@ jobs: client-quality: name: Client Types & Lint - runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -216,7 +256,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -249,7 +289,9 @@ jobs: plugin-sdk-quality: name: Plugin SDK Types - runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-24.04 timeout-minutes: 45 defaults: run: @@ -260,7 +302,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 22 + node-version-file: .nvmrc # plugin-sdk is standalone (own lockfile, not a root workspace), so it # installs from its own directory. @@ -272,7 +314,7 @@ jobs: client-bundle: name: Client Bundle - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] steps: @@ -280,7 +322,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -302,7 +344,7 @@ jobs: client-e2e: name: Client End-to-End - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] steps: @@ -310,7 +352,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -359,7 +401,7 @@ jobs: s3-contract: name: S3 Driver Contract (MinIO) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] # Fork PRs don't get repo secrets, so MINIO_LICENSE_KEY would be empty and @@ -373,7 +415,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -411,7 +453,7 @@ jobs: # 60 x 2s gives a wider margin on cold runners than the 30 x 1s the # pre-6fbff835 job used. run: | - for i in $(seq 1 60); do + for _ in $(seq 1 60); do curl -sf http://127.0.0.1:9000/minio/health/live && exit 0 sleep 2 done @@ -432,7 +474,7 @@ jobs: server-coverage: name: Server Coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] steps: @@ -440,7 +482,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -471,7 +513,7 @@ jobs: client-coverage: name: Client Coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] steps: @@ -479,7 +521,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -505,7 +547,7 @@ jobs: shared-coverage: name: Shared Coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] steps: @@ -513,7 +555,7 @@ jobs: - uses: actions/setup-node@v6 with: - node-version: 24 + node-version-file: .nvmrc cache: npm cache-dependency-path: package-lock.json @@ -532,7 +574,7 @@ jobs: plugin-sdk-coverage: name: Plugin SDK Coverage - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: [plugin-facts, i18n-parity, server-quality, client-quality, plugin-sdk-quality] defaults: @@ -544,9 +586,9 @@ jobs: - uses: actions/setup-node@v6 with: - # 22 matches the TREK server runtime and has node:sqlite, which the - # dev-server tests exercise (same choice as publish-plugin-sdk.yml). - node-version: 22 + # The image's Node runs the plugins, and it has the node:sqlite the + # dev-server tests exercise. + node-version-file: .nvmrc # plugin-sdk is standalone (own lockfile, not a root workspace), so it # installs from its own directory. @@ -565,7 +607,7 @@ jobs: scan: name: Scan - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 45 # Fork PRs don't receive SONAR_TOKEN — skip the scan rather than fail it. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -583,31 +625,31 @@ jobs: fetch-depth: 0 - name: Download server coverage - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: sonar-server-coverage path: server/coverage/ - name: Download client coverage - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: sonar-client-coverage path: client/coverage/ - name: Download shared coverage - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: sonar-shared-coverage path: shared/coverage/ - name: Download plugin-sdk coverage - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: name: sonar-plugin-sdk-coverage path: plugin-sdk/coverage/ - name: Scan - uses: SonarSource/sonarqube-scan-action@v8 + uses: SonarSource/sonarqube-scan-action@d209202bc7d53ff1cc128f7f907dac145c9d6ae9 # v8.3.0 env: # SonarQube Cloud is the scan action's default host, so no # SONAR_HOST_URL — the token is a sonarcloud.io token for the @@ -625,3 +667,191 @@ jobs: # false-positive list — lives in sonar-project.properties so it stays # reviewable in a PR rather than buried in workflow YAML. args: -Dsonar.projectVersion=4.0.0 + + # The image every release ships, built the way docker.yml builds it and + # started the way docker-compose.yml starts it (read-only root, every + # capability dropped but the three the entrypoint needs, bind-mounted data + # and uploads). Nothing else in CI builds the Dockerfile: the unit tests only + # read it as text, which is how a Dockerfile that could not be parsed + # reached dev unseen. It is separate from the vulnerability scan on purpose, + # so a newly published CVE cannot hide a broken build or the other way round. + docker-image: + name: Docker Image + needs: changes + if: needs.changes.outputs.image == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + # Only pushes write the layer cache. Pull requests read what the last + # push to their base left, so a stream of PRs cannot evict it and the + # cache stays within the repository's quota. + - name: Build the image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64 + push: false + load: true + tags: trek:ci + build-args: | + APP_VERSION=0.0.0-ci + cache-from: type=gha,scope=trek-image + cache-to: ${{ github.event_name == 'push' && 'type=gha,mode=max,scope=trek-image' || '' }} + + - name: Start the container + run: | + mkdir -p "$RUNNER_TEMP/trek/data" "$RUNNER_TEMP/trek/uploads" + docker run -d --name trek-ci -p 127.0.0.1:3000:3000 \ + --read-only --tmpfs /tmp:noexec,nosuid,size=128m \ + --security-opt no-new-privileges:true \ + --cap-drop ALL --cap-add CHOWN --cap-add SETUID --cap-add SETGID \ + -v "$RUNNER_TEMP/trek/data:/app/data" \ + -v "$RUNNER_TEMP/trek/uploads:/app/uploads" \ + -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ + trek:ci + + # Ready means migrated, seeded and answering from the database. A first + # boot migrates an empty database, which takes seconds; three minutes + # is the bound, not the expectation. + - name: Wait until the server is ready + run: | + for _ in $(seq 1 90); do + if [ "$(docker inspect -f '{{.State.Running}}' trek-ci)" != "true" ]; then + echo "::error::The container exited before it became ready." + exit 1 + fi + if curl -sf http://127.0.0.1:3000/api/health/ready; then + echo + exit 0 + fi + sleep 2 + done + echo "::error::/api/health/ready did not answer within 180 seconds." + exit 1 + + - name: Check what the image serves + run: | + set -euo pipefail + version="$(docker exec trek-ci cat /app/server/VERSION)" + [ "$version" = "0.0.0-ci" ] || { echo "::error::VERSION holds '$version'"; exit 1; } + curl -sf http://127.0.0.1:3000/ | grep -q '
' \ + || { echo "::error::/ does not serve the client"; exit 1; } + + - name: Container log + if: always() + run: docker logs trek-ci 2>&1 || true + + # The Helm chart and the compose files are what self-hosters deploy from, + # and before this job only a release ever touched them. The chart is linted, + # rendered with the defaults and with each secret and persistence mode, and + # the output checked against the Kubernetes API schemas. Both tools come + # from their release pages, pinned by version and checksum. + deploy-manifests: + name: Helm Chart & Compose + needs: changes + if: needs.changes.outputs.deploy == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + HELM_VERSION: v3.22.0 + HELM_SHA256: 1e4ab49e429626cf6c6958d914248b78c9730803c2751b87627e171dc800e7bb + KUBECONFORM_VERSION: v0.8.0 + KUBECONFORM_SHA256: 9bc2bffbf71f261128533edaf912153948b7ff238f9a531ae6d34466ec287883 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Install helm and kubeconform + run: | + set -euo pipefail + bin="$RUNNER_TEMP/bin" + mkdir -p "$bin" + cd "$RUNNER_TEMP" + curl -sSfLo helm.tgz "https://get.helm.sh/helm-$HELM_VERSION-linux-amd64.tar.gz" + echo "$HELM_SHA256 helm.tgz" | sha256sum -c - + tar -xzf helm.tgz --strip-components=1 -C "$bin" linux-amd64/helm + curl -sSfLo kubeconform.tgz "https://github.com/yannh/kubeconform/releases/download/$KUBECONFORM_VERSION/kubeconform-linux-amd64.tar.gz" + echo "$KUBECONFORM_SHA256 kubeconform.tgz" | sha256sum -c - + tar -xzf kubeconform.tgz -C "$bin" kubeconform + echo "$bin" >> "$GITHUB_PATH" + + - name: Lint the chart + run: helm lint --strict charts/trek + + - name: Render the chart + run: | + set -euo pipefail + out="$RUNNER_TEMP/rendered" + mkdir -p "$out" + helm template trek charts/trek > "$out/defaults.yaml" + helm template trek charts/trek --set existingSecret=trek-secrets > "$out/existing-secret.yaml" + helm template trek charts/trek --set generateEncryptionKey=true > "$out/generated-key.yaml" + helm template trek charts/trek \ + --set persistence.enabled=false \ + --set ingress.enabled=true \ + --set 'ingress.tls[0].secretName=trek-tls' \ + --set 'ingress.tls[0].hosts[0]=chart-example.local' \ + --set 'extraEnv[0].name=SESSION_DURATION' \ + --set 'extraEnv[0].value=7d' \ + --set 'extraEnvFrom[0].secretRef.name=trek-smtp' \ + --set probes.liveness=null \ + > "$out/variants.yaml" + + - name: Check the manifests against the Kubernetes schemas + run: kubeconform -strict -summary "$RUNNER_TEMP"/rendered/*.yaml + + - name: Check every compose file + run: | + set -euo pipefail + shopt -s nullglob + files=(docker-compose*.yml docker-compose*.yaml) + [ "${#files[@]}" -gt 0 ] || { echo "::error::No compose file found."; exit 1; } + for f in "${files[@]}"; do + echo "$f" + docker compose -f "$f" config --quiet + done + + # The single check the branch ruleset requires. It needs every job that + # gates a merge, so listing a job here is what makes it required. The scan + # stays out (see the layout notes at the top). + ci-ok: + needs: + - changes + - plugin-facts + - i18n-parity + - server-quality + - client-quality + - plugin-sdk-quality + - client-bundle + - client-e2e + - s3-contract + - server-coverage + - client-coverage + - shared-coverage + - plugin-sdk-coverage + - docker-image + - deploy-manifests + # always(): without it a failed dependency would skip this job, and the + # ruleset reads a skipped required check as passed. + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Every needed job passed or was skipped + env: + NEEDS: ${{ toJSON(needs) }} + run: | + set -euo pipefail + echo "$NEEDS" | jq -r 'to_entries[] | "\(.value.result)\t\(.key)"' | sort + failed="$(echo "$NEEDS" | jq -r '[to_entries[] | select(.value.result != "success" and .value.result != "skipped") | .key] | join(", ")')" + if [ -n "$failed" ]; then + echo "::error::Not passed: $failed" + exit 1 + fi diff --git a/CLAUDE.md b/CLAUDE.md index 380fbcaa1..15b09232e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -110,7 +110,10 @@ These principles come out of a full-repo audit and shape all new code: ## Quality gates in CI -`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`), `lint:check`, `lint:pages`, `lint:rtl`, `lint:size`, `check:plugin-facts`, `i18n:parity:strict`, the S3 contract suite, the browser end-to-end flows (Chromium and WebKit), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. `lint-prettier.yml` additionally runs `lint` + `format:check` on `shared/`. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: `theme:lint` and the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags. +`.github/workflows/test.yml` runs, per package: typecheck (server also `typecheck:tests`), `lint:check`, `lint:pages`, `lint:rtl`, `lint:size`, `check:plugin-facts`, `i18n:parity:strict`, the S3 contract suite, the browser end-to-end flows (Chromium and WebKit), a client build with `check:gl-split`, coverage for all four packages, then a SonarCloud scan. It also builds the Docker image and boots it until `/api/health/ready` answers (`docker-image`), and lints and renders the Helm chart against the Kubernetes schemas and runs `docker compose config` on every compose file (`deploy-manifests`). `lint-prettier.yml` additionally runs `lint` + `format:check` on `shared/`. `plugin-sdk` has no lint script and is outside the eslint gates. **Not in CI**: `theme:lint` and the screenshot and help-media Playwright projects; run them locally. There are no git hooks; nothing runs before a commit except you. Other workflows build Docker images, scan them, and publish `trek-plugin-sdk` on `plugin-sdk-v*` tags. + +- **`ci-ok` is the required check.** The branch ruleset requires `ci-ok` (plus `lint` and `check-target` from their own workflows), and `ci-ok` needs every job above except the scan. It fails if any of them failed or was cancelled and passes if they passed or were skipped. A new gate job counts only once it is listed in `ci-ok`'s `needs:`. +- **What runs is decided by `changes`, not by `on.paths`**: `scripts/ci/changed-areas.mjs` maps the changed files to the `code`, `image` and `deploy` areas (tests: `npm run test:ci-scripts`). A path filter on the workflow would leave `ci-ok` unreported on a docs-only PR and block it. Add a new top-level file the build reads to the right area there. **SonarCloud** (project `liketrek_TREK`, the built-in Sonar way gate; the run takes 20–25 min, so get it right before pushing) measures **new code only** on a PR; any failing condition blocks it: diff --git a/package.json b/package.json index 1e76d8c5e..a63abbb75 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test": "npm run test --workspace=shared && npm run test --workspace=server && npm run test --workspace=client", "test:cov": "npm run test:coverage --workspace=shared && npm run test:coverage --workspace=server && npm run test:coverage --workspace=client && npm run test:coverage --prefix plugin-sdk", "test:e2e": "npm run test:e2e --workspace=server", + "test:ci-scripts": "node --test scripts/ci/*.test.mjs", "lint": "npm run lint --workspace=shared && npm run lint --workspace=server && npm run lint --workspace=client", "format": "npm run format --workspace=shared && npm run format --workspace=server && npm run format --workspace=client", "format:check": "npm run format:check --workspace=shared && npm run format:check --workspace=server && npm run format:check --workspace=client" diff --git a/scripts/ci/changed-areas.mjs b/scripts/ci/changed-areas.mjs new file mode 100644 index 000000000..424f914a1 --- /dev/null +++ b/scripts/ci/changed-areas.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +// Decides which parts of the test workflow a change needs. +// +// test.yml used to filter on `paths:` at the workflow level. A required check +// cannot live behind such a filter: when the filter skips the whole workflow, +// the check never reports and the pull request waits for it forever. So the +// workflow always runs, its first job lists the changed files and pipes them +// through this script, and every other job asks the answer whether to run. +// The aggregate `ci-ok` job then treats a skipped job as passed. +// +// Usage: +// node scripts/ci/changed-areas.mjs --files one changed path per line +// node scripts/ci/changed-areas.mjs --all no usable list, run everything +// +// Prints `=true|false` per area and appends the same lines to +// $GITHUB_OUTPUT when it is set. +// +// It fails closed. A list it cannot read is an error, and anything it cannot +// classify with confidence (an empty list, a path git had to quote) turns every +// area on: running a job too often costs minutes, skipping one it needed lets a +// break through. + +import { appendFileSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// The workflow file and this script decide what runs, so a change to either +// runs everything. +const ALWAYS = ['.github/workflows/test.yml', 'scripts/ci/']; + +export const AREAS = { + // The type, lint, test, coverage, bundle and end-to-end jobs. npm workspaces + // hoist every dependency into the root lockfile, so a dependency bump touches + // none of the package directories. The Dockerfile and the entrypoint script + // are pinned by server/tests/unit/docker-entrypoint.test.ts. + code: [ + 'server/', + 'client/', + 'shared/', + 'plugin-sdk/', + 'package.json', + 'package-lock.json', + '.nvmrc', + 'sonar-project.properties', + 'Dockerfile', + ...ALWAYS, + ], + // The image build and its boot check: everything the Dockerfile copies or + // installs from. + image: [ + 'server/', + 'client/', + 'shared/', + 'package.json', + 'package-lock.json', + '.nvmrc', + 'Dockerfile', + '.dockerignore', + ...ALWAYS, + ], + // The Helm chart and the compose files. + deploy: ['charts/', /^docker-compose[^/]*\.ya?ml$/, ...ALWAYS], +}; + +// Markdown inside the packages (CLAUDE.md, READMEs, PATTERN.md) changes no +// test, no build and no scan. +const isMarkdown = (path) => path.endsWith('.md'); + +function matches(rule, path) { + if (rule instanceof RegExp) return rule.test(path); + return rule.endsWith('/') ? path.startsWith(rule) : path === rule; +} + +/** + * @param {string[] | null} files changed paths, or null when there is no usable list + * @returns {Record} + */ +export function classify(files) { + const all = Object.fromEntries(Object.keys(AREAS).map((area) => [area, true])); + if (files === null) return all; + const paths = files.map((line) => line.replace(/\r$/, '').trim()).filter(Boolean); + if (paths.length === 0) return all; + // core.quotePath=false keeps ordinary non-ASCII names unquoted. A path that + // still arrives quoted carries control characters, and matching on its + // escaped form could miss it. + if (paths.some((path) => path.startsWith('"'))) return all; + + const result = {}; + for (const [area, rules] of Object.entries(AREAS)) { + result[area] = paths.some((path) => !isMarkdown(path) && rules.some((rule) => matches(rule, path))); + } + return result; +} + +function parseArgs(argv) { + if (argv.length === 1 && argv[0] === '--all') return { all: true }; + if (argv.length === 2 && argv[0] === '--files' && argv[1]) return { all: false, list: argv[1] }; + throw new Error('usage: changed-areas.mjs --files | --all'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + // An unreadable list throws and fails the job instead of running on a guess. + const files = args.all ? null : readFileSync(args.list, 'utf8').split('\n'); + const result = classify(files); + const lines = Object.entries(result).map(([area, on]) => `${area}=${on}`); + console.log(lines.join('\n')); + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + main(); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + } +} diff --git a/scripts/ci/changed-areas.test.mjs b/scripts/ci/changed-areas.test.mjs new file mode 100644 index 000000000..a3110ce41 --- /dev/null +++ b/scripts/ci/changed-areas.test.mjs @@ -0,0 +1,100 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { after, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { AREAS, classify } from './changed-areas.mjs'; + +const script = fileURLToPath(new URL('./changed-areas.mjs', import.meta.url)); +const scratch = mkdtempSync(join(tmpdir(), 'changed-areas-')); +after(() => rmSync(scratch, { recursive: true, force: true })); + +const none = Object.fromEntries(Object.keys(AREAS).map((area) => [area, false])); +const every = Object.fromEntries(Object.keys(AREAS).map((area) => [area, true])); + +function run(args, env = {}) { + return spawnSync(process.execPath, [script, ...args], { + encoding: 'utf8', + env: { ...process.env, GITHUB_OUTPUT: '', ...env }, + }); +} + +describe('classify', () => { + it('runs nothing for a docs-only change', () => { + assert.deepEqual(classify(['wiki/Home.md', 'README.md', 'server/CLAUDE.md', 'docs/logo-trek-dark.svg']), none); + }); + + it('runs the tests and the image for a server change', () => { + assert.deepEqual(classify(['server/src/nest/weather/weather.service.ts']), { code: true, image: true, deploy: false }); + }); + + it('runs the tests but not the image for a plugin-sdk change', () => { + assert.deepEqual(classify(['plugin-sdk/src/index.ts']), { code: true, image: false, deploy: false }); + }); + + it('runs the image but not the tests for a .dockerignore change', () => { + assert.deepEqual(classify(['.dockerignore']), { code: false, image: true, deploy: false }); + }); + + it('runs the chart and compose checks for a chart template or a compose file', () => { + assert.deepEqual(classify(['charts/trek/templates/deployment.yaml']), { code: false, image: false, deploy: true }); + assert.deepEqual(classify(['docker-compose.minio-test.yml']), { code: false, image: false, deploy: true }); + assert.equal(classify(['charts/README.md']).deploy, false); + assert.equal(classify(['server/docker-compose.yml']).deploy, false); + }); + + it('runs everything when the workflow or the classifier itself changes', () => { + assert.deepEqual(classify(['.github/workflows/test.yml']), every); + assert.deepEqual(classify(['scripts/ci/changed-areas.mjs']), every); + }); + + it('matches whole file names, not prefixes of them', () => { + assert.deepEqual(classify(['package.json.bak', 'Dockerfile.old', 'serverless/x.ts']), none); + }); + + it('runs everything when there is no list, an empty one or a quoted path', () => { + assert.deepEqual(classify(null), every); + assert.deepEqual(classify(['', ' ']), every); + assert.deepEqual(classify(['"wiki/\\303\\244.md"']), every); + }); + + it('reads CRLF lists the same as LF ones', () => { + assert.deepEqual(classify(['wiki/Home.md\r', 'server/src/index.ts\r']), { code: true, image: true, deploy: false }); + }); +}); + +describe('the command line', () => { + it('writes the areas to stdout and to GITHUB_OUTPUT', () => { + const list = join(scratch, 'list.txt'); + const output = join(scratch, 'output.txt'); + writeFileSync(list, 'charts/trek/values.yaml\n'); + writeFileSync(output, ''); + const result = run(['--files', list], { GITHUB_OUTPUT: output }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, 'code=false\nimage=false\ndeploy=true\n'); + assert.equal(readFileSync(output, 'utf8'), 'code=false\nimage=false\ndeploy=true\n'); + }); + + it('turns everything on with --all', () => { + const result = run(['--all']); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, 'code=true\nimage=true\ndeploy=true\n'); + }); + + it('fails on a list it cannot read', () => { + const result = run(['--files', join(scratch, 'missing.txt')]); + assert.equal(result.status, 1); + assert.match(result.stderr, /ENOENT/); + assert.equal(result.stdout, ''); + }); + + it('fails on arguments it does not know', () => { + for (const args of [[], ['--files'], ['--all', '--files', 'x'], ['list.txt']]) { + const result = run(args); + assert.equal(result.status, 1, `args ${JSON.stringify(args)}`); + assert.match(result.stderr, /usage/); + } + }); +});