From 04b1fb72bb70283e1b2c6f3cbb4aeeab71d78258 Mon Sep 17 00:00:00 2001 From: John Jahn Date: Tue, 10 Nov 2026 15:03:28 -0500 Subject: [PATCH] perf(static): serve the hashed assets from Brotli copies written at build time Every file under assets/ was compressed again on each download, at the low level a request path can afford. They carry their content hash in the name and never change, so the client build now writes a Brotli copy beside each compressible one (postbuild, scripts/precompress.mjs) and production points a browser that accepts Brotli at that copy before express.static serves it. Measured on a production build of 5.1.2, fetching the 652 precached files the way a service worker install does: 16.3 MB -> 13.9 MB on the wire and about 1.0 s -> 0.2 s of server CPU per install. A client without Brotli, a range request and a build without copies are served exactly as before. --- client/CLAUDE.md | 2 +- client/package.json | 1 + client/scripts/lib/precompress.mjs | 60 ++++++++++ client/scripts/precompress.mjs | 12 ++ client/tests/unit/scripts/precompress.test.ts | 61 ++++++++++ server/src/nest/platform/platform.routes.ts | 6 +- .../src/nest/platform/precompressed-static.ts | 58 +++++++++ .../unit/nest/platform.controller.test.ts | 3 + .../unit/nest/precompressed-static.test.ts | 113 ++++++++++++++++++ 9 files changed, 314 insertions(+), 2 deletions(-) create mode 100644 client/scripts/lib/precompress.mjs create mode 100644 client/scripts/precompress.mjs create mode 100644 client/tests/unit/scripts/precompress.test.ts create mode 100644 server/src/nest/platform/precompressed-static.ts create mode 100644 server/tests/unit/nest/precompressed-static.test.ts diff --git a/client/CLAUDE.md b/client/CLAUDE.md index fc1284c65..f76ebe5bc 100644 --- a/client/CLAUDE.md +++ b/client/CLAUDE.md @@ -6,7 +6,7 @@ Scope: the **`@trek/client`** workspace (React 19 + Vite + Zustand + Tailwind PW ```bash npm run dev # Vite dev server; proxies the API/ws/uploads/MCP/OAuth paths → http://localhost:3001 -npm run build # prebuild generates PWA icons, then vite build +npm run build # prebuild generates PWA icons, then vite build; postbuild writes a Brotli copy beside each hashed asset for the server to hand out npm run typecheck # tsc --noEmit (CI) npm run lint:strict # tsc over tsconfig.strict.json (strictNullChecks + noImplicitAny): errors per file may only shrink against scripts/strict-baseline.json, a new file starts at zero (CI gate; --list shows them, --update lowers it) npm run lint # eslint . (CI runs lint:warnings instead, which fails on the same errors) diff --git a/client/package.json b/client/package.json index 63dac4f40..617220adf 100644 --- a/client/package.json +++ b/client/package.json @@ -18,6 +18,7 @@ "store:graphic": "node scripts/generate-feature-graphic.mjs", "store:check": "node scripts/check-store-metadata.mjs", "build": "vite build", + "postbuild": "node scripts/precompress.mjs", "build:analyze": "vite build --mode analyze", "preview": "vite preview", "typecheck": "tsc --noEmit", diff --git a/client/scripts/lib/precompress.mjs b/client/scripts/lib/precompress.mjs new file mode 100644 index 000000000..10725daa0 --- /dev/null +++ b/client/scripts/lib/precompress.mjs @@ -0,0 +1,60 @@ +/* + * The work behind scripts/precompress.mjs: a Brotli copy (.br) beside every + * compressible build file under dist/assets. + * + * The server compresses a build file on every download otherwise, at the low level + * a request path can afford. A file under assets/ carries its content hash in its + * name and never changes, so it is compressed once here, at the level a build can + * afford, and the server hands that copy out as it is + * (server/src/nest/platform/precompressed-static.ts). + */ +import { readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { brotliCompressSync, constants } from 'node:zlib'; + +/** What the server's own compression would take; fonts, images and video are compressed formats already. */ +const COMPRESSIBLE = /\.(?:m?js|css|svg|json|wasm|html|webmanifest)$/i; +/** The server leaves anything below 1 kB uncompressed too: the headers cost more than the saving. */ +export const MIN_BYTES = 1024; +/** Quality 11 takes about ten times as long per byte as 9, so the largest chunks get 9. */ +const BIG_BYTES = 1024 * 1024; + +function* walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) yield* walk(full); + else yield full; + } +} + +/** + * Writes the .br copies under `dir` and returns what it did. A copy that would not be + * smaller than its file is not written, and a directory that does not exist is no work. + */ +export function precompress(dir) { + const result = { files: 0, before: 0, after: 0 }; + let files; + try { + files = [...walk(dir)]; + } catch (err) { + if (err.code === 'ENOENT') return result; + throw err; + } + for (const file of files) { + if (!COMPRESSIBLE.test(file)) continue; + const input = readFileSync(file); + if (input.length < MIN_BYTES) continue; + const output = brotliCompressSync(input, { + params: { + [constants.BROTLI_PARAM_QUALITY]: input.length > BIG_BYTES ? 9 : 11, + [constants.BROTLI_PARAM_SIZE_HINT]: input.length, + }, + }); + if (output.length >= input.length) continue; + writeFileSync(`${file}.br`, output); + result.files += 1; + result.before += input.length; + result.after += output.length; + } + return result; +} diff --git a/client/scripts/precompress.mjs b/client/scripts/precompress.mjs new file mode 100644 index 000000000..9ffe084de --- /dev/null +++ b/client/scripts/precompress.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +/* + * postbuild: writes a Brotli copy beside every compressible file under dist/assets, + * which the server hands to a browser that accepts Brotli instead of compressing + * the file again on each download. scripts/lib/precompress.mjs holds the work. + */ +import { fileURLToPath } from 'node:url'; +import { precompress } from './lib/precompress.mjs'; + +const { files, before, after } = precompress(fileURLToPath(new URL('../dist/assets', import.meta.url))); +const mb = (bytes) => (bytes / 1048576).toFixed(1); +console.log(`precompress: ${files} files, ${mb(before)} MB -> ${mb(after)} MB (brotli)`); diff --git a/client/tests/unit/scripts/precompress.test.ts b/client/tests/unit/scripts/precompress.test.ts new file mode 100644 index 000000000..f70378498 --- /dev/null +++ b/client/tests/unit/scripts/precompress.test.ts @@ -0,0 +1,61 @@ +import { randomBytes } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { brotliDecompressSync } from 'node:zlib'; +import { afterEach, describe, expect, it } from 'vitest'; +import { MIN_BYTES, precompress } from '../../../scripts/lib/precompress.mjs'; + +let dir: string; +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +function tree(files: Record) { + dir = mkdtempSync(join(tmpdir(), 'trek-precompress-')); + for (const [name, content] of Object.entries(files)) { + mkdirSync(join(dir, name, '..'), { recursive: true }); + writeFileSync(join(dir, name), content); + } + return dir; +} + +const script = 'export const answer = 42;\n'.repeat(200); + +describe('precompress', () => { + it('writes a Brotli copy that unpacks to the file, and counts what it saved', () => { + const result = precompress(tree({ 'index-AbCd1234.js': script, 'nested/Map-AbCd1234.css': script })); + + expect(brotliDecompressSync(readFileSync(join(dir, 'index-AbCd1234.js.br'))).toString()).toBe(script); + expect(existsSync(join(dir, 'nested/Map-AbCd1234.css.br'))).toBe(true); + expect(result.files).toBe(2); + expect(result.before).toBe(script.length * 2); + expect(result.after).toBeLessThan(result.before); + }); + + it('leaves compressed formats and files under the size floor alone', () => { + const result = precompress( + tree({ + 'font-AbCd1234.woff2': script, + 'clip-AbCd1234.mp4': script, + 'tiny-AbCd1234.js': 'x'.repeat(MIN_BYTES - 1), + }) + ); + + expect(result.files).toBe(0); + expect(existsSync(join(dir, 'font-AbCd1234.woff2.br'))).toBe(false); + expect(existsSync(join(dir, 'clip-AbCd1234.mp4.br'))).toBe(false); + expect(existsSync(join(dir, 'tiny-AbCd1234.js.br'))).toBe(false); + }); + + it('writes no copy that would not be smaller than its file', () => { + // Random bytes have nothing for a compressor to find. + const result = precompress(tree({ 'noise-AbCd1234.json': randomBytes(4096) })); + + expect(result.files).toBe(0); + expect(existsSync(join(dir, 'noise-AbCd1234.json.br'))).toBe(false); + }); + + it('does nothing for a build directory that is not there', () => { + dir = join(tmpdir(), 'trek-precompress-missing'); + expect(precompress(dir)).toEqual({ files: 0, before: 0, after: 0 }); + }); +}); diff --git a/server/src/nest/platform/platform.routes.ts b/server/src/nest/platform/platform.routes.ts index c6b0b905a..834f8c7a6 100644 --- a/server/src/nest/platform/platform.routes.ts +++ b/server/src/nest/platform/platform.routes.ts @@ -11,6 +11,7 @@ import { verifyJwtAndLoadUser } from '../auth-core/jwt-verify'; import { withRequestContext } from '../database/request-context'; import { StorageService } from '../storage/storage.service'; import { StorageInvalidKeyError, StorageNotFoundError, type StorageCategory } from '../storage/storage.types'; +import { precompressedStatic } from './precompressed-static'; import type { EntityManager } from '@mikro-orm/core'; import express, { Request, Response, NextFunction } from 'express'; @@ -290,7 +291,9 @@ export function applyPlatformSpa(app: express.Application): void { app.get(/.*/, (req: Request, res: Response) => answerUnmatchedGet(req, res)); } -const HASHED_ASSET = /[\\/]assets[\\/][^\\/]+-[\w-]{8,}\.\w+$/; +// The optional .br is the Brotli copy the client build writes beside a hashed file +// (precompressed-static.ts); it is as immutable as the file it was made from. +const HASHED_ASSET = /[\\/]assets[\\/][^\\/]+-[\w-]{8,}\.\w+(?:\.br)?$/; /** * Production static serving of the built client (JS/CSS/assets). Split out from @@ -302,6 +305,7 @@ const HASHED_ASSET = /[\\/]assets[\\/][^\\/]+-[\w-]{8,}\.\w+$/; export function applyPlatformStatic(app: express.Application): void { // Case-sensitive on purpose (legacy parity). if (readEnv().app.nodeEnv !== 'production') return; + app.use(precompressedStatic(path.join(PUBLIC_DIR, 'assets'))); app.use( express.static(PUBLIC_DIR, { setHeaders: (res, filePath) => { diff --git a/server/src/nest/platform/precompressed-static.ts b/server/src/nest/platform/precompressed-static.ts new file mode 100644 index 000000000..50e36682c --- /dev/null +++ b/server/src/nest/platform/precompressed-static.ts @@ -0,0 +1,58 @@ +import express, { type NextFunction, type Request, type Response } from 'express'; +import { readdirSync } from 'node:fs'; +import path from 'node:path'; + +const BROTLI_SUFFIX = '.br'; + +/** Every URL under `urlPrefix` whose file has a Brotli copy beside it. A missing directory holds none. */ +function collectPrecompressed(dir: string, urlPrefix: string, into: Set): Set { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return into; + } + for (const entry of entries) { + if (entry.isDirectory()) collectPrecompressed(path.join(dir, entry.name), `${urlPrefix}${entry.name}/`, into); + else if (entry.name.endsWith(BROTLI_SUFFIX)) into.add(`${urlPrefix}${entry.name.slice(0, -BROTLI_SUFFIX.length)}`); + } + return into; +} + +/** The Content-Type the static handler gives the original file, charset included, from the same table. */ +function contentTypeOf(file: string): string { + const { mime } = express.static; + const type = mime.lookup(file); + const charset = mime.charsets.lookup(type, ''); + return charset ? `${type}; charset=${charset}` : type; +} + +/** + * Hands a browser that accepts Brotli the copy the client build wrote beside a hashed + * asset (`.br`, client/scripts/precompress.mjs) instead of compressing the file + * again on every download. + * + * It only points the request at the copy and sets the two headers the static handler + * behind it cannot know (the type of the original file, the encoding); that handler + * still sends the file, with its own caching headers, ETag and conditional requests. + * The compression middleware leaves a response that already names its encoding alone. + * + * The copies are listed from disk once, at startup, so only a file the build produced + * is ever rewritten, whatever the request path looks like, and a build without copies + * (a source checkout that skipped the postbuild step) is served as before. A range + * request gets the uncompressed file: its offsets mean the original bytes. + */ +export function precompressedStatic(assetsDir: string, urlPrefix = '/assets/') { + const precompressed = collectPrecompressed(assetsDir, urlPrefix, new Set()); + return (req: Request, res: Response, next: NextFunction): void => { + if ((req.method !== 'GET' && req.method !== 'HEAD') || !precompressed.has(req.path)) return next(); + // Said for every answer under this URL, so a cache never hands the Brotli bytes + // to a client that did not ask for them. + res.vary('Accept-Encoding'); + if (req.headers.range || !req.acceptsEncodings('br')) return next(); + res.setHeader('Content-Type', contentTypeOf(req.path)); + res.setHeader('Content-Encoding', 'br'); + req.url = `${req.path}${BROTLI_SUFFIX}${req.url.slice(req.path.length)}`; + next(); + }; +} diff --git a/server/tests/unit/nest/platform.controller.test.ts b/server/tests/unit/nest/platform.controller.test.ts index 7d0df8377..4d0abc558 100644 --- a/server/tests/unit/nest/platform.controller.test.ts +++ b/server/tests/unit/nest/platform.controller.test.ts @@ -552,6 +552,9 @@ describe('applyPlatformStatic', () => { expect(header('/app/public/sw.js')).toBeUndefined(); expect(header('/app/public/icons/icon-192.png')).toBeUndefined(); expect(header('/app/public/assets/plain.js')).toBeUndefined(); + // The Brotli copy of a hashed file (precompressed-static.ts) never changes either. + expect(header('/app/public/assets/index-DbM4_q2x.js.br')).toBe('public, max-age=31536000, immutable'); + expect(header('/app/public/sw.js.br')).toBeUndefined(); }); }); diff --git a/server/tests/unit/nest/precompressed-static.test.ts b/server/tests/unit/nest/precompressed-static.test.ts new file mode 100644 index 000000000..61a09ddd1 --- /dev/null +++ b/server/tests/unit/nest/precompressed-static.test.ts @@ -0,0 +1,113 @@ +import { precompressedStatic } from '../../../src/nest/platform/precompressed-static'; + +import express from 'express'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { brotliCompressSync } from 'node:zlib'; +import request from 'supertest'; +import { afterAll, describe, expect, it } from 'vitest'; + +const SCRIPT = 'export const answer = 42;\n'.repeat(200); +const STYLE = '.map { inset: 0 }\n'.repeat(200); + +describe('precompressedStatic', () => { + const root = mkdtempSync(path.join(tmpdir(), 'trek-public-')); + const assets = path.join(root, 'assets'); + mkdirSync(path.join(assets, 'nested'), { recursive: true }); + writeFileSync(path.join(assets, 'index-AbCd1234.js'), SCRIPT); + writeFileSync(path.join(assets, 'index-AbCd1234.js.br'), brotliCompressSync(SCRIPT)); + writeFileSync(path.join(assets, 'nested', 'Map-AbCd1234.css'), STYLE); + writeFileSync(path.join(assets, 'nested', 'Map-AbCd1234.css.br'), brotliCompressSync(STYLE)); + writeFileSync(path.join(assets, 'plain-AbCd1234.js'), SCRIPT); + writeFileSync(path.join(root, 'secret.js'), SCRIPT); + writeFileSync(path.join(root, 'secret.js.br'), brotliCompressSync(SCRIPT)); + afterAll(() => rmSync(root, { recursive: true, force: true })); + + const app = express(); + app.use(precompressedStatic(assets)); + app.use(express.static(root)); + app.use((_req, res) => res.status(404).json({ error: 'Not found' })); + + const get = (url: string, acceptEncoding: string) => request(app).get(url).set('Accept-Encoding', acceptEncoding); + + it('PRECOMP-001: hands the Brotli copy to a browser that accepts br, typed as the original file', async () => { + const res = await get('/assets/index-AbCd1234.js', 'gzip, deflate, br'); + + expect(res.status).toBe(200); + expect(res.headers['content-encoding']).toBe('br'); + // The very header the static handler sends for the file itself. + const original = await get('/assets/index-AbCd1234.js', 'identity'); + expect(res.headers['content-type']).toBe(original.headers['content-type']); + expect(res.headers['content-type']).toMatch(/javascript; charset=utf-8$/i); + expect(res.headers['vary']).toContain('Accept-Encoding'); + expect(Number(res.headers['content-length'])).toBe(brotliCompressSync(SCRIPT).length); + // supertest unpacks the body by its Content-Encoding, as a browser does. + expect(res.text).toBe(SCRIPT); + }); + + it('PRECOMP-002: finds copies in nested folders and keeps the query string', async () => { + const res = await get('/assets/nested/Map-AbCd1234.css?v=1', 'br'); + + expect(res.headers['content-encoding']).toBe('br'); + expect(res.headers['content-type']).toMatch(/^text\/css/); + expect(res.text).toBe(STYLE); + }); + + it.each(['gzip, deflate', 'identity', 'gzip, br;q=0'])( + 'PRECOMP-003: a client sending "%s" gets the file itself, still marked as varying', + async (accept) => { + const res = await get('/assets/index-AbCd1234.js', accept); + + expect(res.status).toBe(200); + expect(res.headers['content-encoding']).toBeUndefined(); + expect(res.headers['vary']).toContain('Accept-Encoding'); + expect(res.text).toBe(SCRIPT); + }, + ); + + it('PRECOMP-004: a range request is answered from the uncompressed file', async () => { + const res = await get('/assets/index-AbCd1234.js', 'br').set('Range', 'bytes=0-9'); + + expect(res.status).toBe(206); + expect(res.headers['content-encoding']).toBeUndefined(); + expect(res.text).toBe(SCRIPT.slice(0, 10)); + }); + + it('PRECOMP-005: answers HEAD like GET, and a conditional request with 304', async () => { + const head = await request(app).head('/assets/index-AbCd1234.js').set('Accept-Encoding', 'br'); + expect(head.status).toBe(200); + expect(head.headers['content-encoding']).toBe('br'); + + const again = await get('/assets/index-AbCd1234.js', 'br').set('If-None-Match', head.headers['etag']); + expect(again.status).toBe(304); + }); + + it('PRECOMP-006: leaves a file without a copy, another method and a path outside assets/ alone', async () => { + const plain = await get('/assets/plain-AbCd1234.js', 'br'); + expect(plain.headers['content-encoding']).toBeUndefined(); + expect(plain.headers['vary']).toBeUndefined(); + expect(plain.text).toBe(SCRIPT); + + const post = await request(app).post('/assets/index-AbCd1234.js').set('Accept-Encoding', 'br'); + expect(post.status).toBe(404); + expect(post.headers['content-encoding']).toBeUndefined(); + + // A copy exists on disk, but outside the directory the middleware was given. + const outside = await get('/secret.js', 'br'); + expect(outside.headers['content-encoding']).toBeUndefined(); + const climbing = await get('/assets/%2e%2e/secret.js', 'br'); + expect(climbing.headers['content-encoding']).toBeUndefined(); + }); + + it('PRECOMP-007: a build without copies is served as before', async () => { + const bare = express(); + bare.use(precompressedStatic(path.join(root, 'missing'))); + bare.use(express.static(root)); + + const res = await request(bare).get('/assets/index-AbCd1234.js').set('Accept-Encoding', 'br'); + expect(res.status).toBe(200); + expect(res.headers['content-encoding']).toBeUndefined(); + expect(res.text).toBe(SCRIPT); + }); +});