diff --git a/.dockerignore b/.dockerignore index 25672b03f..72a0b6b6a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,8 @@ uploads chart/ docs/ docker-compose.yml +docker-compose.minio-test.yml +minio-licence.key unraid-template.xml *.sqlite *.sqlite-shm diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6785e180a..61b7539a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -175,6 +175,75 @@ jobs: path: server/coverage/ retention-days: 7 + s3-contract: + name: S3 Driver Contract (MinIO) + runs-on: ubuntu-latest + # Fork PRs don't get repo secrets, so MINIO_LICENSE_KEY would be empty + # and the AIStor container would fail to boot. + if: github.event.pull_request.head.repo.full_name == github.repository + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Ensure @swc/core's Linux binary for unplugin-swc + # The lockfile was generated on Windows and omits @swc/core's Linux + # optional native binary, so npm ci/install skips it on the runner. + # Install the matching version explicitly so the server's SWC transform + # (server/vitest.config.ts) can load. + run: | + SWC_VERSION=$(node -p "require('@swc/core/package.json').version") + npm install --no-save --legacy-peer-deps "@swc/core-linux-x64-gnu@$SWC_VERSION" + + - name: Build shared + run: npm run build --workspace=shared + + - name: Write MinIO license + run: printf '%s' "${{ secrets.MINIO_LICENSE_KEY }}" > minio-licence.key + + - name: Start MinIO + # `services:` can't pass the `server /data --license ...` command to + # the AIStor image, so it starts via docker run. 127.0.0.1 endpoint is + # load-bearing: aws-lite picks path-style only for IP/localhost hosts, + # which is all stock MinIO/AIStor speaks. + run: | + docker run -d --name minio -p 127.0.0.1:9000:9000 \ + -v "$PWD/minio-licence.key:/minio.license:ro" \ + -e MINIO_ROOT_USER=trekci -e MINIO_ROOT_PASSWORD=trekci-secret \ + quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z server /data --license /minio.license + + - name: Wait for MinIO + # curl was removed from the AIStor image — poll from the runner instead. + # 60 x 2s gives a wider margin on cold runners than the 30 x 1s the + # pre-6fbff835 job used. + run: | + for i in $(seq 1 60); do + curl -sf http://127.0.0.1:9000/minio/health/live && exit 0 + sleep 2 + done + echo "MinIO never became healthy" >&2 + docker logs minio >&2 + exit 1 + + - name: Run the S3 driver contract suite + # No --coverage: the src/nest/storage/** 94/84/97/94 pin must keep + # holding from hermetic tests alone. The suite's beforeAll creates + # the trek-test bucket idempotently — no separate bucket-init step. + env: + TREK_TEST_S3_ENDPOINT: http://127.0.0.1:9000 + TREK_TEST_S3_BUCKET: trek-test + TREK_TEST_S3_ACCESS_KEY_ID: trekci + TREK_TEST_S3_SECRET_ACCESS_KEY: trekci-secret + run: cd server && npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts + client-quality: # Split out of client-tests: the suite takes ~11 minutes, and having the # gates in front of it meant a single lint finding threw away the whole test diff --git a/.gitignore b/.gitignore index fa3baa3aa..c3aaa38a2 100644 --- a/.gitignore +++ b/.gitignore @@ -65,5 +65,8 @@ coverage-*/ .scannerwork test-data +# Live-MinIO/AIStor manual test license key — never commit, operators obtain their own free key +minio-licence.key + .run .full-review diff --git a/README.md b/README.md index 043215299..35e54d882 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,9 @@ Most of what follows is an addon an admin switches on or off. Lists, Costs, Docu - **2FA**: TOTP with ten single-use backup codes, and an admin can require it instance-wide - **Passkeys**: WebAuthn login by fingerprint, face, PIN or security key, off until an admin enables it. A passkey also satisfies the 2FA requirement - **Hardening**: per-IP limits on login, password reset and 2FA attempts, a password policy, secrets encrypted at rest and masked on read, and an SSRF guard on every URL you configure -- **Admin panel**: users and invites, the permission matrix, packing templates, categories, addons, plugins, API keys, MCP tokens and OAuth sessions, backups, audit log, and GitHub releases +- **Admin panel**: users and invites, the permission matrix, packing templates, categories, addons, plugins, API keys, MCP tokens and OAuth sessions, backups, storage, audit log, and GitHub releases - **Backups**: manual or scheduled hourly, daily, weekly or monthly, with retention in days. The zip carries the at-rest key, so a restore can decrypt its own secrets +- **Storage**: pluggable storage backends per content category — keep everything on local disk, or add S3-compatible backends and replicate any category to them, configured entirely from the admin panel - **Notifications**: a per-user matrix of events against in-app, email (SMTP), webhook and ntfy, plus any channel a plugin registers - **Appearance**: light, dark or follow the OS, seven colour schemes plus a custom accent, transparency, compact density, reduce motion, and text size per tier - **23 languages**: en, de, es, fr, it, nl, hu, ru, zh, zh-TW, pl, cs, ar (right to left), br, id, tr, ja, ko, uk, gr, sv, vi, ca @@ -488,10 +489,113 @@ Caddy handles TLS and WebSockets automatically.
+

Storage

+ +TREK separates *what* it stores (eight content categories — trip documents, +journey photos, cover images, profile pictures, place images, the two photo +caches, and backups) from *where* it stores it (named backends). Out of the +box everything lives on local disk (`uploads/` and `data/backups`). From +**Admin → Storage** you can: + +- **Add S3-compatible backends** (AWS S3, Cloudflare R2, Backblaze B2, Garage, + MinIO/AIStor) and assign any category to them. +- **Replicate a backend**: edit it and pick **Mirror targets** — every write is + then also copied to each target. The classic off-site backup setup is: + add an S3 backend, edit `backups-local`, tick the S3 backend as a mirror + target, save. Replica writes happen one after another during each upload, so + a slow or unreachable target slows every upload of every category on that + backend — fine for backups, worth weighing for hot categories. Existing + objects are not replicated when you add a target — press **Sync now** on + the backend to copy them (one sync at a time; progress and cancel in the + panel). +- **Watch the Health strip**: replica failures never fail the original + request; they are recorded and shown there instead. All-clear means every + replicated write landed. Failures also notify admins over the configured + notification channels (first failure immediately, then hourly summaries) — + tune it under Admin → Notifications. +- **Test** any backend from its row (for a replicated backend it probes the + primary and each target individually). Targets must be saved before Test + can probe them. +- **Usage**: per-category and per-backend object counts and sizes, scanned + nightly and on demand. + +Backend credentials are encrypted at rest. Without an explicit +`ENCRYPTION_KEY` the implicit key from the data directory is used — that +works, but the implicit key rides inside backups, so setting `ENCRYPTION_KEY` +explicitly is recommended for credentialed backends. On an existing install +that has been running without an explicit key, do not just set a new one — +that would orphan already-stored secrets; use the key-rotation procedure (see +*Rotating the Encryption Key* under Updating) to move to an explicit key. + +Notes and limits: reassigning a populated category prompts to move its +existing objects — accepting copies everything to the new backend, flips the +category, then sweeps any writes that raced the copy; source objects are kept +(not deleted) and reported as reclaimable (count + size) to reclaim manually; +declining routes only new writes to the new backend, leaving old objects in +place as before. Only one storage job (a mirror sync or a category migration) +runs at a time, cancelling before the flip is always safe, and a failed copy +never flips the category; media categories on S3 work but every served byte +proxies through the server — the proxy answers conditional GETs (ETag / +Last-Modified) and a single `Range` request, while a multi-range request is +answered with the full object; AWS buckets +with Object-Lock or checksum-requiring policies +reject the server's uploads; self-hosted endpoints (MinIO/Garage) should be +addressed by IP or `localhost` unless configured for virtual-hosted buckets. +The legacy `/uploads/photos` directory from older TREK versions is still +served and included in backups, but it is not a configurable category. +Running the storage contract suite against a live MinIO/AIStor is a +license-gated procedure — CI runs it automatically for same-repo pull +requests (fork PRs skip it, since they have no access to the license +secret); to run it yourself locally, see `docker-compose.minio-test.yml`'s +header for the manual procedure. + +### Provisioning at first boot (seed file) + +For infrastructure-as-code setups, mount a `storage-config.json` into the +data directory — it is imported **once**, on the first boot that has no +stored storage configuration, and loudly ignored afterwards: + +```jsonc +// storage-config.json — secrets may be plaintext (encrypted on import) +// or already-encrypted enc:v1: values. +{ + "backends": [ + { "name": "off-site", "type": "s3", "options": { + "endpoint": "https://s3.example.com", "bucket": "trek", + "accessKeyId": "…", "secretAccessKey": "…" } }, + { "name": "backups-mirror", "type": "mirror", "options": { + "primary": "backups-local", "replicas": ["off-site"] } } + ], + "categories": { "backups": "backups-mirror" } +} +``` + +(The panel presents this exact setup as **Mirror targets** on `backups-local`.) + +```yaml +# docker-compose: add under the trek service's volumes + - ./storage-config.json:/app/data/storage-config.json:ro +``` + +An invalid seed file aborts boot with the exact validation error — an +actively-provisioning operator sees the problem instead of a silent default. + +### Recovery and restore + +To reset storage configuration to the built-in defaults (or to re-import a +seed file): stop the server, run +`sqlite3 data/travel.db "DELETE FROM app_settings WHERE key LIKE 'storage.%';"`, +and start it again. + +Backups include the storage configuration (it lives in the database). If your +only backup sits on S3 and the credentials for it sit inside that backup: +start a fresh instance, enter the S3 credentials in **Admin → Storage** (or +mount a seed file), then restore the backup from the Backup panel. + ## Data & Backups - **Database** — SQLite, stored in `./data/travel.db` -- **Uploads** — stored in `./uploads/` +- **Uploads** — stored in `./uploads/` by default; every content category can be reassigned to another backend in **Admin → Storage** - **Logs** — `./data/logs/trek.log` (auto-rotated) - **Backups** — create and restore via Admin Panel - **Auto-Backups** — configurable schedule and retention in Admin Panel diff --git a/client/src/api/client.surface.test.ts b/client/src/api/client.surface.test.ts index 5c0afe7f3..e3d76cb2b 100644 --- a/client/src/api/client.surface.test.ts +++ b/client/src/api/client.surface.test.ts @@ -267,6 +267,14 @@ describe('client > endpoint wiring', () => { { n: 'updateNotificationPreferences', r: () => adminApi.updateNotificationPreferences({ email: { trip_invite: true } }), e: 'PUT /api/admin/notification-preferences' }, { n: 'getDefaultUserSettings', r: () => adminApi.getDefaultUserSettings(), e: 'GET /api/admin/default-user-settings' }, { n: 'updateDefaultUserSettings', r: () => adminApi.updateDefaultUserSettings({ language: 'de' }), e: 'PUT /api/admin/default-user-settings' }, + { n: 'getStorage', r: () => adminApi.getStorage(), e: 'GET /api/admin/storage' }, + { n: 'updateStorage', r: () => adminApi.updateStorage({ backends: [], categories: {}, version: 0 }), e: 'PUT /api/admin/storage' }, + { n: 'testStorageBackend', r: () => adminApi.testStorageBackend({ name: 'x', type: 'local', options: { root: '/data' } }), e: 'POST /api/admin/storage/test' }, + { n: 'startStorageBackfill', r: () => adminApi.startStorageBackfill('m'), e: 'POST /api/admin/storage/backends/m/backfill' }, + { n: 'cancelStorageBackfill', r: () => adminApi.cancelStorageBackfill('m'), e: 'DELETE /api/admin/storage/backends/m/backfill' }, + { n: 'startStorageMigration', r: () => adminApi.startStorageMigration('files', 'dest'), e: 'POST /api/admin/storage/migrations' }, + { n: 'cancelStorageMigration', r: () => adminApi.cancelStorageMigration('files'), e: 'DELETE /api/admin/storage/migrations/files' }, + { n: 'refreshStorageStats', r: () => adminApi.refreshStorageStats(), e: 'POST /api/admin/storage/stats/refresh' }, { n: 'mcpTokens', r: () => adminApi.mcpTokens(), e: 'GET /api/admin/mcp-tokens' }, { n: 'deleteMcpToken', r: () => adminApi.deleteMcpToken(4), e: 'DELETE /api/admin/mcp-tokens/4' }, { n: 'oauthSessions', r: () => adminApi.oauthSessions(), e: 'GET /api/admin/oauth-sessions' }, diff --git a/client/src/api/client.ts b/client/src/api/client.ts index af4c5f61e..79aa8de2f 100644 --- a/client/src/api/client.ts +++ b/client/src/api/client.ts @@ -47,6 +47,11 @@ import { type BookingImportPreviewResponse, type BookingImportConfirmResponse, type BookingImportMode, + type StorageAdminState, + type StorageBackend, + type StorageConfigPut, + type StorageTestResponse, + type StorageUsage, } from '@trek/shared' import { getSocketId } from './websocket' import { probeNow } from '../sync/connectivity' @@ -637,6 +642,26 @@ export const adminApi = { updateNotificationPreferences: (prefs: Record>) => apiClient.put('/admin/notification-preferences', prefs).then(r => r.data), getDefaultUserSettings: () => apiClient.get('/admin/default-user-settings').then(r => r.data), updateDefaultUserSettings: (settings: Record) => apiClient.put('/admin/default-user-settings', settings).then(r => r.data), + getStorage: (): Promise => apiClient.get('/admin/storage').then(r => r.data), + updateStorage: (config: StorageConfigPut): Promise => + apiClient.put('/admin/storage', config).then(r => r.data), + // A probe is bounded by the target driver's own timeout (default 30s, ×2 with + // one retry) and a mirror probes its targets sequentially — the instance-wide + // 8s axios timeout would abort a legitimate slow probe, so this call carries + // its own generous ceiling. + testStorageBackend: (backend: StorageBackend): Promise => + apiClient.post('/admin/storage/test', { backend }, { timeout: 120_000 }).then(r => r.data), + startStorageBackfill: (backend: string): Promise<{ started: true }> => + apiClient.post(`/admin/storage/backends/${encodeURIComponent(backend)}/backfill`).then(r => r.data), + cancelStorageBackfill: (backend: string): Promise<{ cancelled: true }> => + apiClient.delete(`/admin/storage/backends/${encodeURIComponent(backend)}/backfill`).then(r => r.data), + startStorageMigration: (category: string, to: string): Promise<{ started: true }> => + apiClient.post('/admin/storage/migrations', { category, to }).then(r => r.data), + cancelStorageMigration: (category: string): Promise<{ cancelled: true }> => + apiClient.delete(`/admin/storage/migrations/${encodeURIComponent(category)}`).then(r => r.data), + // A full scan of a large install can exceed the 8s instance timeout. + refreshStorageStats: (): Promise => + apiClient.post('/admin/storage/stats/refresh', undefined, { timeout: 120_000 }).then(r => r.data), } export const addonsApi = { diff --git a/client/src/components/Admin/storage/AdminStoragePanel.test.tsx b/client/src/components/Admin/storage/AdminStoragePanel.test.tsx new file mode 100644 index 000000000..8c7141e2f --- /dev/null +++ b/client/src/components/Admin/storage/AdminStoragePanel.test.tsx @@ -0,0 +1,1201 @@ +import { http, HttpResponse } from 'msw'; +import { beforeEach, describe, expect, it } from 'vitest'; +import { MASKED_SETTING_VALUE, type StorageAdminState, type StorageCategory, type StorageConfig } from '@trek/shared'; +import { server } from '../../../../tests/helpers/msw/server'; +import { fireEvent, render, screen, waitFor, within } from '../../../../tests/helpers/render'; +import { ToastContainer } from '../../shared/Toast'; +import AdminStoragePanel from './AdminStoragePanel'; + +const S3_MASKED = { + endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', + secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000, +}; + +function baseState(overrides: Partial = {}): StorageAdminState { + return { + backends: [ + { name: 'uploads-local', type: 'local', source: 'built-in', options: { root: '/data/uploads' }, categories: ['files', 'journey', 'covers', 'avatars', 'photos-google', 'photos-trek'] }, + { name: 'backups-local', type: 'local', source: 'built-in', options: { root: '/data/backups' }, categories: ['backups'] }, + { name: 'place-photos-local', type: 'local', source: 'env', options: { root: '/photos' }, categories: ['places'] }, + { name: 'off-box', type: 's3', source: 'settings', options: S3_MASKED, categories: ['covers'] }, + // Unassigned on purpose: the only row the mirror-target picker may + // offer, since a backend that serves a category can never also be a + // replica (the sync sweep would delete that category's objects). + { name: 'cold-store', type: 'local', source: 'settings', options: { root: '/data/cold' }, categories: [] }, + ], + categories: { + files: { backend: 'uploads-local', source: 'default' }, + journey: { backend: 'uploads-local', source: 'default' }, + covers: { backend: 'off-box', source: 'settings' }, + avatars: { backend: 'uploads-local', source: 'default' }, + places: { backend: 'place-photos-local', source: 'default' }, + 'photos-google': { backend: 'uploads-local', source: 'default' }, + 'photos-trek': { backend: 'uploads-local', source: 'default' }, + backups: { backend: 'backups-local', source: 'default' }, + }, + health: { replicaFailures: [] }, + seedFilePresent: false, + usage: null, + backfills: [], + migrations: [], + version: 0, + configError: null, + ...overrides, + }; +} + +function mirroredState(): StorageAdminState { + const state = baseState(); + // off-box is this fixture's REPLICA, so it must not also be a direct + // category target — the server refuses that config outright (a backups + // sweep would delete the covers objects), so covers goes back to its + // default. cold-store stays free as the second offerable target. + state.backends.find((b) => b.name === 'off-box')!.categories = []; + state.backends.find((b) => b.name === 'uploads-local')!.categories.push('covers'); + state.categories.covers = { backend: 'uploads-local', source: 'default' }; + state.backends.push({ + name: 'mirror', type: 'mirror', source: 'settings', + options: { primary: 'backups-local', replicas: ['off-box'] }, categories: ['backups'], + }); + state.categories.backups = { backend: 'mirror', source: 'settings' }; + return state; +} + +function stubGet(state: StorageAdminState) { + server.use(http.get('/api/admin/storage', () => HttpResponse.json(state))); +} + +async function renderPanel(state: StorageAdminState = baseState()) { + stubGet(state); + render( + <> + + + , + ); + await waitFor(() => expect(screen.getByText('Backends')).toBeInTheDocument()); +} + +const backendRow = (name: string) => screen.getByTestId(`storage-backend-${name}`); +const categoryRow = (category: string) => screen.getByTestId(`storage-category-${category}`); + +describe('AdminStoragePanel', () => { + beforeEach(() => { + // Each test re-stubs GET; PUT/POST are stubbed where used. + }); + + it('FE-ADMIN-STOR-001: renders every backend with type badge, source tag and its categories', async () => { + await renderPanel(); + const uploads = backendRow('uploads-local'); + expect(within(uploads).getByText('Local')).toBeInTheDocument(); + expect(within(uploads).getByText('Built-in')).toBeInTheDocument(); + expect(within(uploads).getByText(/Used by: Trip documents/)).toBeInTheDocument(); + const env = backendRow('place-photos-local'); + expect(within(env).getByText('Environment')).toBeInTheDocument(); + expect(within(env).getByText(/read-only/)).toBeInTheDocument(); + expect(within(env).queryByRole('button', { name: 'Edit' })).not.toBeInTheDocument(); + expect(within(env).queryByRole('button', { name: 'Remove' })).not.toBeInTheDocument(); + const offBox = backendRow('off-box'); + expect(within(offBox).getByText('S3')).toBeInTheDocument(); + expect(within(offBox).getByText('Settings')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-002: the mask echoes through edit → save untouched (no-op by contract)', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(backendRow('off-box')).getByRole('button', { name: 'Edit' })); + expect(screen.getByLabelText(/Secret access key/)).toHaveValue(MASKED_SETTING_VALUE); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + const body = putBody as StorageConfig; + const offBox = body.backends.find((b) => b.name === 'off-box')!; + expect((offBox.options as Record).secretAccessKey).toBe(MASKED_SETTING_VALUE); + }); + + it('FE-ADMIN-STOR-003: the PUT carries only the settings-owned document', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + // Touch something to enable Save: reassign the files category. + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned (baseState().usage === null) — the migrate + // prompt fires; this test is about PUT shape, not the prompt, so route + // new writes only (today's save behavior, unstripped). + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + const body = putBody as StorageConfig; + expect(body.backends.map((b) => b.name)).toEqual(['off-box', 'cold-store']); // built-ins/env never in the body + expect(body.categories).toEqual({ covers: 'off-box', files: 'off-box' }); + }); + + it('FE-ADMIN-STOR-004: reassigning a category shows the objects-do-not-move warning inline', async () => { + await renderPanel(); + expect(screen.queryByText(/Existing objects do not move/)).not.toBeInTheDocument(); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + expect(within(categoryRow('files')).getByText(/Existing objects do not move/)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-005: a 400 renders the server message verbatim next to Save', async () => { + await renderPanel(); + const registryError = + "backend 'off-box' has a plaintext secret 'secretAccessKey' but ENCRYPTION_KEY is not set — set ENCRYPTION_KEY explicitly to save credentialed storage backends (the implicit key persisted in the data directory is not accepted: it rides inside backups)"; + server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: registryError }, { status: 400 }))); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned — the migrate prompt fires first; route new + // writes only, matching this test's focus on the save-error rendering. + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + expect(await screen.findByText(registryError)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-006: Remove pre-checks assignments in the confirm dialog, then omits the backend from the PUT', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(backendRow('off-box')).getByRole('button', { name: 'Remove' })); + expect(screen.getByText(/Still assigned to: Cover images/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Remove backend' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as StorageConfig).backends.map((b) => b.name)).toEqual(['cold-store']); + }); + + it('FE-ADMIN-STOR-007: Test on a mirrored primary probes the draft primary and each replica individually (never the mirror stub), merging into one result', async () => { + const posted: Array<{ name: string; type: string }> = []; + await renderPanel(mirroredState()); + server.use( + http.post('/api/admin/storage/test', async ({ request }) => { + const body = (await request.json()) as { backend: { name: string; type: string } }; + posted.push(body.backend); + const ok = body.backend.name === 'backups-local'; + return HttpResponse.json({ + ok, + targets: [{ name: body.backend.name, ok, ...(ok ? {} : { error: 'connect ECONNREFUSED' }) }], + }); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Test' })); + await within(backendRow('backups-local')).findByText('Test failed'); + expect(within(backendRow('backups-local')).getByText(/connect ECONNREFUSED/)).toBeInTheDocument(); + // Two separate probes, each a concrete backend — never the mirror object itself. + expect(posted.map((b) => b.name).sort()).toEqual(['backups-local', 'off-box']); + expect(posted.every((b) => b.type !== 'mirror')).toBe(true); + }); + + it('FE-ADMIN-STOR-042: mirrored-row Test uses the DRAFT options of a target, not the saved ones (client-side mirror expansion)', async () => { + const posted: Array<{ name: string; options: Record }> = []; + await renderPanel(mirroredState()); + server.use( + http.post('/api/admin/storage/test', async ({ request }) => { + const body = (await request.json()) as { backend: { name: string; options: Record } }; + posted.push(body.backend); + return HttpResponse.json({ ok: true, targets: [{ name: body.backend.name, ok: true }] }); + }), + ); + // Edit the replica (off-box) in the draft without saving — an unsaved endpoint change. + fireEvent.click(within(backendRow('off-box')).getByRole('button', { name: 'Edit' })); + fireEvent.change(screen.getByLabelText(/Endpoint URL/), { target: { value: 'http://edited.example:9000' } }); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Test' })); + await within(backendRow('backups-local')).findByText('Connection OK'); + const offBoxProbe = posted.find((b) => b.name === 'off-box'); + expect(offBoxProbe?.options.endpoint).toBe('http://edited.example:9000'); + }); + + it('FE-ADMIN-STOR-008: the health strip lists replica failures with a relative age; all-clear otherwise', async () => { + const now = Date.now(); + await renderPanel( + baseState({ + health: { replicaFailures: [{ backend: 'off-box', key: 'backups/db.sqlite3', op: 'put', error: 'timeout', at: now - 120_000 }] }, + }), + ); + expect(screen.getByText(/put of backups\/db\.sqlite3 on off-box failed: timeout/)).toBeInTheDocument(); + expect(screen.getByText(/2 minutes ago/)).toBeInTheDocument(); + expect(screen.queryByText('No replica failures recorded.')).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-009: all-clear health and the seed-file note', async () => { + await renderPanel(baseState({ seedFilePresent: true })); + expect(screen.getByText('No replica failures recorded.')).toBeInTheDocument(); + expect(screen.getByText(/seed file is present but ignored/)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-010: a managed-mode 403 on GET renders its body message gracefully', async () => { + server.use( + http.get('/api/admin/storage', () => + HttpResponse.json( + { error: 'This is configured by the operator of this instance.', code: 'MANAGED_FORBIDDEN' }, + { status: 403 }, + ), + ), + ); + render(); + expect(await screen.findByText('This is configured by the operator of this instance.')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-011: mirrors fold — no mirror row, primary and replica are decorated, categories union', async () => { + await renderPanel(mirroredState()); + expect(screen.queryByTestId('storage-backend-mirror')).not.toBeInTheDocument(); + const primary = backendRow('backups-local'); + expect(within(primary).getByText('Mirrored to: off-box')).toBeInTheDocument(); + expect(within(primary).getByText(/Used by: .*Backups/)).toBeInTheDocument(); + const replica = backendRow('off-box'); + expect(within(replica).getByText('Replica of: backups-local')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-012: checking a target on an unmirrored primary synthesizes the mirror and reroutes its categories', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + // off-box serves covers, so the picker never offers it as a replica. + expect(screen.queryByRole('checkbox', { name: 'off-box' })).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('checkbox', { name: 'cold-store' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + const body = putBody as StorageConfig; + expect(body.backends.map((b) => b.name)).toEqual( + expect.arrayContaining(['off-box', 'backups-local', 'backups-local-mirror']), + ); + expect(body.backends.find((b) => b.name === 'backups-local-mirror')!.options).toEqual({ + primary: 'backups-local', replicas: ['cold-store'], + }); + expect(body.categories.backups).toBe('backups-local-mirror'); // default-sourced category rewritten + }); + + it('FE-ADMIN-STOR-013: editing an already-mirrored primary adopts the foreign-named mirror in place', async () => { + let putBody: unknown; + await renderPanel(mirroredState()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(mirroredState()); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + expect(screen.getByRole('checkbox', { name: 'off-box' })).toBeChecked(); // initialTargets from the fold + fireEvent.click(screen.getByRole('checkbox', { name: 'cold-store' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + const mirrors = (putBody as StorageConfig).backends.filter((b) => b.type === 'mirror'); + expect(mirrors).toHaveLength(1); + expect(mirrors[0]!.name).toBe('mirror'); // adopted, not renamed + expect(mirrors[0]!.options).toEqual({ primary: 'backups-local', replicas: ['off-box', 'cold-store'] }); + }); + + it('FE-ADMIN-STOR-048: the mirror-target picker never offers a backend that serves a category — but keeps already-selected targets visible', async () => { + await renderPanel(mirroredState()); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + // Serving rows are refused as replicas by the server (a sync sweep would + // delete their categories' objects) — so they are never offered here. + expect(screen.queryByRole('checkbox', { name: 'uploads-local' })).not.toBeInTheDocument(); + expect(screen.queryByRole('checkbox', { name: 'place-photos-local' })).not.toBeInTheDocument(); + // ...and never the row being edited itself, which serves backups. + expect(screen.queryByRole('checkbox', { name: 'backups-local' })).not.toBeInTheDocument(); + // Free rows are offered; the current target stays listed so it can be unchecked. + expect(screen.getByRole('checkbox', { name: 'cold-store' })).not.toBeChecked(); + expect(screen.getByRole('checkbox', { name: 'off-box' })).toBeChecked(); + }); + + it('FE-ADMIN-STOR-014: unchecking every target dissolves the mirror and re-points its categories', async () => { + let putBody: unknown; + await renderPanel(mirroredState()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('checkbox', { name: 'off-box' })); // uncheck the only target + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + const body = putBody as StorageConfig; + expect(body.backends.some((b) => b.type === 'mirror')).toBe(false); + // backups was settings-sourced at the mirror → re-pointed at the primary, not dropped. + expect(body.categories.backups).toBe('backups-local'); + }); + + it('FE-ADMIN-STOR-015: category selects deal in primaries — picking a mirrored primary writes its mirror, caches get the advisory', async () => { + let putBody: unknown; + await renderPanel(mirroredState()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(mirroredState()); + }), + ); + // The backups row displays the PRIMARY name, never 'mirror'. + expect(within(categoryRow('backups')).getByText('backups-local')).toBeInTheDocument(); + // Route the places cache through the mirrored primary. + fireEvent.click(within(categoryRow('places')).getByText('place-photos-local (default)')); + const choices = screen.getAllByText('backups-local'); + fireEvent.click(choices[choices.length - 1]!); + expect(within(categoryRow('places')).getByText(/re-fetchable/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // A genuine reassignment (place-photos-local → backups-local) with no + // usage scan — the migrate prompt fires; route new writes only, matching + // this test's focus on mirror-routing of the category value. + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as StorageConfig).categories.places).toBe('mirror'); // the adopted mirror, under the hood + }); + + it('FE-ADMIN-STOR-016: a second mirror on the same primary renders unfolded with the degenerate note, Test+Remove only', async () => { + const state = mirroredState(); + state.backends.push({ + name: 'mirror2', type: 'mirror', source: 'settings', + options: { primary: 'backups-local', replicas: ['cold-store'] }, categories: [], + }); + await renderPanel(state); + const row = screen.getByTestId('storage-backend-mirror2'); + expect(within(row).getByText(/A second mirror wraps backups-local/)).toBeInTheDocument(); + expect(within(row).getByRole('button', { name: 'Test' })).toBeInTheDocument(); + expect(within(row).getByRole('button', { name: 'Remove' })).toBeInTheDocument(); + expect(within(row).queryByRole('button', { name: 'Edit' })).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-017: category rows show the display name, the raw id badge, and the description; photos is gone', async () => { + await renderPanel(); + const files = categoryRow('files'); + expect(within(files).getByText('Trip documents')).toBeInTheDocument(); + expect(within(files).getByText('files')).toBeInTheDocument(); // the monospace id badge + expect(within(files).getByText(/tickets, PDFs, booking confirmations/)).toBeInTheDocument(); + expect(within(categoryRow('photos-google')).getByText(/re-fetchable, safe to lose/)).toBeInTheDocument(); + expect(screen.queryByTestId('storage-category-photos')).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-018: usage renders on the header, backend rows, and category rows; never-computed degrades', async () => { + const usage = { + computedAt: Date.now() - 3_600_000, + categories: Object.fromEntries( + (['files', 'journey', 'covers', 'avatars', 'places', 'photos-google', 'photos-trek', 'backups'] as const).map( + (c) => [c, { objects: 2, bytes: 1024 * 1024 }], + ), + ), + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + expect(screen.getByText(/Usage computed .*hour/)).toBeInTheDocument(); + expect(within(backendRow('backups-local')).getByText(/2 objects · 1\.0 MB/)).toBeInTheDocument(); + expect(within(categoryRow('files')).getByText(/2 objects · 1\.0 MB/)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-019: never-computed shows the compute prompt; Refresh triggers the scan and re-renders', async () => { + await renderPanel(); + expect(screen.getByText('Usage not computed yet')).toBeInTheDocument(); + server.use( + http.post('/api/admin/storage/stats/refresh', () => + HttpResponse.json({ + computedAt: Date.now(), + categories: Object.fromEntries( + (['files', 'journey', 'covers', 'avatars', 'places', 'photos-google', 'photos-trek', 'backups'] as const).map( + (c) => [c, { objects: 1, bytes: 2048 }], + ), + ), + legacyPhotos: { objects: 0, bytes: 0 }, + }), + ), + ); + fireEvent.click(screen.getByRole('button', { name: 'Compute now' })); + await screen.findByText(/Usage computed/); + expect(within(categoryRow('files')).getByText(/1 objects · 2\.0 KB/)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-020: Sync now runs the backfill — running line with counts, then the done line (50ms test poll)', async () => { + let polls = 0; + await renderPanel(mirroredState()); + server.use( + http.post('/api/admin/storage/backends/mirror/backfill', () => HttpResponse.json({ started: true })), + http.get('/api/admin/storage', () => { + polls += 1; + const state = mirroredState(); + (state as StorageAdminState).backfills = + polls < 3 + ? [{ backend: 'mirror', status: 'running', done: 3, total: 10, copied: 2, skipped: 1, failed: 0, deleted: 0, startedAt: 1 }] + : [{ backend: 'mirror', status: 'done', done: 10, total: 10, copied: 8, skipped: 2, failed: 0, deleted: 1, startedAt: 1, finishedAt: 2 }]; + return HttpResponse.json(state); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Sync now' })); + await within(backendRow('backups-local')).findByText(/Syncing… 3\/10/); + expect(within(backendRow('backups-local')).getByText(/2 copied · 1 skipped · 0 failed/)).toBeInTheDocument(); + await within(backendRow('backups-local')).findByText(/Sync finished: 8 copied, 1 deleted, 0 failed/); + }); + + it('FE-ADMIN-STOR-021: Cancel sync calls the DELETE endpoint', async () => { + let cancelled = false; + const state = mirroredState(); + (state as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'running', done: 1, total: 10, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }, + ]; + await renderPanel(state); + server.use( + http.delete('/api/admin/storage/backends/mirror/backfill', () => { + cancelled = true; + return HttpResponse.json({ cancelled: true }); + }), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Cancel sync' })); + await waitFor(() => expect(cancelled).toBe(true)); + }); + + it('FE-ADMIN-STOR-022: a save that ADDED mirror targets raises the sync prompt on that row; dismiss clears it', async () => { + await renderPanel(); // no mirror yet + server.use( + http.put('/api/admin/storage', async () => HttpResponse.json(mirroredState())), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('checkbox', { name: 'cold-store' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await within(backendRow('backups-local')).findByText(/Existing objects are not replicated yet/); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Dismiss' })); + expect(screen.queryByText(/Existing objects are not replicated yet/)).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-023: the poll never clobbers a dirty draft', async () => { + const state = mirroredState(); + (state as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'running', done: 1, total: 5, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }, + ]; + await renderPanel(state); + // Dirty the draft: reassign files. + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + expect(screen.getByText('Unsaved changes')).toBeInTheDocument(); + // Let several polls elapse (50ms test poll): the dirty marker must survive. + await new Promise((r) => setTimeout(r, 300)); + expect(screen.getByText('Unsaved changes')).toBeInTheDocument(); + expect(within(categoryRow('files')).getByText(/Existing objects do not move/)).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-024: a poll GET that resolves after a save must not overwrite the saved state', async () => { + const runningState = mirroredState(); + (runningState as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'running', done: 1, total: 5, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }, + ]; + await renderPanel(runningState); + + // The save's own PUT will land a distinct, fresh world: files reassigned + // to off-box, backfill finished. This must be what the panel shows. + const savedState = mirroredState(); + savedState.categories.files = { backend: 'off-box', source: 'settings' }; + + // Hold ONLY the first poll GET open (the genuinely stale one, issued + // before the save) until the test releases it explicitly. Every later + // poll GET is answered immediately with the world the mock server + // currently holds — pre-save before the PUT, saved after — because a + // real server can never answer a GET issued after the PUT committed + // with the pre-save world. (An earlier version of this stub deferred + // EVERY GET and rebound the release handle to the newest one; a poll + // tick squeezing in after the save then got released with the pre-save + // payload, which no real backend can produce — and the seq guard + // rightly let it through, failing the test.) + let releaseStalePoll: (() => void) | undefined; + let putLanded = false; + server.use( + http.get('/api/admin/storage', () => { + if (releaseStalePoll) { + return HttpResponse.json(putLanded ? savedState : runningState); + } + return new Promise((resolve) => { + releaseStalePoll = () => resolve(HttpResponse.json(runningState) as unknown as Response); + }); + }), + ); + // Wait for the 50ms interval to fire and get stuck on the deferred GET. + await waitFor(() => expect(releaseStalePoll).toBeDefined()); + + server.use( + http.put('/api/admin/storage', async ({ request }) => { + await request.json(); + putLanded = true; + return HttpResponse.json(savedState); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned — the migrate prompt fires; route new writes + // only, matching this test's focus on the poll/save race. + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect(within(categoryRow('files')).getByText('off-box')).toBeInTheDocument(); + + // Now let the stale poll (issued before the save, carrying the pre-save + // world) resolve. It must be dropped, not applied over the save. + releaseStalePoll!(); + await new Promise((r) => setTimeout(r, 150)); + expect(within(categoryRow('files')).getByText('off-box')).toBeInTheDocument(); + expect(within(categoryRow('files')).queryByText('uploads-local')).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-030: reassigning a populated category prompts before saving; Move strips it from the PUT and POSTs the migration after save', async () => { + let putBody: unknown; + let putCalled = false; + let migrationBody: unknown; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putCalled = true; + putBody = await request.json(); + const saved = baseState(); + saved.categories.files = { backend: 'off-box', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', async ({ request }) => { + migrationBody = await request.json(); + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + + expect(await screen.findByRole('alertdialog')).toBeInTheDocument(); + expect(putCalled).toBe(false); + expect(screen.getByText(/Trip documents: 3 objects \(3\.0 KB\) from uploads-local to off-box/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + // files is default-sourced (uploads-local) in baseState() — stripping restores "no override". + expect((putBody as StorageConfig).categories.files).toBeUndefined(); + expect(migrationBody).toEqual({ category: 'files', to: 'off-box' }); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-031: Just route new writes saves the reassignment as today, no migration POST', async () => { + let putBody: unknown; + let migrationPosted = false; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + const saved = baseState(); + saved.categories.files = { backend: 'off-box', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', () => { + migrationPosted = true; + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as StorageConfig).categories.files).toBe('off-box'); + expect(migrationPosted).toBe(false); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); + + it("FE-ADMIN-STOR-036: reassigning onto a mirrored primary POSTs the migration to the mirror's wire name, not the bare primary", async () => { + let migrationBody: unknown; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...mirroredState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + await request.json(); + const saved = mirroredState(); + saved.categories.files = { backend: 'mirror', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', async ({ request }) => { + migrationBody = await request.json(); + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('backups-local'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + + expect(await screen.findByRole('alertdialog')).toBeInTheDocument(); + // The prompt still displays the PRIMARY name — the mirror stays hidden. + expect(screen.getByText(/Trip documents: 3 objects \(3\.0 KB\) from uploads-local to backups-local/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + // The POST carries the mirror's raw wire name, not 'backups-local' — + // posting the bare primary would silently drop replication. + expect(migrationBody).toEqual({ category: 'files', to: 'mirror' }); + }); + + it('FE-ADMIN-STOR-037: a queued migration waits out a running backfill (the server 409s on either while the other runs)', async () => { + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + const state = mirroredState(); + (state as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'running', done: 1, total: 5, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }, + ]; + await renderPanel({ ...state, usage } as StorageAdminState); + const savedState = mirroredState(); + savedState.categories.files = { backend: 'off-box', source: 'settings' }; + // The save itself doesn't touch the backfill — a real server's PUT + // response reflects it still running, exactly like the GET poll below. + (savedState as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'running', done: 1, total: 5, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }, + ]; + (savedState as StorageAdminState).usage = usage as StorageAdminState['usage']; + let migrationPosted = false; + // Held running until the test flips it — the backfill only ever turns + // terminal via this GET stub, never via a real timer. + let backfillDone = false; + // The running backfill has the panel polling GET from first render, so the + // stub must stay faithful to a real server: the files override appears in + // GET responses only AFTER the PUT has landed. Serving the post-save world + // early lets a poll that resolves while the confirm dialog is open erase + // the candidate (state already shows off-box, so from === to) and nothing + // ever gets queued. + let putSeen = false; + let polls = 0; + server.use( + http.put('/api/admin/storage', () => { + putSeen = true; + return HttpResponse.json(savedState); + }), + http.post('/api/admin/storage/migrations', () => { + migrationPosted = true; + return HttpResponse.json({ started: true }); + }), + http.get('/api/admin/storage', () => { + polls += 1; + const next = putSeen ? { ...savedState } : ({ ...mirroredState(), usage } as StorageAdminState); + next.backfills = [ + { + backend: 'mirror', status: backfillDone ? 'done' : 'running', done: backfillDone ? 5 : 1, total: 5, + copied: backfillDone ? 5 : 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1, + ...(backfillDone ? { finishedAt: 2 } : {}), + }, + ]; + return HttpResponse.json(next); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + // Force the race this test once flaked on: let polls land while the dialog + // is open — they carry the pre-save world, so the candidate must survive. + const pollsAtOpen = polls; + await waitFor(() => expect(polls).toBeGreaterThan(pollsAtOpen + 1)); + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + + // The backfill is still running — the queued migration must not POST yet. + await new Promise((r) => setTimeout(r, 200)); + expect(migrationPosted).toBe(false); + + // Once the backfill turns terminal, the queued migration is free to start. + backfillDone = true; + await waitFor(() => expect(migrationPosted).toBe(true), { timeout: 2000 }); + }); + + it('FE-ADMIN-STOR-032: zero-object reassigns save without any prompt', async () => { + let putBody: unknown; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 0, bytes: 0 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + expect((putBody as StorageConfig).categories.files).toBe('off-box'); + }); + + it('FE-ADMIN-STOR-039 (audit #7): a 409 shows the distinct conflict message, refreshes state, and preserves the dirty draft for review', async () => { + let getCalls = 0; + let putVersion: number | undefined; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 0, bytes: 0 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + const initial = { ...baseState(), usage, version: 3 } as StorageAdminState; + // Simulates the audit #7 scenario: a category migration's flip lands + // between this load and the operator's save — 'files' moves to a + // different backend, at a newer version. + const refreshed = { + ...initial, + version: 4, + categories: { ...initial.categories, files: { backend: 'backups-local', source: 'settings' as const } }, + }; + server.use(http.get('/api/admin/storage', () => HttpResponse.json(++getCalls === 1 ? initial : refreshed))); + render( + <> + + + , + ); + await waitFor(() => expect(screen.getByText('Backends')).toBeInTheDocument()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putVersion = (await request.json() as StorageConfig & { version: number }).version; + return HttpResponse.json({ error: 'stale' }, { status: 409 }); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + + // The version this PUT submits is the one the draft was loaded at. + expect( + await screen.findByText( + 'Storage settings changed since you loaded them, so your changes were not saved. Discard them and reload the saved settings to start over.', + ), + ).toBeInTheDocument(); + // The copy points at a real escape, and the escape is on screen. + expect(screen.getByRole('button', { name: 'Discard my changes and reload' })).toBeInTheDocument(); + expect(putVersion).toBe(3); + + // refreshState() ran (a second GET landed) and the panel now reflects + // the fresh world, yet the dirty draft — still showing the operator's + // own off-box pick — was never clobbered. + await waitFor(() => expect(getCalls).toBeGreaterThanOrEqual(2)); + expect(screen.getByText('Unsaved changes')).toBeInTheDocument(); + expect(within(categoryRow('files')).getByText('off-box')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-049: a 409 is recoverable in place — Discard reloads the saved settings and the next save carries the FRESH version', async () => { + let getCalls = 0; + const putVersions: number[] = []; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 0, bytes: 0 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + const initial = { ...baseState(), usage, version: 3 } as StorageAdminState; + const refreshed = { ...initial, version: 4 }; + server.use(http.get('/api/admin/storage', () => HttpResponse.json(++getCalls === 1 ? initial : refreshed))); + render( + <> + + + , + ); + await waitFor(() => expect(screen.getByText('Backends')).toBeInTheDocument()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + const body = (await request.json()) as StorageConfig & { version: number }; + putVersions.push(body.version); + // Only the stale version conflicts; the fresh one goes through. + return body.version === 4 + ? HttpResponse.json({ ...refreshed, version: 5 }) + : HttpResponse.json({ error: 'stale' }, { status: 409 }); + }), + ); + + const reassignFiles = () => { + fireEvent.click(within(categoryRow('files')).getByText(/^uploads-local/)); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + }; + + reassignFiles(); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText(/Storage settings changed since you loaded them/); + expect(putVersions).toEqual([3]); + + // Without this action the draft keeps its stale version forever: setDraft + // re-attaches it to every later edit, so every retry 409s again. + fireEvent.click(screen.getByRole('button', { name: 'Discard my changes and reload' })); + await waitFor(() => expect(screen.queryByText('Unsaved changes')).not.toBeInTheDocument()); + expect(screen.queryByText(/Storage settings changed since you loaded them/)).not.toBeInTheDocument(); + expect(within(categoryRow('files')).getByText(/^uploads-local/)).toBeInTheDocument(); // saved world, not the draft + + reassignFiles(); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + expect(putVersions).toEqual([3, 4]); + }); + + it('FE-ADMIN-STOR-033: two candidates queue sequentially: the second POSTs only after the first turns terminal', async () => { + const usage = { + computedAt: Date.now(), + categories: { + files: { objects: 3, bytes: 3072 }, + journey: { objects: 2, bytes: 2048 }, + }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + const savedState = baseState(); + savedState.categories.files = { backend: 'off-box', source: 'settings' }; + savedState.categories.journey = { backend: 'off-box', source: 'settings' }; + const migrationPosts: Array<{ category: StorageCategory; to: string }> = []; + // Held 'running' until the test explicitly flips this — deterministic, + // unlike counting poll ticks against the 50ms test interval. + let firstDone = false; + let postCount = 0; + server.use( + http.put('/api/admin/storage', () => HttpResponse.json(savedState)), + http.post('/api/admin/storage/migrations', async ({ request }) => { + postCount += 1; + const body = (await request.json()) as { category: StorageCategory; to: string }; + migrationPosts.push(body); + // Pins the single-flight fix: the second POST must never arrive + // before the first migration is server-confirmed terminal — a + // regression here means the queue's synchronous re-fire raced ahead + // of admin.state showing the first as done. + if (postCount === 2) expect(firstDone).toBe(true); + return HttpResponse.json({ started: true }); + }), + http.get('/api/admin/storage', () => { + const state = { ...savedState }; + state.migrations = + migrationPosts.length === 0 + ? [] + : [ + { + category: migrationPosts[0]!.category, from: 'uploads-local', to: migrationPosts[0]!.to, + status: firstDone ? 'done' : 'running', done: firstDone ? 3 : 1, total: 3, + copied: firstDone ? 3 : 1, skipped: 0, failed: 0, startedAt: 1, + ...(firstDone ? { finishedAt: 2 } : {}), + }, + ]; + return HttpResponse.json(state); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(within(categoryRow('journey')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + + await waitFor(() => expect(postCount).toBe(1)); + expect(migrationPosts[0]!.category).toBe('files'); + // At the moment the first POST arrives, exactly one POST has happened — + // and the second must not fire while the first is still running, no + // matter how many poll ticks elapse. + expect(postCount).toBe(1); + await new Promise((r) => setTimeout(r, 200)); + expect(postCount).toBe(1); + // Once the first turns terminal, the second is dequeued — the POST + // handler's own assertion (above) pins that it never arrives early. + firstDone = true; + await waitFor(() => expect(postCount).toBe(2), { timeout: 2000 }); + expect(migrationPosts[1]!.category).toBe('journey'); + }); + + it('FE-ADMIN-STOR-034: a done migration row shows the reclaimable line; a running one shows progress + cancel wired to DELETE', async () => { + let cancelledCategory: string | null = null; + const state = baseState(); + (state as StorageAdminState).migrations = [ + { + category: 'files', from: 'uploads-local', to: 'off-box', + status: 'running', done: 2, total: 5, copied: 1, skipped: 1, failed: 0, startedAt: 1, + }, + { + category: 'journey', from: 'uploads-local', to: 'off-box', + status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, startedAt: 1, finishedAt: 2, + reclaimable: { objects: 4, bytes: 4096 }, + }, + ]; + await renderPanel(state); + server.use( + http.delete('/api/admin/storage/migrations/files', () => { + cancelledCategory = 'files'; + return HttpResponse.json({ cancelled: true }); + }), + ); + expect(screen.getByText(/Moving Trip documents… 2\/5/)).toBeInTheDocument(); + expect(screen.getByText(/Move finished: 4 copied, 0 skipped/)).toBeInTheDocument(); + expect(screen.getByText(/4 objects \(4\.0 KB\) remain on uploads-local — reclaim manually/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Cancel move' })); + await waitFor(() => expect(cancelledCategory).toBe('files')); + }); + + it('FE-ADMIN-STOR-038: a done migration with sweep failures shows the doneFailures line; a clean one shows nothing extra', async () => { + const state = baseState(); + (state as StorageAdminState).migrations = [ + { + category: 'files', from: 'uploads-local', to: 'off-box', + status: 'done', done: 4, total: 4, copied: 3, skipped: 0, failed: 2, startedAt: 1, finishedAt: 2, + }, + { + category: 'journey', from: 'uploads-local', to: 'off-box', + status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, startedAt: 1, finishedAt: 2, + }, + ]; + await renderPanel(state); + expect(screen.getByText(/2 failed — those objects were not copied to the new backend/)).toBeInTheDocument(); + // The clean migration (failed: 0) must not render the line at all. + expect(within(screen.getByTestId('storage-migration-journey')).queryByText(/failed —/)).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-035: a failed save disarms the sync-prompt snapshot', async () => { + await renderPanel(); + server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: 'nope' }, { status: 400 }))); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('checkbox', { name: 'cold-store' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('nope'); + + // The draft-only mirror (never saved) still renders its "Sync now" — + // clicking it drives a genuine admin.state refresh (via refreshState()) + // that lands a world whose mirror targets grew. With the pending-prompt + // snapshot disarmed by the failed save, the sync prompt must never appear. + server.use( + http.post('/api/admin/storage/backends/backups-local-mirror/backfill', () => HttpResponse.json({ started: true })), + http.get('/api/admin/storage', () => HttpResponse.json(mirroredState())), + ); + fireEvent.click(within(backendRow('backups-local')).getByRole('button', { name: 'Sync now' })); + await new Promise((r) => setTimeout(r, 100)); + expect(screen.queryByText(/Existing objects are not replicated yet/)).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-040: a non-null configError renders a warning banner naming the error; save stays enabled', async () => { + await renderPanel(baseState({ configError: "'storage.categories' must be a JSON object" })); + const banner = screen.getByText( + "Stored storage settings failed to load — saving will replace them: 'storage.categories' must be a JSON object", + ); + expect(banner).toHaveAttribute('role', 'alert'); + // Save is the recovery path — it must not be force-disabled by the banner + // (it's still gated on `dirty` like any other save, unrelated to this). + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + expect(screen.getByRole('button', { name: 'Save changes' })).not.toBeDisabled(); + }); + + it('FE-ADMIN-STOR-041: no configError renders no banner', async () => { + await renderPanel(baseState({ configError: null })); + expect(screen.queryByText(/failed to load/)).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-043: the migrate-prompt dialog recomputes on render — reassigning another category while it is open adds it, and the confirm strips/queues BOTH', async () => { + let putBody: unknown; + const migrationBodies: unknown[] = []; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + const saved = baseState(); + saved.categories.files = { backend: 'off-box', source: 'settings' }; + saved.categories.journey = { backend: 'off-box', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', async ({ request }) => { + migrationBodies.push(await request.json()); + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + const dialog = await screen.findByRole('alertdialog'); + expect(within(dialog).getByText(/Trip documents: 3 objects/)).toBeInTheDocument(); + expect(within(dialog).queryByText(/Journey photos/)).not.toBeInTheDocument(); + + // Reassign a SECOND category while the dialog is still open — no usage + // scan for it, so it prompts with the unknown-size line. + fireEvent.click(within(categoryRow('journey')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + + // The dialog re-rendered with the fresh candidate set — never the stale + // one-category snapshot from when it opened. + expect(within(dialog).getByText(/Journey photos: unknown size/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + // Both categories were stripped from the PUT body (moveAndSave's own + // recompute at confirm time), not just the one visible when Save was clicked. + expect((putBody as StorageConfig).categories.files).toBeUndefined(); + expect((putBody as StorageConfig).categories.journey).toBeUndefined(); + // ...and both were queued/POSTed as migrations. + await waitFor(() => expect(migrationBodies).toHaveLength(2)); + expect(migrationBodies).toEqual( + expect.arrayContaining([ + { category: 'files', to: 'off-box' }, + { category: 'journey', to: 'off-box' }, + ]), + ); + }); + + it('FE-ADMIN-STOR-044: the migrate-prompt dialog has a Cancel button that dismisses it without saving or losing the draft edit', async () => { + let putCalled = false; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', () => { + putCalled = true; + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + const choices = screen.getAllByText('off-box'); + fireEvent.click(choices[choices.length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + const dialog = await screen.findByRole('alertdialog'); + fireEvent.click(within(dialog).getByRole('button', { name: 'Cancel' })); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + expect(putCalled).toBe(false); + // The draft edit itself survives — only the dialog closed. + expect(screen.getByText('Unsaved changes')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STOR-045: a failed migration start clears the remaining queue with a toast naming the dropped categories, refreshes state, and the queued line clears', async () => { + const usage = { + computedAt: Date.now(), + categories: { + files: { objects: 3, bytes: 3072 }, + journey: { objects: 2, bytes: 2048 }, + }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + const savedState = baseState(); + savedState.categories.files = { backend: 'off-box', source: 'settings' }; + savedState.categories.journey = { backend: 'off-box', source: 'settings' }; + let getCalls = 0; + let releasePost: (() => void) | undefined; + server.use( + http.put('/api/admin/storage', () => HttpResponse.json(savedState)), + http.post('/api/admin/storage/migrations', () => { + return new Promise((resolve) => { + releasePost = () => resolve(HttpResponse.json({ error: 'busy' }, { status: 409 }) as unknown as Response); + }); + }), + http.get('/api/admin/storage', () => { + getCalls += 1; + return HttpResponse.json(savedState); + }), + ); + fireEvent.click(within(categoryRow('files')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(within(categoryRow('journey')).getByText('uploads-local (default)')); + fireEvent.click(screen.getAllByText('off-box')[screen.getAllByText('off-box').length - 1]!); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + + // The first candidate (files) is dequeued and its POST is in flight; the + // remaining queue (journey) renders as an explicit, visible line. + await waitFor(() => expect(releasePost).toBeDefined()); + expect(screen.getByText('Queued: Journey photos')).toBeInTheDocument(); + + // The POST fails (409 busy) — the remaining queue is dropped, named in a + // toast, and never silently retried. + releasePost!(); + await screen.findByText('Could not start the next migration — the remaining queue was cleared: Journey photos'); + expect(screen.queryByText(/^Queued:/)).not.toBeInTheDocument(); + await waitFor(() => expect(getCalls).toBeGreaterThanOrEqual(1)); + }); + + it('FE-ADMIN-STOR-046: a done sync still renders Sync now alongside the done line — re-runnable without a reload', async () => { + const state = mirroredState(); + (state as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'done', done: 5, total: 5, copied: 5, skipped: 0, failed: 0, deleted: 0, startedAt: 1, finishedAt: 2 }, + ]; + let started = false; + await renderPanel(state); + server.use( + http.post('/api/admin/storage/backends/mirror/backfill', () => { + started = true; + return HttpResponse.json({ started: true }); + }), + ); + const row = backendRow('backups-local'); + expect(within(row).getByText(/Sync finished: 5 copied, 0 deleted, 0 failed/)).toBeInTheDocument(); + fireEvent.click(within(row).getByRole('button', { name: 'Sync now' })); + await waitFor(() => expect(started).toBe(true)); + }); + + it('FE-ADMIN-STOR-047: an errored sync also still renders Sync now alongside the error line', async () => { + const state = mirroredState(); + (state as StorageAdminState).backfills = [ + { backend: 'mirror', status: 'error', done: 2, total: 5, copied: 1, skipped: 0, failed: 1, deleted: 0, startedAt: 1, error: 'disk full' }, + ]; + await renderPanel(state); + const row = backendRow('backups-local'); + expect(within(row).getByText(/Sync failed: disk full/)).toBeInTheDocument(); + expect(within(row).getByRole('button', { name: 'Sync now' })).toBeInTheDocument(); + }); +}); diff --git a/client/src/components/Admin/storage/AdminStoragePanel.tsx b/client/src/components/Admin/storage/AdminStoragePanel.tsx new file mode 100644 index 000000000..d22abab2d --- /dev/null +++ b/client/src/components/Admin/storage/AdminStoragePanel.tsx @@ -0,0 +1,779 @@ +import React, { useEffect, useRef, useState } from 'react' +import { Activity, FolderTree, HardDrive } from 'lucide-react' +import { + STORAGE_CATEGORIES, + type StorageBackend, + type StorageCategory, + type StorageConfig, + type StorageMigrationStatus, + type StorageTestResponse, +} from '@trek/shared' +import { useTranslation } from '../../../i18n' +import { formatBytes } from '../../../utils/formatBytes' +import { relativeTime } from '../../../utils/relativeTime' +import ConfirmDialog from '../../shared/ConfirmDialog' +import CustomSelect from '../../shared/CustomSelect' +import { useToast } from '../../shared/Toast' +import Section from '../../Settings/Section' +import BackendForm, { type BackendFormMirrorProps } from './BackendForm' +import { + CACHE_CATEGORIES, + adoptedMirrorFor, + computeMigrationCandidates, + effectiveCategoryMap, + foldBackends, + mirrorProbeTargets, + primaryNameOf, + removeBackend, + removeBackendAndMirrors, + renameBackendRefs, + replicaCandidates, + replicaOfPrimaries, + settingsDocumentOf, + setMirrorTargets, + stripCategories, + upsertBackend, + usageByBackend, + type FoldedBackendRow, + type MigrationCandidate, +} from './storageModel' +import { useStorageAdmin } from './useStorageAdmin' + +/** Display-name mapper for joined category lists — the raw id renders only in the badge. */ +const categoryNames = (t: (key: string) => string, ids: readonly string[]): string => + ids.map((id) => t(`storage.category.${id}`)).join(', ') + +function TestResult({ result, failedLabel, okLabel }: { + result: StorageTestResponse | undefined + failedLabel: string + okLabel: string +}): React.ReactElement | null { + if (result === undefined) return null + return ( +
+

{result.ok ? okLabel : failedLabel}

+ {result.targets.map((target) => ( +

+ {target.ok ? '✓' : '✗'} {target.name} + {target.error ? ` — ${target.error}` : ''} +

+ ))} +
+ ) +} + +const LINK_BUTTON_STYLE: React.CSSProperties = { background: 'none', border: 'none', cursor: 'pointer' } + +export default function AdminStoragePanel(): React.ReactElement { + const { t, locale } = useTranslation() + const toast = useToast() + const admin = useStorageAdmin(t('common.error'), t('storage.saveConflict')) + const [editing, setEditing] = useState<{ + initial: StorageBackend | null + originalName: string | null + mirror: BackendFormMirrorProps + } | null>(null) + const [confirmRemove, setConfirmRemove] = useState<{ name: string; degenerate: boolean } | null>(null) + const [syncPrompt, setSyncPrompt] = useState(null) + // Open/closed only — never a snapshot of candidates. Storing the candidate + // array here would go stale the moment the operator edits a category while + // the dialog is open; the render below and moveAndSave both recompute + // computeMigrationCandidates(draft, state) fresh instead. + const [migratePromptOpen, setMigratePromptOpen] = useState(false) + const [migrationQueue, setMigrationQueue] = useState([]) + // Set by `save` right before it calls admin.save(): the pre-save mirror + // target count per row name. Consumed (and cleared) by the effect below the + // first time `admin.state` changes afterward — never touched otherwise, so + // unrelated state changes (the backfill poll included) are no-ops here. + const pendingPromptCheck = useRef | null>(null) + // Synchronous single-flight lock for the queue effect below: `admin.state` + // only reflects a just-started migration once startMigration's awaited + // refreshState() resolves, so `setMigrationQueue(rest)` re-firing the + // effect synchronously (still against the pre-POST `admin.state`) would + // otherwise dequeue and POST the next candidate before the server has + // confirmed the first — a ref (not state) so the guard is visible on that + // very next synchronous re-render, not just after a state-driven one. + const migrationStartInFlight = useRef(false) + + useEffect(() => { + if (!pendingPromptCheck.current || !admin.state) return + const before = pendingPromptCheck.current + pendingPromptCheck.current = null + const { rows: afterRows } = foldBackends(admin.state, settingsDocumentOf(admin.state)) + const grown = afterRows.find((r) => r.mirrorTargets.length > (before.get(r.name) ?? 0)) + if (grown) setSyncPrompt(grown.name) + }, [admin.state]) + + // Queued category migrations run strictly sequentially: once a slot opens + // (no migration currently running), dequeue the next candidate and start + // it. Depends on admin.state so a poll landing a terminal status re-fires + // this without any user interaction. The in-flight lock closes the race + // where setMigrationQueue(rest) re-fires this effect synchronously while + // admin.state still shows the pre-POST world — without it, the next + // candidate could dequeue and POST before the first is server-confirmed. + // Also waits out a running BACKFILL — the server's one-storage-job-at-a-time + // rule spans backfills and migrations alike, so starting while one runs + // would 409, and the queued candidate would be lost (no retry). + // + // A failed startMigration POST would otherwise strand `rest`: it was + // already dequeued into local state before the POST, so on failure nothing + // ever changes `migrationQueue` or `admin.state` again — the effect never + // re-fires and the remaining candidates sit invisibly forever, silently + // never migrated. On failure the whole remaining queue is explicitly + // cleared (never retried — the operator re-triggers via Save), named in a + // toast, and admin.state is refreshed so the panel reflects whatever the + // failed attempt's category ended up as server-side. + useEffect(() => { + if (migrationQueue.length === 0 || !admin.state) return + if (migrationStartInFlight.current) return + if (admin.state.migrations.some((m) => m.status === 'running')) return + if (admin.state.backfills.some((b) => b.status === 'running')) return + const [next, ...rest] = migrationQueue + migrationStartInFlight.current = true + setMigrationQueue(rest) + void admin.startMigration(next!.category, next!.toWire).then((error) => { + migrationStartInFlight.current = false + if (error) { + toast.error(error) + if (rest.length > 0) { + toast.error(t('storage.migrate.queueDropped', { categories: categoryNames(t, rest.map((c) => c.category)) })) + setMigrationQueue([]) + } + void admin.refreshState().catch(() => {}) + } + }) + }, [migrationQueue, admin.state]) + + if (admin.loading) { + return

{t('storage.loading')}

+ } + if (!admin.state || !admin.draft) { + return ( +
+ {admin.loadError || t('common.error')} +
+ ) + } + const { state, draft } = admin + + // Pure/cheap — recomputed every render so the migrate-prompt dialog (below) + // never shows a stale candidate set while it's open (fix: migration prompt + // staleness). moveAndSave recomputes independently at confirm time rather + // than reading this render-scoped value, since the click handler and the + // render that produced it are not guaranteed to be the same one. + const migrationCandidates = computeMigrationCandidates(draft, state) + + // Duplicate pre-check needs EVERY wire name (hidden mirror names included); + // mirror-target candidates are the visible primaries only. + const backendNames = [...new Set([...state.backends.map((b) => b.name), ...draft.backends.map((b) => b.name)])] + const { rows, degenerate } = foldBackends(state, draft) + const effective = effectiveCategoryMap(state, draft) + const usageSums = usageByBackend(state, draft) + + const startEdit = (row: FoldedBackendRow) => { + // Editing a built-in creates a settings override row bearing its name — + // the first-class relocation path (merge-by-name). + setEditing({ + initial: row.backend, + originalName: row.name, + mirror: { + candidates: replicaCandidates(rows, row.name, row.mirrorTargets), + initialTargets: row.mirrorTargets, + }, + }) + } + + const commitBackend = (backend: StorageBackend, mirrorTargets?: string[]) => { + const renamedFrom = + editing?.originalName && editing.originalName !== backend.name ? editing.originalName : null + // Widened to StorageConfig: these edit helpers are version-blind (they + // return plain StorageConfig), and admin.setDraft re-attaches the + // draft's own `version` regardless of what shape it's handed. + let next: StorageConfig | null = draft + if (renamedFrom) next = renameBackendRefs(removeBackend(next, renamedFrom), renamedFrom, backend.name) + next = upsertBackend(next, backend) + if (mirrorTargets !== undefined) next = setMirrorTargets(state, next, backend.name, mirrorTargets) + admin.setDraft(next) + setEditing(null) + } + + const removeMessage = (name: string, isDegenerate: boolean): string => { + const row = rows.find((r) => r.name === name) + const assigned = isDegenerate + ? degenerate.find((d) => d.backend.name === name)?.categories ?? [] + : row?.categories ?? [] + const usedAsReplicaBy = isDegenerate ? [] : replicaOfPrimaries(draft, name) + return [ + t('storage.remove.body', { name }), + assigned.length > 0 ? t('storage.remove.stillAssigned', { categories: categoryNames(t, assigned) }) : '', + usedAsReplicaBy.length > 0 ? t('storage.remove.usedAsReplicaBy', { primaries: usedAsReplicaBy.join(', ') }) : '', + ] + .filter(Boolean) + .join(' ') + } + + const setCategory = (category: StorageCategory, primaryName: string) => { + // Picking a mirrored primary routes through its adopted mirror silently. + const target = adoptedMirrorFor(draft, primaryName)?.name ?? primaryName + // The admin state's category record is exhaustive by schema contract. + const stateEntry = state.categories[category]! + const categories = { ...draft.categories } + if (stateEntry.source === 'default' && target === stateEntry.backend) { + delete categories[category] // back to the default → no longer settings-owned + } else { + categories[category] = target + } + admin.setDraft({ ...draft, categories }) + } + + // Snapshot the LAST-CONFIRMED (pre-save `state`'s own fold, not the + // in-progress `draft`) mirror target counts; the effect above compares + // them against the fresh post-save state to detect newly-added + // replicas. Folding `draft` here would already include the unsaved + // edit and mask the very growth this is meant to detect. + const snapshotMirrorTargets = (): Map => { + const { rows: beforeRows } = foldBackends(state, settingsDocumentOf(state)) + return new Map(beforeRows.map((r) => [r.name, r.mirrorTargets.length])) + } + + const doPlainSave = async () => { + pendingPromptCheck.current = snapshotMirrorTargets() + if (await admin.save()) toast.success(t('storage.saved')) + else pendingPromptCheck.current = null + } + + // Recomputes the candidate set at confirm time — never trusts whatever was + // true when the dialog opened. The operator can edit categories (or a + // background poll can land a fresher usage scan) while the dialog is on + // screen; the recomputed set is what actually drives both stripCategories + // (what gets reverted from the PUT body) and the queue (what gets POSTed), + // so the two can never diverge from what the dialog most recently showed. + const moveAndSave = async () => { + const candidates = computeMigrationCandidates(draft, state) + pendingPromptCheck.current = snapshotMirrorTargets() + const ok = await admin.save(stripCategories(draft, state, candidates.map((c) => c.category))) + if (ok) { + setMigrationQueue(candidates) + toast.success(t('storage.saved')) + } else { + pendingPromptCheck.current = null + } + setMigratePromptOpen(false) + } + + const routeOnlySave = async () => { + setMigratePromptOpen(false) + await doPlainSave() + } + + const save = async () => { + if (migrationCandidates.length > 0) { + setMigratePromptOpen(true) + return + } + await doPlainSave() + } + + const handleCancelMigration = async (m: StorageMigrationStatus) => { + const error = await admin.cancelMigration(m.category) + if (error) toast.error(error) + } + + const testResultFor = (key: string): StorageTestResponse | 'running' | undefined => admin.testResults[key] + + const handleRefreshStats = async () => { + const error = await admin.refreshStats() + if (error) toast.error(error) + } + + const handleStartBackfill = async (row: FoldedBackendRow) => { + if (!row.mirrorName) return + setSyncPrompt((prev) => (prev === row.name ? null : prev)) + const error = await admin.startBackfill(row.mirrorName) + if (error) toast.error(error) + } + + const handleCancelBackfill = async (row: FoldedBackendRow) => { + if (!row.mirrorName) return + const error = await admin.cancelBackfill(row.mirrorName) + if (error) toast.error(error) + } + + return ( +
+ {state.configError && ( +
+ {t('storage.configError.banner', { error: state.configError })} +
+ )} +
+ {state.health.replicaFailures.length === 0 ? ( +

{t('storage.health.allClear')}

+ ) : ( +
    + {state.health.replicaFailures.map((failure) => ( +
  • + {t('storage.health.failureLine', { + op: failure.op, + key: failure.key, + backend: failure.backend, + error: failure.error, + })} + · {relativeTime(failure.at, locale)} +
    • + ))} +
+ )} + {state.seedFilePresent &&

{t('storage.health.seedFile')}

} +
+ +
+

{t('storage.description')}

+

+ {state.usage + ? t('storage.usage.computed', { age: relativeTime(state.usage.computedAt, locale) }) + : t('storage.usage.never')} + {' '} + +

+
+ {rows.map((row) => { + const resultKey = row.mirrorName ?? row.name + const result = testResultFor(resultKey) + // row.mirrorName only exists when foldBackends adopted a draft mirror + // for this row, so the draft lookup below cannot miss. + const testCandidate = row.mirrorName + ? draft.backends.find((b) => b.name === row.mirrorName)! + : row.backend + const rowUsage = usageSums?.[row.name] + const backfill = row.mirrorName ? state.backfills.find((b) => b.backend === row.mirrorName) : undefined + return ( +
+
+ {row.name} + + {t(`storage.type.${row.type}`)} + + + {t(`storage.source.${row.source}`)} + + + + {row.source !== 'env' && ( + + )} + {row.source === 'settings' && ( + + )} +
+

+ {row.categories.length > 0 + ? t('storage.backends.usedBy', { categories: categoryNames(t, row.categories) }) + : t('storage.backends.unused')} +

+ {rowUsage && ( +

+ {t('storage.usage.line', { objects: String(rowUsage.objects), size: formatBytes(rowUsage.bytes) })} + {row.name === 'uploads-local' && state.usage!.legacyPhotos.objects > 0 + ? ` (${t('storage.usage.legacyNote')})` + : ''} +

+ )} + {row.mirrorTargets.length > 0 && ( +

+ {t('storage.mirror.mirroredTo', { targets: row.mirrorTargets.join(', ') })} +

+ )} + {row.replicaOf.length > 0 && ( +

+ {t('storage.mirror.replicaOf', { primaries: row.replicaOf.join(', ') })} +

+ )} + {row.source === 'env' &&

{t('storage.backends.envReadOnly')}

} + {result === 'running' ? ( +

{t('storage.test.running')}

+ ) : ( + + )} + {row.mirrorTargets.length > 0 && ( +
+ {backfill?.status === 'running' ? ( + <> +

+ {t('storage.sync.running', { done: String(backfill.done), total: String(backfill.total) })} +

+

+ {t('storage.sync.counts', { + copied: String(backfill.copied), + skipped: String(backfill.skipped), + failed: String(backfill.failed), + })} +

+ + + ) : ( + <> + {/* Terminal statuses render ALONGSIDE the button (not instead-of) — + a completed/cancelled/errored backfill must stay re-runnable + without a page reload. */} + {backfill?.status === 'done' && ( +

+ {t('storage.sync.done', { + copied: String(backfill.copied), + deleted: String(backfill.deleted), + failed: String(backfill.failed), + })} +

+ )} + {backfill?.status === 'cancelled' && ( +

{t('storage.sync.cancelled')}

+ )} + {backfill?.status === 'error' && ( +

+ {t('storage.sync.error', { error: backfill.error ?? '' })} +

+ )} + {syncPrompt !== row.name && ( + + )} + + )} +
+ )} + {syncPrompt === row.name && ( +
+

{t('storage.sync.prompt')}

+
+ + +
+
+ )} +
+ ) + })} + + {degenerate.map(({ backend, reason }) => { + const result = testResultFor(backend.name) + const primary = backend.type === 'mirror' ? backend.options.primary : '' + return ( +
+
+ {backend.name} + + {t('storage.type.mirror')} + + + + +
+

{t(`storage.mirror.degenerate.${reason}`, { primary })}

+ {result === 'running' ? ( +

{t('storage.test.running')}

+ ) : ( + + )} +
+ ) + })} +
+ + {state.migrations.length > 0 && ( +
+ {state.migrations.map((m) => ( +
+

{t(`storage.category.${m.category}`)}

+ {m.status === 'running' ? ( + <> +

+ {t('storage.migrate.running', { + category: t(`storage.category.${m.category}`), + done: String(m.done), + total: String(m.total), + })} +

+ + + ) : m.status === 'done' ? ( + <> +

+ {t('storage.migrate.done', { copied: String(m.copied), skipped: String(m.skipped) })} +

+ {m.failed > 0 && ( +

+ {t('storage.migrate.doneFailures', { failed: String(m.failed) })} +

+ )} + {m.reclaimable && ( +

+ {t('storage.migrate.reclaimable', { + objects: String(m.reclaimable.objects), + size: formatBytes(m.reclaimable.bytes), + from: m.from, + })} +

+ )} + + ) : m.status === 'failed' ? ( +

+ {t('storage.migrate.failed', { error: m.error ?? '' })} +

+ ) : ( +

{t('storage.migrate.cancelled')}

+ )} +
+ ))} +
+ )} + + {editing ? ( + setEditing(null)} + /> + ) : ( + + )} +
+ +
+
+ {STORAGE_CATEGORIES.map((category) => { + // The admin state's category record is exhaustive by schema contract. + const stateEntry = state.categories[category]! + const selectedPrimary = primaryNameOf(state, draft, effective[category]) + const changed = selectedPrimary !== primaryNameOf(state, draft, stateEntry.backend) + const viaMirror = effective[category] !== selectedPrimary + return ( +
+ +

{t(`storage.categoryDesc.${category}`)}

+ {state.usage?.categories[category] && ( +

+ {t('storage.usage.line', { + objects: String(state.usage.categories[category]!.objects), + size: formatBytes(state.usage.categories[category]!.bytes), + })} +

+ )} + setCategory(category, String(value))} + options={rows.map((row) => ({ + value: row.name, + label: + stateEntry.source === 'default' && row.name === stateEntry.backend + ? `${row.name} (${t('storage.categories.default')})` + : row.name, + }))} + size="sm" + style={{ maxWidth: 320 }} + /> + {changed && ( +

+ {t('storage.categories.reassignWarning')} +

+ )} + {viaMirror && CACHE_CATEGORIES.includes(category) && ( +

+ {t('storage.mirror.cacheWarning')} +

+ )} +
+ ) + })} +
+
+ +
+ + {admin.dirty && {t('storage.unsaved')}} +
+ {migrationQueue.length > 0 && ( +

+ {t('storage.migrate.queued', { categories: categoryNames(t, migrationQueue.map((c) => c.category)) })} +

+ )} + {migratePromptOpen && ( +
+

{t('storage.migrate.promptTitle')}

+ {migrationCandidates.map((c) => ( +

+ {c.objects === null + ? t('storage.migrate.promptLineUnknown', { + category: t(`storage.category.${c.category}`), + from: c.from, + to: c.to, + }) + : t('storage.migrate.promptLine', { + category: t(`storage.category.${c.category}`), + objects: String(c.objects), + size: formatBytes(c.bytes ?? 0), + from: c.from, + to: c.to, + })} +

+ ))} +
+ + + +
+
+ )} + {admin.saveError && ( +
+

{admin.saveError}

+ {/* A 409 leaves the draft pinned to a version the server has moved + past; "save again" can never clear it, so the banner offers the + one action that can — see useStorageAdmin.discardDraft. */} + {admin.saveConflict && ( + + )} +
+ )} + + setConfirmRemove(null)} + onConfirm={() => { + if (confirmRemove) { + admin.setDraft( + confirmRemove.degenerate + ? removeBackend(draft, confirmRemove.name) + : removeBackendAndMirrors(state, draft, confirmRemove.name), + ) + } + setConfirmRemove(null) + }} + title={t('storage.remove.title')} + message={confirmRemove ? removeMessage(confirmRemove.name, confirmRemove.degenerate) : ''} + confirmLabel={t('storage.remove.title')} + /> +
+ ) +} diff --git a/client/src/components/Admin/storage/BackendForm.test.tsx b/client/src/components/Admin/storage/BackendForm.test.tsx new file mode 100644 index 000000000..2bb69afee --- /dev/null +++ b/client/src/components/Admin/storage/BackendForm.test.tsx @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from 'vitest'; +import { MASKED_SETTING_VALUE, type StorageBackend } from '@trek/shared'; +import { fireEvent, render, screen } from '../../../../tests/helpers/render'; +import BackendForm from './BackendForm'; + +const NAMES = ['uploads-local', 'backups-local', 'off-box']; + +type FormProps = Parameters[0]; + +function renderForm(overrides: Partial = {}) { + const onCommit = vi.fn(); + const onCancel = vi.fn(); + render( + , + ); + return { onCommit, onCancel }; +} + +const S3_INITIAL: StorageBackend = { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', + secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000, + }, +}; + +describe('BackendForm', () => { + it('FE-ADMIN-STORF-001: local renders the root path field from the registry', () => { + renderForm(); + expect(screen.getByLabelText(/Root directory/)).toBeInTheDocument(); + expect(screen.queryByLabelText(/Bucket/)).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STORF-002: s3 renders every field by kind — password secret, number inputs, defaultValue placeholders', () => { + renderForm({ initial: S3_INITIAL }); + expect(screen.getByLabelText(/Endpoint URL/)).toHaveAttribute('type', 'text'); + expect(screen.getByLabelText(/Secret access key/)).toHaveAttribute('type', 'password'); + expect(screen.getByLabelText(/Region/)).toHaveAttribute('placeholder', 'us-east-1'); + expect(screen.getByLabelText(/Retries/)).toHaveAttribute('type', 'number'); + expect(screen.getByLabelText(/Retries/)).toHaveAttribute('placeholder', '1'); + expect(screen.getByLabelText(/Timeout \(ms\)/)).toHaveAttribute('placeholder', '30000'); + }); + + it('FE-ADMIN-STORF-004: the mask echoes back through commit untouched (no-op by contract)', () => { + const { onCommit } = renderForm({ initial: S3_INITIAL }); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + expect(onCommit).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'off-box', + type: 's3', + options: expect.objectContaining({ secretAccessKey: MASKED_SETTING_VALUE, retries: 1, timeoutMs: 30000 }), + }), + ); + }); + + it('FE-ADMIN-STORF-005: a typed plaintext secret keeps Apply offered (no encryption-key gate)', () => { + renderForm({ initial: S3_INITIAL }); + expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText(/Secret access key/), { target: { value: 'sk-new' } }); + expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument(); + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + }); + + it('FE-ADMIN-STORF-006: Apply stays disabled until name and every required field are filled', () => { + const { onCommit } = renderForm(); + const apply = screen.getByRole('button', { name: 'Apply' }); + expect(apply).toBeDisabled(); + fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'nas' } }); + expect(apply).toBeDisabled(); // root still empty + fireEvent.change(screen.getByLabelText(/Root directory/), { target: { value: '/mnt/nas' } }); + expect(apply).toBeEnabled(); + fireEvent.click(apply); + expect(onCommit).toHaveBeenCalledWith({ name: 'nas', type: 'local', options: { root: '/mnt/nas' } }); + }); + + it('FE-ADMIN-STORF-007: a duplicate name on a NEW backend warns and blocks Apply', () => { + renderForm(); + fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'off-box' } }); + fireEvent.change(screen.getByLabelText(/Root directory/), { target: { value: '/x' } }); + expect(screen.getByText(/A backend named off-box already exists/)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled(); + }); + + it('FE-ADMIN-STORF-008: empty optional fields are omitted so the shared schema defaults apply', () => { + const { onCommit } = renderForm(); + // Switch to s3 via the type select (CustomSelect renders options into a portal). + fireEvent.click(screen.getByText('Local')); + fireEvent.click(screen.getByText('S3')); + fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'fresh' } }); + fireEvent.change(screen.getByLabelText(/Endpoint URL/), { target: { value: 'http://127.0.0.1:9000' } }); + fireEvent.change(screen.getByLabelText(/Bucket/), { target: { value: 'trek' } }); + fireEvent.change(screen.getByLabelText(/Access key ID/), { target: { value: 'ak' } }); + fireEvent.change(screen.getByLabelText(/Secret access key/), { target: { value: 'sk' } }); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + const committed = onCommit.mock.calls[0]![0] as StorageBackend; + expect(committed.options).toEqual({ + endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: 'sk', + }); + expect(committed.options).not.toHaveProperty('region'); + }); + + it('FE-ADMIN-STORF-009: renaming onto another existing backend warns and blocks Apply', () => { + renderForm({ initial: S3_INITIAL }); + expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled(); + fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'uploads-local' } }); + expect(screen.getByText(/A backend named uploads-local already exists/)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled(); + // Keeping the original name stays allowed. + fireEvent.change(screen.getByLabelText(/Name/), { target: { value: 'off-box' } }); + expect(screen.getByRole('button', { name: 'Apply' })).toBeEnabled(); + }); + + it('FE-ADMIN-STORF-010: the type select no longer offers Mirror', () => { + renderForm(); + fireEvent.click(screen.getByText('Local')); // open the type select + expect(screen.queryByText('Mirror')).not.toBeInTheDocument(); + expect(screen.getByText('S3')).toBeInTheDocument(); + }); + + it('FE-ADMIN-STORF-011: the mirror composer renders candidates minus self and commits targets', () => { + const { onCommit } = renderForm({ + initial: S3_INITIAL, + mirror: { candidates: ['uploads-local', 'backups-local', 'off-box'], initialTargets: ['backups-local'] }, + }); + expect(screen.getByText('Mirror targets')).toBeInTheDocument(); + const boxes = screen.getAllByRole('checkbox'); + expect(boxes).toHaveLength(2); // off-box (self) excluded + expect(screen.getByRole('checkbox', { name: 'backups-local' })).toBeChecked(); + expect(screen.getByRole('note').textContent).toContain('slows every upload'); + fireEvent.click(screen.getByRole('checkbox', { name: 'uploads-local' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + expect(onCommit).toHaveBeenCalledWith( + expect.objectContaining({ name: 'off-box', type: 's3' }), + ['backups-local', 'uploads-local'], + ); + }); + + it('FE-ADMIN-STORF-012: unchecking every target commits an empty array (dissolve) and hides the latency note', () => { + const { onCommit } = renderForm({ + initial: S3_INITIAL, + mirror: { candidates: ['uploads-local'], initialTargets: ['uploads-local'] }, + }); + fireEvent.click(screen.getByRole('checkbox', { name: 'uploads-local' })); + expect(screen.queryByRole('note')).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + expect(onCommit).toHaveBeenCalledWith(expect.objectContaining({ name: 'off-box' }), []); + }); +}); diff --git a/client/src/components/Admin/storage/BackendForm.tsx b/client/src/components/Admin/storage/BackendForm.tsx new file mode 100644 index 000000000..c2293798c --- /dev/null +++ b/client/src/components/Admin/storage/BackendForm.tsx @@ -0,0 +1,264 @@ +import React, { useState } from 'react' +import { + STORAGE_BACKEND_TYPES, + STORAGE_BACKEND_TYPE_IDS, + type StorageBackend, + type StorageBackendFieldDef, + type StorageBackendTypeId, +} from '@trek/shared' +import { useTranslation } from '../../../i18n' +import CustomSelect from '../../shared/CustomSelect' + +type FieldValues = Record + +function valuesOf(backend: StorageBackend | null): FieldValues { + if (!backend) return {} + const values: FieldValues = {} + for (const [key, value] of Object.entries(backend.options)) { + values[key] = Array.isArray(value) ? value : String(value) + } + return values +} + +export interface BackendFormMirrorProps { + /** Selectable replica targets (the form excludes its own name at render). */ + candidates: string[] + initialTargets: string[] +} + +interface BackendFormProps { + /** null = new backend */ + initial: StorageBackend | null + /** Every defined backend name — mirror-target options and the duplicate pre-check. */ + backendNames: string[] + /** + * Present on non-mirror backends: renders the Mirror-targets composer + * (replicas-on-primary — panel-supplied chrome, never a registry field). + * When present, onCommit's second argument is always an array. + */ + mirror?: BackendFormMirrorProps + onCommit: (backend: StorageBackend, mirrorTargets?: string[]) => void + onCancel: () => void +} + +const LABEL_CLASS = 'block text-sm font-medium mb-1.5 text-content-secondary' +const INPUT_CLASS = + 'mt-1.5 w-full px-3 py-2 border rounded-lg text-sm border-edge bg-surface-card text-content' + +/** + * Renders whatever STORAGE_BACKEND_TYPES declares, by field kind. The raw + * `mirror` type is hidden from the type select — mirrors are composed via the + * Mirror-targets prop block and synthesized by the panels (the ref-kind + * renderers below stay for future registry types). + */ +export default function BackendForm({ + initial, + backendNames, + mirror, + onCommit, + onCancel, +}: BackendFormProps): React.ReactElement { + const { t } = useTranslation() + const [type, setType] = useState(initial?.type ?? 'local') + const [name, setName] = useState(initial?.name ?? '') + const [values, setValues] = useState(() => valuesOf(initial)) + const [targets, setTargets] = useState(mirror?.initialTargets ?? []) + + const fields = STORAGE_BACKEND_TYPES[type].fields as readonly StorageBackendFieldDef[] + const refOptions = backendNames.filter((candidate) => candidate !== name.trim()) + const setValue = (key: string, value: string | string[]) => + setValues((prev) => ({ ...prev, [key]: value })) + + const filled = (field: StorageBackendFieldDef): boolean => { + const value = values[field.key] + if (field.kind === 'backend-ref-list') return Array.isArray(value) && value.length > 0 + return typeof value === 'string' && value.trim() !== '' + } + const duplicate = name.trim() !== (initial?.name ?? '') && backendNames.includes(name.trim()) + const canApply = + name.trim() !== '' && !duplicate && fields.every((f) => !f.required || filled(f)) + + const apply = () => { + const options: Record = {} + for (const field of fields) { + const value = values[field.key] + if (field.kind === 'backend-ref-list') { + options[field.key] = Array.isArray(value) ? value : [] + continue + } + const text = typeof value === 'string' ? value : '' + if (text === '' && !field.required) continue // omitted → the shared schema default applies + options[field.key] = field.kind === 'number' ? Number(text) : text + } + // The options were built from the same field defs the schema is generated + // from — this is the same sanctioned cast storageModel.asWireBackend makes. + const payload = { name: name.trim(), type, options } as StorageBackend + // Arity matters: the landed tests pin single-argument calls when no mirror + // block is supplied (toHaveBeenCalledWith treats a trailing undefined as a + // mismatch), so the second argument exists only when the composer does. + if (mirror) onCommit(payload, targets) + else onCommit(payload) + } + + const renderField = (field: StorageBackendFieldDef): React.ReactElement => { + const value = values[field.key] + if (field.kind === 'backend-ref') { + return ( +
+ {t(field.labelKey)} + setValue(field.key, String(next))} + options={refOptions.map((candidate) => ({ value: candidate, label: candidate }))} + placeholder={t(field.labelKey)} + size="sm" + /> +
+ ) + } + if (field.kind === 'backend-ref-list') { + const selected = Array.isArray(value) ? value : [] + return ( +
+ {t(field.labelKey)} +
+ {refOptions.map((candidate) => ( + + ))} +
+
+ ) + } + const inputType = field.kind === 'secret' ? 'password' : field.kind === 'number' ? 'number' : 'text' + return ( + + ) + } + + return ( +
+

+ {initial ? t('storage.form.editTitle') : t('storage.form.addTitle')} +

+ + + {duplicate && ( +

+ {t('storage.form.duplicateName', { name: name.trim() })} +

+ )} + +
+ {t('storage.form.type')} + { + setType(next as StorageBackendTypeId) + setValues({}) // a different type has different fields + }} + options={STORAGE_BACKEND_TYPE_IDS.filter((id) => id !== 'mirror').map((id) => ({ + value: id, + label: t(`storage.type.${id}`), + }))} + size="sm" + disabled={initial !== null} + /> +
+ + {fields.map(renderField)} + + {mirror && ( +
+ {t('storage.mirror.targets')} +

{t('storage.mirror.targetsHelp')}

+
+ {mirror.candidates + .filter((candidate) => candidate !== name.trim()) + .map((candidate) => ( + + ))} +
+ {targets.length > 0 && ( +

+ {t('storage.mirror.latencyNote')} +

+ )} +
+ )} + +
+ + +
+
+ ) +} diff --git a/client/src/components/Admin/storage/storageModel.test.ts b/client/src/components/Admin/storage/storageModel.test.ts new file mode 100644 index 000000000..1d400ee21 --- /dev/null +++ b/client/src/components/Admin/storage/storageModel.test.ts @@ -0,0 +1,407 @@ +import { describe, expect, it } from 'vitest'; +import { + MASKED_SETTING_VALUE, + STORAGE_CATEGORIES, + type StorageAdminState, + type StorageBackend, + type StorageCategory, + type StorageConfig, +} from '@trek/shared'; +import { + categoriesPointingAt, + computeMigrationCandidates, + foldBackends, + mirrorProbeTargets, + mirrorsReferencing, + primaryNameOf, + removeBackend, + removeBackendAndMirrors, + renameBackendRefs, + replicaCandidates, + replicaOfPrimaries, + setMirrorTargets, + settingsDocumentOf, + stripCategories, + upsertBackend, + usageByBackend, +} from './storageModel'; + +const STATE: StorageAdminState = { + backends: [ + { name: 'uploads-local', type: 'local', source: 'built-in', options: { root: '/data/uploads' }, categories: ['files'] }, + { name: 'place-photos-local', type: 'local', source: 'env', options: { root: '/photos' }, categories: ['places'] }, + { + name: 'off-box', + type: 's3', + source: 'settings', + options: { + endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', + secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000, + }, + categories: ['covers'], + }, + ], + categories: { + files: { backend: 'uploads-local', source: 'default' }, + journey: { backend: 'uploads-local', source: 'default' }, + covers: { backend: 'off-box', source: 'settings' }, + avatars: { backend: 'uploads-local', source: 'default' }, + places: { backend: 'place-photos-local', source: 'default' }, + 'photos-google': { backend: 'place-photos-local', source: 'default' }, + 'photos-trek': { backend: 'uploads-local', source: 'default' }, + backups: { backend: 'backups-local', source: 'default' }, + }, + health: { replicaFailures: [] }, + seedFilePresent: false, + usage: null, + backfills: [], + migrations: [], + version: 5, + configError: null, +}; + +describe('settingsDocumentOf', () => { + it('FE-ADMIN-STORM-001: carries ONLY settings-sourced backends and categories (the PUT contract)', () => { + const doc = settingsDocumentOf(STATE); + expect(doc.backends.map((b) => b.name)).toEqual(['off-box']); + expect(doc.categories).toEqual({ covers: 'off-box' }); + }); + + it('FE-ADMIN-STORM-001b: carries version from state (audit #7 — the PUT body\'s optimistic-concurrency check)', () => { + expect(settingsDocumentOf(STATE).version).toBe(5); + expect(settingsDocumentOf({ ...STATE, version: 12 }).version).toBe(12); + }); +}); + +describe('remove pre-check helpers', () => { + const draft: StorageConfig = { + backends: [ + { name: 'nas', type: 'local', options: { root: '/mnt/nas' } }, + { name: 'mir', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ], + categories: { covers: 'nas', backups: 'mir' }, + }; + + it('FE-ADMIN-STORM-002: lists categories pointing at a backend', () => { + expect(categoriesPointingAt(draft, 'nas')).toEqual(['covers']); + expect(categoriesPointingAt(draft, 'unused')).toEqual([]); + }); + + it('FE-ADMIN-STORM-003: lists mirrors referencing a backend (primary or replica)', () => { + expect(mirrorsReferencing(draft, 'nas')).toEqual(['mir']); + expect(mirrorsReferencing(draft, 'backups-local')).toEqual(['mir']); + expect(mirrorsReferencing(draft, 'unused')).toEqual([]); + }); +}); + + +describe('draft edits', () => { + const draft: StorageConfig = { + backends: [{ name: 'nas', type: 'local', options: { root: '/mnt/nas' } }], + categories: {}, + }; + + it('FE-ADMIN-STORM-005: upsert replaces by name in place, appends new ones, never mutates', () => { + const edited = upsertBackend(draft, { name: 'nas', type: 'local', options: { root: '/mnt/nas2' } }); + expect(edited.backends).toEqual([{ name: 'nas', type: 'local', options: { root: '/mnt/nas2' } }]); + const added = upsertBackend(draft, { name: 'other', type: 'local', options: { root: '/o' } }); + expect(added.backends.map((b) => b.name)).toEqual(['nas', 'other']); + expect(draft.backends).toHaveLength(1); // untouched + }); + + it('FE-ADMIN-STORM-006: removeBackend filters by name', () => { + expect(removeBackend(draft, 'nas').backends).toEqual([]); + expect(removeBackend(draft, 'ghost').backends).toHaveLength(1); + }); +}); + +// Hoisted to module scope (not just the mirror-fold describe below) so the +// usageByBackend suite can reuse the same mirrored fixture the brief's test +// snippet assumes is in scope. +const MIRRORED_STATE: StorageAdminState = { + ...STATE, + backends: [ + ...STATE.backends, + { + name: 'mirror', type: 'mirror', source: 'settings', + options: { primary: 'backups-local', replicas: ['off-box'] }, categories: ['backups'], + }, + { name: 'backups-local', type: 'local', source: 'built-in', options: { root: '/data/backups' }, categories: [] }, + ], + categories: { ...STATE.categories, backups: { backend: 'mirror', source: 'settings' } }, +}; +const mirroredDraft = (): StorageConfig => settingsDocumentOf(MIRRORED_STATE); + +describe('mirror fold/synthesize (replicas-on-primary)', () => { + it('FE-ADMIN-STORM-007: foldBackends hides the mirror, decorates primary and replica, unions categories', () => { + const { rows, degenerate } = foldBackends(MIRRORED_STATE, mirroredDraft()); + expect(degenerate).toEqual([]); + expect(rows.map((r) => r.name)).not.toContain('mirror'); + const primary = rows.find((r) => r.name === 'backups-local')!; + expect(primary.mirrorTargets).toEqual(['off-box']); + expect(primary.mirrorName).toBe('mirror'); + expect(primary.categories).toContain('backups'); // via the mirror + const replica = rows.find((r) => r.name === 'off-box')!; + expect(replica.replicaOf).toEqual(['backups-local']); + expect(replica.categories).toContain('covers'); // its own direct assignment survives + }); + + it('FE-ADMIN-STORM-008: setMirrorTargets synthesizes a mirror and reroutes ALL effective categories of the primary, defaults included', () => { + const draft = settingsDocumentOf(STATE); // no mirror yet; backups is default → backups-local + const next = setMirrorTargets(STATE, draft, 'backups-local', ['off-box']); + const mirror = next.backends.find((b) => b.type === 'mirror')!; + expect(mirror.name).toBe('backups-local-mirror'); + expect(mirror.options).toEqual({ primary: 'backups-local', replicas: ['off-box'] }); + expect(next.categories.backups).toBe('backups-local-mirror'); // default-sourced category rewritten + expect(next.categories.covers).toBe('off-box'); // categories of OTHER backends untouched + }); + + it('FE-ADMIN-STORM-009: setMirrorTargets adopts a foreign-named mirror in place', () => { + const next = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', ['off-box', 'uploads-local']); + const mirrors = next.backends.filter((b) => b.type === 'mirror'); + expect(mirrors).toHaveLength(1); + expect(mirrors[0]!.name).toBe('mirror'); // adopted, not renamed + expect(mirrors[0]!.options).toEqual({ primary: 'backups-local', replicas: ['off-box', 'uploads-local'] }); + }); + + it('FE-ADMIN-STORM-010: empty targets dissolve the mirror and revert default-sourced categories', () => { + // Build the synthesized state locally: default backups routed via a fresh mirror… + const withMirror = setMirrorTargets(STATE, settingsDocumentOf(STATE), 'backups-local', ['off-box']); + // …then dissolve it: the backups entry must DROP (state says its default is backups-local). + const dissolved = setMirrorTargets(STATE, withMirror, 'backups-local', []); + expect(dissolved.backends.some((b) => b.type === 'mirror')).toBe(false); + expect(dissolved.categories).not.toHaveProperty('backups'); + // Against MIRRORED_STATE (backups already settings-sourced at the mirror), dissolve re-points instead of dropping. + const repointed = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', []); + expect(repointed.categories.backups).toBe('backups-local'); + }); + + it('FE-ADMIN-STORM-011: degenerate mirrors are classified, never hidden', () => { + const draft: StorageConfig = { + backends: [ + { name: 'off-box', type: 's3', options: STATE.backends[2]!.options } as StorageBackend, + { name: 'm1', type: 'mirror', options: { primary: 'uploads-local', replicas: ['off-box'] } }, + { name: 'm2', type: 'mirror', options: { primary: 'uploads-local', replicas: ['off-box'] } }, + { name: 'm-env', type: 'mirror', options: { primary: 'place-photos-local', replicas: ['off-box'] } }, + { name: 'm-ghost', type: 'mirror', options: { primary: 'nope', replicas: [] } }, + ], + categories: {}, + }; + const { rows, degenerate } = foldBackends(STATE, draft); + expect(rows.find((r) => r.name === 'uploads-local')!.mirrorName).toBe('m1'); // first wins + expect(degenerate.map((d) => [d.backend.name, d.reason])).toEqual([ + ['m2', 'duplicate-mirror'], + ['m-env', 'env-primary'], + ['m-ghost', 'missing-primary'], + ]); + }); + + it('FE-ADMIN-STORM-012: renameBackendRefs rewrites mirror primaries, replicas, and category entries', () => { + const draft: StorageConfig = { + backends: [ + { name: 'nas', type: 'local', options: { root: '/mnt/nas' } }, + { name: 'm', type: 'mirror', options: { primary: 'nas', replicas: ['nas'] } }, + ], + categories: { covers: 'nas' }, + }; + const renamed = renameBackendRefs(draft, 'nas', 'nas2'); + const mirror = renamed.backends.find((b) => b.type === 'mirror')!; + expect(mirror.options).toEqual({ primary: 'nas2', replicas: ['nas2'] }); + expect(renamed.categories.covers).toBe('nas2'); + expect(renamed.backends[0]!.name).toBe('nas'); // the backend row itself is the caller's job + }); + + it('FE-ADMIN-STORM-013: removeBackendAndMirrors cascades; replicaOfPrimaries phrases in primary names', () => { + const draft = mirroredDraft(); + expect(replicaOfPrimaries(draft, 'off-box')).toEqual(['backups-local']); + const removed = removeBackendAndMirrors(MIRRORED_STATE, draft, 'backups-local'); + expect(removed.backends.some((b) => b.name === 'mirror')).toBe(false); + expect(removed.backends.some((b) => b.name === 'backups-local')).toBe(false); + }); + + it('FE-ADMIN-STORM-015: removing a replica strips it from remaining mirrors; a mirror stripped empty dissolves', () => { + // Single-replica mirror: stripping off-box empties it → dissolve + re-point + // (backups is settings-sourced at the mirror in MIRRORED_STATE, so it re-points, not drops). + const dissolved = removeBackendAndMirrors(MIRRORED_STATE, mirroredDraft(), 'off-box'); + expect(dissolved.backends.some((b) => b.name === 'off-box')).toBe(false); + expect(dissolved.backends.some((b) => b.type === 'mirror')).toBe(false); + expect(dissolved.categories.backups).toBe('backups-local'); + // Multi-replica mirror: stripping one target keeps the mirror alive with the rest. + const twoTargets = setMirrorTargets(MIRRORED_STATE, mirroredDraft(), 'backups-local', ['off-box', 'uploads-local']); + const survived = removeBackendAndMirrors(MIRRORED_STATE, twoTargets, 'off-box'); + const mirror = survived.backends.find((b) => b.type === 'mirror')!; + expect(mirror.options).toEqual({ primary: 'backups-local', replicas: ['uploads-local'] }); + expect(survived.categories.backups).toBe('mirror'); // still routed via the surviving mirror + }); + + it('FE-ADMIN-STORM-014: primaryNameOf maps mirror names (draft or state) to their primary; others pass through', () => { + expect(primaryNameOf(MIRRORED_STATE, mirroredDraft(), 'mirror')).toBe('backups-local'); + expect(primaryNameOf(MIRRORED_STATE, { backends: [], categories: {} }, 'mirror')).toBe('backups-local'); // state fallback + expect(primaryNameOf(STATE, settingsDocumentOf(STATE), 'uploads-local')).toBe('uploads-local'); + }); +}); + +describe('replicaCandidates', () => { + it('FE-ADMIN-STORM-020: offers only rows that serve no category, never the row being edited', () => { + const { rows } = foldBackends(MIRRORED_STATE, mirroredDraft()); + // uploads-local/place-photos-local serve categories directly; + // backups-local serves backups through the mirror it is the primary of; + // off-box carries a direct `covers` assignment. Nothing is offerable. + expect(replicaCandidates(rows, 'backups-local')).toEqual([]); + }); + + it('FE-ADMIN-STORM-021: keeps an already-selected target listed so it can be unchecked', () => { + const { rows } = foldBackends(MIRRORED_STATE, mirroredDraft()); + expect(replicaCandidates(rows, 'backups-local', ['off-box'])).toEqual(['off-box']); + }); + + it('FE-ADMIN-STORM-022: an unassigned row is offered', () => { + const state: StorageAdminState = { + ...MIRRORED_STATE, + backends: [ + ...MIRRORED_STATE.backends, + { name: 'cold-store', type: 'local', source: 'settings', options: { root: '/data/cold' }, categories: [] }, + ], + }; + const { rows } = foldBackends(state, settingsDocumentOf(state)); + expect(replicaCandidates(rows, 'backups-local')).toEqual(['cold-store']); + }); +}); + +describe('usageByBackend', () => { + it('FE-ADMIN-STORM-016: sums categories onto their effective backend rows, mirrors onto the primary, legacy onto uploads-local', () => { + const usage = { + computedAt: 1, + categories: Object.fromEntries( + STORAGE_CATEGORIES.map((c) => [c, { objects: 1, bytes: 10 }]), + ) as Record, + legacyPhotos: { objects: 5, bytes: 50 }, + }; + const state = { ...MIRRORED_STATE, usage }; + const sums = usageByBackend(state, mirroredDraft())!; + // backups routes via the mirror → attributed to backups-local (the primary row). + expect(sums['backups-local']).toEqual({ objects: 1, bytes: 10 }); + // uploads-local serves 4 direct categories (files, journey, avatars, photos-trek — + // this fixture's `places` and `photos-google` default to the env-sourced + // place-photos-local backend, not uploads-local) + legacy photos. + expect(sums['uploads-local']).toEqual({ objects: 4 + 5, bytes: 40 + 50 }); + // places + photos-google both default to place-photos-local. + expect(sums['place-photos-local']).toEqual({ objects: 2, bytes: 20 }); + // covers routes to off-box directly. + expect(sums['off-box']).toEqual({ objects: 1, bytes: 10 }); + }); + + it('FE-ADMIN-STORM-017: null usage yields null', () => { + expect(usageByBackend({ ...STATE, usage: null }, settingsDocumentOf(STATE))).toBeNull(); + }); +}); + +describe('computeMigrationCandidates', () => { + it('FE-ADMIN-STORM-018: lists changed categories with usage counts; zero-object ones are excluded; null usage keeps them with null counts', () => { + const state: StorageAdminState = { + ...STATE, + usage: { + computedAt: 1, + categories: { + files: { objects: 3, bytes: 300 }, + journey: { objects: 0, bytes: 0 }, + } as Record, + legacyPhotos: { objects: 0, bytes: 0 }, + }, + }; + const base = settingsDocumentOf(state); + const draft: StorageConfig = { + ...base, + categories: { ...base.categories, files: 'off-box', journey: 'off-box', avatars: 'off-box' }, + }; + const candidates = computeMigrationCandidates(draft, state); + expect(candidates).toEqual([ + { category: 'files', from: 'uploads-local', to: 'off-box', toWire: 'off-box', objects: 3, bytes: 300 }, + { category: 'avatars', from: 'uploads-local', to: 'off-box', toWire: 'off-box', objects: null, bytes: null }, + ]); + }); + + it('FE-ADMIN-STORM-018b: reassigning onto a mirrored primary yields toWire = the mirror wire name while `to` stays the display primary', () => { + const state: StorageAdminState = { + ...STATE, + backends: [ + ...STATE.backends, + { name: 'nas', type: 'local', source: 'settings', options: { root: '/mnt/nas' }, categories: [] }, + { + name: 'off-box-mirror', + type: 'mirror', + source: 'settings', + options: { primary: 'off-box', replicas: ['nas'] }, + categories: [], + }, + ], + usage: { + computedAt: 1, + categories: { files: { objects: 3, bytes: 300 } } as Record, + legacyPhotos: { objects: 0, bytes: 0 }, + }, + }; + const base = settingsDocumentOf(state); + // Reassign 'files' to the mirrored primary's WIRE name — the same value + // setCategory() writes into the draft when the operator picks a mirrored + // primary in the UI (adoptedMirrorFor(draft, 'off-box')?.name). + const draft: StorageConfig = { ...base, categories: { ...base.categories, files: 'off-box-mirror' } }; + const candidates = computeMigrationCandidates(draft, state); + expect(candidates).toEqual([ + { category: 'files', from: 'uploads-local', to: 'off-box', toWire: 'off-box-mirror', objects: 3, bytes: 300 }, + ]); + }); + + it('FE-ADMIN-STORM-019: stripCategories restores the saved assignment for exactly the named categories', () => { + const base = settingsDocumentOf(STATE); + const draft: StorageConfig = { + ...base, + categories: { ...base.categories, files: 'off-box', covers: 'uploads-local' }, + }; + + const strippedFiles = stripCategories(draft, STATE, ['files']); + expect(strippedFiles.categories.files).toBeUndefined(); // default-sourced: saved value is "no override" + expect(strippedFiles.categories.covers).toBe('uploads-local'); // untouched — not in the strip list + + const strippedCovers = stripCategories(draft, STATE, ['covers']); + expect(strippedCovers.categories.covers).toBe('off-box'); // settings-sourced saved value restored explicitly + expect(strippedCovers.categories.files).toBe('off-box'); // untouched — not in the strip list + }); +}); + +describe('mirrorProbeTargets', () => { + it('FE-ADMIN-STORM-020: a non-mirror candidate passes through unchanged', () => { + const local: StorageBackend = { name: 'uploads-local', type: 'local', options: { root: '/data/uploads' } }; + expect(mirrorProbeTargets(settingsDocumentOf(STATE), STATE, local)).toEqual([local]); + }); + + it('FE-ADMIN-STORM-021: expands primary + replicas — draft override wins per name, falling back to state for names the draft never touched', () => { + const mirror: StorageBackend = { + name: 'mirror', type: 'mirror', + options: { primary: 'uploads-local', replicas: ['off-box'] }, + }; + // The draft carries an EDITED off-box (unsaved endpoint change) but never + // touches uploads-local (a built-in, never draft-owned). + const editedOffBox = { + name: 'off-box', type: 's3', + options: { ...(STATE.backends[2]!.options as Record), endpoint: 'http://edited:9000' }, + } as StorageBackend; + const draft: StorageConfig = { backends: [editedOffBox, mirror], categories: {} }; + + const targets = mirrorProbeTargets(draft, STATE, mirror); + expect(targets.map((t) => t.name)).toEqual(['uploads-local', 'off-box']); + // Primary: no draft entry — resolved from state (the live built-in). + expect(targets[0]).toEqual({ name: 'uploads-local', type: 'local', options: { root: '/data/uploads' } }); + // Replica: draft override wins — the unsaved edit, not the saved options. + expect((targets[1]!.options as Record).endpoint).toBe('http://edited:9000'); + }); + + it('FE-ADMIN-STORM-022: a name resolving to neither draft nor state is dropped, not thrown', () => { + const mirror: StorageBackend = { + name: 'mirror', type: 'mirror', + options: { primary: 'uploads-local', replicas: ['ghost'] }, + }; + const targets = mirrorProbeTargets(settingsDocumentOf(STATE), STATE, mirror); + expect(targets.map((t) => t.name)).toEqual(['uploads-local']); + }); +}); diff --git a/client/src/components/Admin/storage/storageModel.ts b/client/src/components/Admin/storage/storageModel.ts new file mode 100644 index 000000000..e7796edb7 --- /dev/null +++ b/client/src/components/Admin/storage/storageModel.ts @@ -0,0 +1,440 @@ +import { + STORAGE_CATEGORIES, + type StorageAdminState, + type StorageBackend, + type StorageCategory, + type StorageConfig, + type StorageConfigPut, +} from '@trek/shared' + +/** One entry of the effective world (GET state), as the panels render it. */ +export type StateBackend = StorageAdminState['backends'][number] + +/** + * The settings-owned document the PUT carries: settings-sourced backends and + * settings-sourced category assignments ONLY — built-ins and env backends are + * never in the body unless the operator overrides one by name (in which case + * the override already reports source 'settings'). Carries `version` from + * the state it was built at (audit #7 — optimistic concurrency): the hook + * re-attaches this same version on every subsequent local edit, so a PUT + * always submits the version the operator's form was loaded at, not + * whatever the server currently holds. + */ +export function settingsDocumentOf(state: StorageAdminState): StorageConfigPut { + return { + backends: state.backends.filter((b) => b.source === 'settings').map(asWireBackend), + categories: Object.fromEntries( + Object.entries(state.categories) + .filter(([, entry]) => entry.source === 'settings') + .map(([category, entry]) => [category, entry.backend]), + ) as StorageConfig['categories'], + version: state.version, + } +} + +/** + * State options are already schema-shaped (the wire contract guarantees it; + * secret fields carry the mask, which the server unmasks by name on PUT/test) — + * the cast re-attaches the discriminated-union type the loose state record dropped. + */ +export function asWireBackend(backend: Pick): StorageBackend { + return { name: backend.name, type: backend.type, options: backend.options } as StorageBackend +} + +/** Draft categories pointing at a backend name (the friendly remove pre-check). */ +export function categoriesPointingAt(draft: StorageConfig, name: string): StorageCategory[] { + return (Object.entries(draft.categories) as Array<[StorageCategory, string]>) + .filter(([, backend]) => backend === name) + .map(([category]) => category) +} + +/** Draft mirrors referencing a backend name as primary or replica (remove pre-check). */ +export function mirrorsReferencing(draft: StorageConfig, name: string): string[] { + return draft.backends + .filter((b) => b.type === 'mirror' && (b.options.primary === name || b.options.replicas.includes(name))) + .map((b) => b.name) +} + +/** Replace by name in place, or append. Never mutates. */ +export function upsertBackend(draft: StorageConfig, backend: StorageBackend): StorageConfig { + const exists = draft.backends.some((b) => b.name === backend.name) + return { + ...draft, + backends: exists + ? draft.backends.map((b) => (b.name === backend.name ? backend : b)) + : [...draft.backends, backend], + } +} + +export function removeBackend(draft: StorageConfig, name: string): StorageConfig { + return { ...draft, backends: draft.backends.filter((b) => b.name !== name) } +} + +// ── Mirror fold/synthesize (replicas-on-primary spec) ───────────────────────── +// The draft stays the wire document; folding is a view rule, synthesis a set +// of pure draft operations. Mirror NAMES never reach the UI — every helper +// that renders phrases mirrors as their primary. + +type MirrorBackend = Extract +const isMirror = (b: StorageBackend): b is MirrorBackend => b.type === 'mirror' + +export interface FoldedBackendRow { + name: string + type: 'local' | 's3' + source: 'built-in' | 'env' | 'settings' + backend: StorageBackend + /** Direct assignments ∪ assignments routed via the adopted mirror. */ + categories: StorageCategory[] + /** Adopted mirror's replicas — [] when unmirrored. */ + mirrorTargets: string[] + /** Adopted mirror's wire name — needed by draft ops, hidden from the UI. */ + mirrorName: string | null + /** Primaries whose adopted mirror lists this backend as a replica. */ + replicaOf: string[] +} + +export interface DegenerateMirror { + backend: StorageBackend + reason: 'duplicate-mirror' | 'env-primary' | 'missing-primary' + categories: StorageCategory[] +} + +/** + * Re-fetchable content — replicating it is usually wasteful. photos-google is + * the Google photo cache (place-photo-cache.service.ts); places is mostly + * provider-derived imagery. + */ +export const CACHE_CATEGORIES: readonly StorageCategory[] = ['photos-google', 'places'] + +/** The configurable categories resolved draft-over-state (draft entries win; defaults fill the rest). */ +export function effectiveCategoryMap(state: StorageAdminState, draft: StorageConfig): Record { + const map = {} as Record + for (const category of STORAGE_CATEGORIES) { + // The admin state's category record is exhaustive by schema contract. + map[category] = draft.categories[category] ?? state.categories[category]!.backend + } + return map +} + +/** First draft mirror wrapping the named primary — the one the UI manages. */ +export function adoptedMirrorFor(draft: StorageConfig, primaryName: string): MirrorBackend | undefined { + return draft.backends.filter(isMirror).find((m) => m.options.primary === primaryName) +} + +/** + * Client-side mirror expansion for the test probe (audit fix — the server's + * `testBackend` resolves a mirror candidate's primary/replicas against its + * OWN live snapshot by name, so probing a mirrored row with unsaved draft + * edits to those targets would silently test the SAVED options instead of + * what's on the form). Draft overrides win per name, falling back to state + * (built-in/env); a name that resolves to neither is dropped — that + * primary/replica no longer exists, and the row already renders as + * degenerate in that case. + */ +export function mirrorProbeTargets( + draft: StorageConfig, + state: StorageAdminState, + mirror: StorageBackend, +): StorageBackend[] { + if (mirror.type !== 'mirror') return [mirror] + const names = [mirror.options.primary, ...mirror.options.replicas] + return names + .map((name): StorageBackend | null => { + const draftMatch = draft.backends.find((b) => b.name === name) + if (draftMatch) return draftMatch + const stateMatch = state.backends.find((b) => b.name === name) + return stateMatch ? asWireBackend(stateMatch) : null + }) + .filter((b): b is StorageBackend => b !== null) +} + +export function foldBackends( + state: StorageAdminState, + draft: StorageConfig, +): { rows: FoldedBackendRow[]; degenerate: DegenerateMirror[] } { + const draftNames = draft.backends.map((b) => b.name) + const effective = effectiveCategoryMap(state, draft) + const categoriesOf = (name: string): StorageCategory[] => + STORAGE_CATEGORIES.filter((category) => effective[category] === name) + + // Base (non-mirror) rows, slice-2 precedence: built-ins/env from the state + // unless a draft row overrides the name; settings rows from the draft. + const base: Array> = [] + for (const b of state.backends) { + if (b.type === 'mirror') continue // mirrors fold below; the draft owns them + if (draftNames.includes(b.name)) continue + if (b.source === 'settings') continue // removed from the draft, pending save + base.push({ name: b.name, type: b.type, source: b.source, backend: asWireBackend(b) }) + } + for (const b of draft.backends) { + if (isMirror(b)) continue + base.push({ name: b.name, type: b.type, source: 'settings', backend: b }) + } + + // Adoption: the first mirror per editable primary; the rest are degenerate — + // rendered unfolded with a reason, never silently dropped. + const adopted = new Map() + const degenerate: DegenerateMirror[] = [] + for (const mirror of draft.backends.filter(isMirror)) { + const primary = base.find((row) => row.name === mirror.options.primary) + if (!primary) degenerate.push({ backend: mirror, reason: 'missing-primary', categories: categoriesOf(mirror.name) }) + else if (primary.source === 'env') degenerate.push({ backend: mirror, reason: 'env-primary', categories: categoriesOf(mirror.name) }) + else if (adopted.has(primary.name)) degenerate.push({ backend: mirror, reason: 'duplicate-mirror', categories: categoriesOf(mirror.name) }) + else adopted.set(primary.name, mirror) + } + + const rows = base.map((row): FoldedBackendRow => { + const mirror = adopted.get(row.name) + return { + ...row, + categories: [...categoriesOf(row.name), ...(mirror ? categoriesOf(mirror.name) : [])], + mirrorTargets: mirror ? [...mirror.options.replicas] : [], + mirrorName: mirror?.name ?? null, + replicaOf: [...adopted.values()] + .filter((m) => m.options.replicas.includes(row.name)) + .map((m) => m.options.primary), + } + }) + return { rows, degenerate } +} + +/** + * The rows offerable as mirror replicas (the "Mirror targets" picker). + * + * A backend that already serves a category — directly, or as the primary of a + * mirror a category routes to, which is exactly what `row.categories` unions — + * can never also be a replica: the mirror's sync sweep makes each replica + * MATCH the primary, deleting whatever the primary doesn't hold, and `backups` + * sweeps the replica's entire root. The server refuses such a config outright + * (storage-registry.service.ts, assertNoSharedReplicas); the picker simply + * never offers one, so the operator can't walk into the refusal. + * + * Targets already selected on this mirror stay listed regardless — they must + * remain visible to be unchecked (and a config that predates this rule is + * repaired here, not hidden). + */ +export function replicaCandidates( + rows: FoldedBackendRow[], + selfName: string | null, + currentTargets: readonly string[] = [], +): string[] { + return rows + .filter((row) => row.name !== selfName) + .filter((row) => row.categories.length === 0 || currentTargets.includes(row.name)) + .map((row) => row.name) +} + +function uniqueMirrorName(state: StorageAdminState, draft: StorageConfig, primaryName: string): string { + const taken = new Set([...draft.backends.map((b) => b.name), ...state.backends.map((b) => b.name)]) + const base = `${primaryName}-mirror` + if (!taken.has(base)) return base + for (let n = 2; ; n++) { + if (!taken.has(`${base}-${n}`)) return `${base}-${n}` + } +} + +/** + * targets non-empty: update the adopted mirror in place (or synthesize one) + * and route every category that effectively resolves to the primary — + * default-sourced included — through it. targets empty: dissolve, re-pointing + * the mirror's categories at the primary and dropping entries that revert to + * their built-in default. + */ +export function setMirrorTargets( + state: StorageAdminState, + draft: StorageConfig, + primaryName: string, + targets: string[], +): StorageConfig { + const adopted = adoptedMirrorFor(draft, primaryName) + + if (targets.length === 0) { + if (!adopted) return draft + const categories = { ...draft.categories } + for (const category of STORAGE_CATEGORIES) { + if (categories[category] !== adopted.name) continue + const stateEntry = state.categories[category]! // exhaustive by schema contract + if (stateEntry.source === 'default' && stateEntry.backend === primaryName) delete categories[category] + else categories[category] = primaryName + } + return { backends: draft.backends.filter((b) => b.name !== adopted.name), categories } + } + + const effective = effectiveCategoryMap(state, draft) + const mirror: StorageBackend = { + name: adopted?.name ?? uniqueMirrorName(state, draft, primaryName), + type: 'mirror', + options: { primary: primaryName, replicas: [...targets] }, + } + const backends = adopted + ? draft.backends.map((b) => (b.name === adopted.name ? mirror : b)) + : [...draft.backends, mirror] + const categories = { ...draft.categories } + for (const category of STORAGE_CATEGORIES) { + if (effective[category] === primaryName) categories[category] = mirror.name + } + return { backends, categories } +} + +/** Rewrite every reference to a renamed backend: mirror primaries, replicas, category entries. */ +export function renameBackendRefs(draft: StorageConfig, oldName: string, newName: string): StorageConfig { + return { + backends: draft.backends.map((b) => { + if (!isMirror(b)) return b + return { + ...b, + options: { + primary: b.options.primary === oldName ? newName : b.options.primary, + replicas: b.options.replicas.map((r) => (r === oldName ? newName : r)), + }, + } + }), + categories: Object.fromEntries( + Object.entries(draft.categories).map(([category, backend]) => [category, backend === oldName ? newName : backend]), + ) as StorageConfig['categories'], + } +} + +/** + * Remove a backend, every mirror wrapping it as primary, and every replica + * reference to it in the remaining mirrors — a mirror stripped of its last + * replica dissolves (with the same category re-pointing as an explicit + * dissolve), so the draft never carries a phantom reference the composer + * cannot render or repair. + */ +export function removeBackendAndMirrors( + state: StorageAdminState, + draft: StorageConfig, + name: string, +): StorageConfig { + const stripped = draft.backends + .filter((b) => b.name !== name && !(isMirror(b) && b.options.primary === name)) + .map((b) => + isMirror(b) && b.options.replicas.includes(name) + ? { ...b, options: { ...b.options, replicas: b.options.replicas.filter((r) => r !== name) } } + : b, + ) + let next: StorageConfig = { ...draft, backends: stripped } + for (const mirror of stripped.filter(isMirror)) { + if (mirror.options.replicas.length > 0) continue + if (adoptedMirrorFor(next, mirror.options.primary)?.name === mirror.name) { + next = setMirrorTargets(state, next, mirror.options.primary, []) + } else { + // A degenerate duplicate stripped empty: drop it and re-point its categories. + const categories = { ...next.categories } + for (const category of STORAGE_CATEGORIES) { + if (categories[category] === mirror.name) categories[category] = mirror.options.primary + } + next = { backends: next.backends.filter((b) => b.name !== mirror.name), categories } + } + } + return next +} + +/** Primaries whose mirrors use the named backend as a replica — pre-check copy stays in primary names. */ +export function replicaOfPrimaries(draft: StorageConfig, name: string): string[] { + return draft.backends + .filter(isMirror) + .filter((m) => m.options.replicas.includes(name)) + .map((m) => m.options.primary) +} + +/** Map a mirror name (draft first, then state) to its primary; anything else passes through. */ +export function primaryNameOf(state: StorageAdminState, draft: StorageConfig, backendName: string): string { + const draftMirror = draft.backends.filter(isMirror).find((m) => m.name === backendName) + if (draftMirror) return draftMirror.options.primary + const stateMirror = state.backends.find((b) => b.name === backendName && b.type === 'mirror') + if (stateMirror) return String((stateMirror.options as Record).primary) + return backendName +} + +/** + * Usage sums per VISIBLE backend row: each category's numbers land on the row + * that serves it (a mirror-routed category on the mirror's PRIMARY row), and + * the legacy photos directory lands on uploads-local — the only backend the + * served-legacy `photos` category ever resolves to. + */ +export function usageByBackend( + state: StorageAdminState, + draft: StorageConfig, +): Record | null { + if (!state.usage) return null + const sums: Record = {} + const add = (row: string, entry: { objects: number; bytes: number }) => { + const current = (sums[row] ??= { objects: 0, bytes: 0 }) + current.objects += entry.objects + current.bytes += entry.bytes + } + const effective = effectiveCategoryMap(state, draft) + for (const category of STORAGE_CATEGORIES) { + const entry = state.usage.categories[category] + if (entry) add(primaryNameOf(state, draft, effective[category]), entry) + } + // The served-legacy `photos` directory always resolves to uploads-local — + // hardcoded, not derived, because it is not one of the configurable categories. + add(primaryNameOf(state, draft, 'uploads-local'), state.usage.legacyPhotos) + return sums +} + +// ── Category migration (copy → flip → delta sweep) ──────────────────────────── + +/** One reassigned category the operator might want to move existing objects for. */ +export interface MigrationCandidate { + category: StorageCategory + from: string + /** Display primary — never a hidden mirror name; used by the prompt lines and comparisons. */ + to: string + /** + * The RAW effective draft backend name for this category — possibly a + * mirror's wire name when `to` is a mirrored primary. The migration POST + * must carry this, not `to`: posting the bare primary would silently drop + * replication for a category that was just routed onto a mirror. + */ + toWire: string + /** null when usage was never scanned — the prompt still fires, with an unknown-size line. */ + objects: number | null + bytes: number | null +} + +/** + * Every category whose effective (draft-over-state) PRIMARY backend differs + * from what's currently saved — normalized through `primaryNameOf` so simply + * wrapping (or unwrapping) a mirror around the same primary never counts as + * a candidate; mirroring is the existing sync/backfill flow, not a category + * migration. Zero-object categories (usage present, 0 objects) are excluded — + * nothing to move; null usage (never scanned) keeps the category as a + * candidate with null counts, per spec ("unknown still prompts"). + */ +export function computeMigrationCandidates(draft: StorageConfig, state: StorageAdminState): MigrationCandidate[] { + const effective = effectiveCategoryMap(state, draft) + const candidates: MigrationCandidate[] = [] + for (const category of STORAGE_CATEGORIES) { + const from = primaryNameOf(state, draft, state.categories[category]!.backend) // exhaustive by schema contract + const toWire = effective[category] + const to = primaryNameOf(state, draft, toWire) + if (from === to) continue + const usage = state.usage?.categories[category] + candidates.push({ category, from, to, toWire, objects: usage ? usage.objects : null, bytes: usage ? usage.bytes : null }) + } + return candidates.filter((c) => c.objects !== 0) +} + +/** + * Revert exactly the named categories in `draft` back to what's currently + * saved in `saved` — same default/settings convention as the rest of this + * file (drop the override when the saved entry is default-sourced, write the + * explicit backend name otherwise). Every other draft edit is untouched. + */ +export function stripCategories( + draft: StorageConfig, + saved: StorageAdminState, + categories: StorageCategory[], +): StorageConfig { + const next = { ...draft.categories } + for (const category of categories) { + const savedEntry = saved.categories[category]! // exhaustive by schema contract + if (savedEntry.source === 'default') delete next[category] + else next[category] = savedEntry.backend + } + return { ...draft, categories: next } +} diff --git a/client/src/components/Admin/storage/useStorageAdmin.ts b/client/src/components/Admin/storage/useStorageAdmin.ts new file mode 100644 index 000000000..880122493 --- /dev/null +++ b/client/src/components/Admin/storage/useStorageAdmin.ts @@ -0,0 +1,349 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import type { StorageAdminState, StorageBackend, StorageConfig, StorageConfigPut, StorageTestResponse } from '@trek/shared' +import { adminApi } from '../../../api/client' +import { getApiErrorMessage, type ApiError } from '../../../types' +import { settingsDocumentOf } from './storageModel' + +export type StorageTestResults = Record + +/** 50ms under vitest so poll tests run on real timers (MSW + fake timers don't mix). */ +export const BACKFILL_POLL_MS = import.meta.env.MODE === 'test' ? 50 : 5000 + +/** True for a 409 response — the version this draft was built at is no longer current (audit #7). */ +function isConflictError(err: unknown): boolean { + return typeof err === 'object' && err !== null && 'response' in err && (err as ApiError).response?.status === 409 +} + +export interface StorageAdmin { + state: StorageAdminState | null + /** The settings-owned document (the PUT body), with local edits layered in. Carries the version the draft was built at. */ + draft: StorageConfigPut | null + dirty: boolean + loading: boolean + loadError: string | null + /** Server 400s/409s land here VERBATIM (400) or as the distinct conflict copy (409) — long registry messages outlive a toast. */ + saveError: string | null + /** True while `saveError` is the 409 copy — the panels render the discard-and-reload escape next to it. */ + saveConflict: boolean + saving: boolean + testResults: StorageTestResults + setDraft: (next: StorageConfig) => void + /** PUTs `overrideDraft ?? draft` — an override lets a caller strip categories from the wire body without staging them (and marking dirty) via setDraft. */ + save: (overrideDraft?: StorageConfig) => Promise + test: (backend: StorageBackend) => Promise + /** + * Probe each target individually (client-expanded mirror primary + + * replicas — see `mirrorProbeTargets`) and merge the results into one + * `StorageTestResponse` under `key` (the row's mirror wire name). Never + * sends the mirror stub itself — each call probes a concrete backend. + */ + testMirror: (key: string, targets: StorageBackend[]) => Promise + /** null on success; the verbatim server error otherwise. */ + startBackfill: (mirrorName: string) => Promise + cancelBackfill: (mirrorName: string) => Promise + startMigration: (category: string, to: string) => Promise + cancelMigration: (category: string) => Promise + refreshStats: () => Promise + /** Re-GETs admin state (poll-safe — see the implementation comment below). Exposed for the panel's failed-queue recovery path. */ + refreshState: () => Promise + /** + * The 409's only exit: throw the local edits away and reload the saved + * settings (fresh GET → fresh draft → fresh version), clearing dirty and + * saveError. See the implementation comment for why refreshState can't do + * this itself. + */ + discardDraft: () => Promise +} + +/** + * Shared by the desktop and phone storage panels (DefaultUserSettingsTab + * model: self-contained, adminApi directly, deliberately no offline core — + * hoster-level config is online-only). + */ +export function useStorageAdmin(genericError: string, conflictError: string): StorageAdmin { + const [state, setState] = useState(null) + const [draft, setDraftState] = useState(null) + const [dirty, setDirty] = useState(false) + const [loading, setLoading] = useState(true) + const [loadError, setLoadError] = useState(null) + const [saveError, setSaveError] = useState(null) + const [saveConflict, setSaveConflict] = useState(false) + const [saving, setSaving] = useState(false) + const [testResults, setTestResults] = useState({}) + + // Read imperatively from the poll effect/refreshState so a stale closure + // never re-derives the draft out from under an in-flight edit. + const dirtyRef = useRef(dirty) + useEffect(() => { + dirtyRef.current = dirty + }, [dirty]) + + // Read imperatively at refreshState's resolve time so a poll GET that was + // in flight when a save started never applies after the save's own state. + const savingRef = useRef(saving) + useEffect(() => { + savingRef.current = saving + }, [saving]) + + // Bumped by every deliberate state application (initial load, save + // success). A refreshState() call captures the seq before its GET and + // drops the response if the seq moved on — i.e. a save applied its own + // fresher state — while that GET was in flight. + const stateSeq = useRef(0) + + const applyState = useCallback((next: StorageAdminState) => { + stateSeq.current += 1 + setState(next) + setDraftState(settingsDocumentOf(next)) + setDirty(false) + setSaveError(null) + setSaveConflict(false) + }, []) + + useEffect(() => { + let cancelled = false + adminApi + .getStorage() + .then((data) => { + if (!cancelled) applyState(data) + }) + .catch((err: unknown) => { + if (!cancelled) setLoadError(getApiErrorMessage(err, genericError)) + }) + .finally(() => { + if (!cancelled) setLoading(false) + }) + return () => { + cancelled = true + } + }, [applyState, genericError]) + + // Re-attaches the CURRENT draft's version onto whatever shape the caller + // hands in (the storageModel edit helpers are version-blind — they operate + // on plain StorageConfig — so this is the one place that keeps `version` + // pinned to what the operator's form was loaded at, no matter how many + // local edits run in between). + const setDraft = useCallback((next: StorageConfig) => { + setDraftState((prev) => ({ ...next, version: prev?.version ?? 0 })) + setDirty(true) + }, []) + + // Poll-safe refresh: replaces `state` unconditionally, but only re-derives + // `draft` when the operator has no unsaved edits in flight — never touches + // `dirty`/`saveError`, so a running poll cannot clobber a dirty draft or + // mask a pending save error. + // + // Race guard: a poll GET issued before a save's PUT resolves can resolve + // AFTER the save applied its own (fresher) response, overwriting it with + // the pre-save world. Capture the state-application sequence number before + // the await and drop the response if it moved on, or if a save is + // in-flight/just landed (read via a ref so it's current at resolve time). + // + // Declared before `save` (which calls it on a 409) — a hook's `const`s + // must exist before another callback's dependency array references them. + const refreshState = useCallback(async (): Promise => { + const seq = stateSeq.current + const next = await adminApi.getStorage() + if (stateSeq.current !== seq || savingRef.current) return + setState(next) + if (!dirtyRef.current) setDraftState(settingsDocumentOf(next)) + }, []) + + const save = useCallback( + async (overrideDraft?: StorageConfig): Promise => { + if (!draft) return false + // Always the DRAFT's own version, even for an overrideDraft (a variant + // of the same draft with categories stripped, per stripCategories) — + // both share the same version basis: what the operator's form was + // loaded/refreshed at, never re-read from `state` here (that would + // defeat the whole check — see setDraft's re-attach comment above). + const body: StorageConfigPut = { ...(overrideDraft ?? draft), version: draft.version } + setSaving(true) + setSaveError(null) + setSaveConflict(false) + try { + // The response is the fresh effective world, never the request echo. + applyState(await adminApi.updateStorage(body)) + return true + } catch (err: unknown) { + if (isConflictError(err)) { + // Something else (another admin's save, or a category migration's + // flip) wrote since this draft's version was captured. Refresh + // `state` so the operator sees the fresh world, but the dirty + // draft itself is left untouched for review (dirty is never + // cleared here) — refreshState() already respects dirtyRef. + setSaveError(conflictError) + setSaveConflict(true) + void refreshState().catch(() => {}) + } else { + setSaveError(getApiErrorMessage(err, genericError)) + } + return false + } finally { + setSaving(false) + } + }, + [draft, applyState, genericError, conflictError, refreshState], + ) + + /** + * Conflict recovery (final-review fix): after a 409 the dirty draft keeps + * the version it was built at. refreshState deliberately leaves a dirty + * draft's version alone — bumping it there would resurrect audit #7, where + * the background poll floats a mid-edit draft past a migration flip it + * never carried — so nothing else can ever move it, and every retry 409s + * again. The operator's way out is to say so explicitly: discard the local + * edits, reload the saved settings, and start over from the fresh version. + * A fresh GET (not the possibly-dropped post-409 refresh) guarantees that + * version is current, and applyState clears dirty/saveError with it. + */ + const discardDraft = useCallback(async (): Promise => { + try { + applyState(await adminApi.getStorage()) + } catch (err: unknown) { + setSaveError(getApiErrorMessage(err, genericError)) + setSaveConflict(false) + } + }, [applyState, genericError]) + + const test = useCallback( + async (backend: StorageBackend): Promise => { + setTestResults((prev) => ({ ...prev, [backend.name]: 'running' })) + try { + const result = await adminApi.testStorageBackend(backend) + setTestResults((prev) => ({ ...prev, [backend.name]: result })) + } catch (err: unknown) { + const error = getApiErrorMessage(err, genericError) + setTestResults((prev) => ({ + ...prev, + [backend.name]: { ok: false, targets: [{ name: backend.name, ok: false, error }] }, + })) + } + }, + [genericError], + ) + + const testMirror = useCallback( + async (key: string, targets: StorageBackend[]): Promise => { + setTestResults((prev) => ({ ...prev, [key]: 'running' })) + const settled = await Promise.all( + targets.map(async (target) => { + try { + const result = await adminApi.testStorageBackend(target) + return result.targets[0] ?? { name: target.name, ok: result.ok } + } catch (err: unknown) { + return { name: target.name, ok: false, error: getApiErrorMessage(err, genericError) } + } + }), + ) + setTestResults((prev) => ({ ...prev, [key]: { ok: settled.every((r) => r.ok), targets: settled } })) + }, + [genericError], + ) + + // While any backfill or category migration is running, poll GET state so + // progress/counts advance without the operator refreshing the page. + useEffect(() => { + if ( + !state?.backfills.some((b) => b.status === 'running') && + !state?.migrations.some((m) => m.status === 'running') + ) + return + const iv = setInterval(() => { + // A transient poll failure is retried on the next tick — it must never + // surface as a toast or interrupt the operator's in-progress edit. + void refreshState().catch(() => {}) + }, BACKFILL_POLL_MS) + return () => clearInterval(iv) + }, [state, refreshState]) + + const startBackfill = useCallback( + async (mirrorName: string): Promise => { + try { + await adminApi.startStorageBackfill(mirrorName) + void refreshState().catch(() => {}) + return null + } catch (err: unknown) { + return getApiErrorMessage(err, genericError) + } + }, + [refreshState, genericError], + ) + + const cancelBackfill = useCallback( + async (mirrorName: string): Promise => { + try { + await adminApi.cancelStorageBackfill(mirrorName) + void refreshState().catch(() => {}) + return null + } catch (err: unknown) { + return getApiErrorMessage(err, genericError) + } + }, + [refreshState, genericError], + ) + + const startMigration = useCallback( + async (category: string, to: string): Promise => { + try { + await adminApi.startStorageMigration(category, to) + // Awaited (unlike cancelMigration/startBackfill/cancelBackfill): the + // queue effect's in-flight lock releases as soon as this promise + // resolves, so admin.state must already reflect the just-started + // migration by then — otherwise the next queued candidate can + // dequeue and POST before the server has confirmed this one. + await refreshState().catch(() => {}) + return null + } catch (err: unknown) { + return getApiErrorMessage(err, genericError) + } + }, + [refreshState, genericError], + ) + + const cancelMigration = useCallback( + async (category: string): Promise => { + try { + await adminApi.cancelStorageMigration(category) + void refreshState().catch(() => {}) + return null + } catch (err: unknown) { + return getApiErrorMessage(err, genericError) + } + }, + [refreshState, genericError], + ) + + const refreshStats = useCallback(async (): Promise => { + try { + const usage = await adminApi.refreshStorageStats() + setState((prev) => (prev ? { ...prev, usage } : prev)) + return null + } catch (err: unknown) { + return getApiErrorMessage(err, genericError) + } + }, [genericError]) + + return { + state, + draft, + dirty, + loading, + loadError, + saveError, + saveConflict, + saving, + testResults, + setDraft, + save, + test, + testMirror, + startBackfill, + cancelBackfill, + startMigration, + cancelMigration, + refreshStats, + refreshState, + discardDraft, + } +} diff --git a/client/src/mobile/screens/admin/MAdmin.tsx b/client/src/mobile/screens/admin/MAdmin.tsx index ff5c26d38..5f4923281 100644 --- a/client/src/mobile/screens/admin/MAdmin.tsx +++ b/client/src/mobile/screens/admin/MAdmin.tsx @@ -10,6 +10,7 @@ import { Database, FileText, Github, + HardDrive, Map, Plug, Puzzle, @@ -39,6 +40,7 @@ import MAdminPackingTemplateManager from './MAdminPackingTemplateManager' import MAdminNotificationsSection from './MAdminNotificationsSection' import MAdminSettingsSection from './MAdminSettingsSection' import MAdminSheets from './MAdminSheets' +import MAdminStoragePanel from './MAdminStoragePanel' import MAdminUsersSection from './MAdminUsersSection' import { MAdminButton } from './MAdminUi' @@ -86,6 +88,7 @@ export default function MAdmin() { { id: 'settings', label: t('admin.tabs.settings'), icon: SettingsIcon }, { id: 'addons', label: t('admin.tabs.addons'), icon: Blocks }, { id: 'plugins', label: t('admin.tabs.plugins'), icon: Puzzle }, + ...(managed ? [] : [{ id: 'storage', label: t('admin.tabs.storage'), icon: HardDrive }]), { id: 'notifications', label: t('admin.tabs.notifications'), icon: Bell }, ...(mcpEnabled ? [{ id: 'mcp-tokens', label: t('admin.tabs.mcpTokens'), icon: Plug }] : []), // Same two the desktop list drops: releases and backup schedule belong to @@ -238,6 +241,7 @@ export default function MAdmin() { /> )} {activeTab === 'plugins' && } + {activeTab === 'storage' && } {activeTab === 'mcp-tokens' && } {activeTab === 'github' && } {activeTab === 'backup' && } diff --git a/client/src/mobile/screens/admin/MAdminStoragePanel.tsx b/client/src/mobile/screens/admin/MAdminStoragePanel.tsx new file mode 100644 index 000000000..9c90d2a59 --- /dev/null +++ b/client/src/mobile/screens/admin/MAdminStoragePanel.tsx @@ -0,0 +1,890 @@ +import React, { useEffect, useRef, useState } from 'react' +import { ChevronDown } from 'lucide-react' +import { + STORAGE_BACKEND_TYPES, + STORAGE_BACKEND_TYPE_IDS, + STORAGE_CATEGORIES, + type StorageBackend, + type StorageBackendFieldDef, + type StorageBackendTypeId, + type StorageCategory, + type StorageConfig, + type StorageMigrationStatus, +} from '@trek/shared' +import { useTranslation } from '../../../i18n' +import { useToast } from '../../../components/shared/Toast' +import { formatBytes } from '../../../utils/formatBytes' +import { relativeTime } from '../../../utils/relativeTime' +import { + CACHE_CATEGORIES, + adoptedMirrorFor, + computeMigrationCandidates, + effectiveCategoryMap, + foldBackends, + primaryNameOf, + removeBackend, + removeBackendAndMirrors, + renameBackendRefs, + replicaCandidates, + replicaOfPrimaries, + settingsDocumentOf, + setMirrorTargets, + stripCategories, + upsertBackend, + usageByBackend, + type FoldedBackendRow, + type MigrationCandidate, +} from '../../../components/Admin/storage/storageModel' +import { useStorageAdmin } from '../../../components/Admin/storage/useStorageAdmin' +import MToggle from '../../components/MToggle' +import MSetPickerSheet from '../settings/MSetPickerSheet' +import MConfirmSheet from '../settings/MConfirmSheet' +import { MSetSelectRow } from '../settings/MSettingsUi' +import { MAdminButton, MAdminCard, MAdminCardHead, MAdminField, MAdminInput, MAdminSecretInput } from './MAdminUi' + +type FieldValues = Record + +function valuesOf(backend: StorageBackend | null): FieldValues { + if (!backend) return {} + const values: FieldValues = {} + for (const [key, value] of Object.entries(backend.options)) { + values[key] = Array.isArray(value) ? value : String(value) + } + return values +} + +/** Display-name mapper for joined category lists — the raw id renders only in the badge. */ +const categoryNames = (t: (key: string) => string, ids: readonly string[]): string => + ids.map((id) => t(`storage.category.${id}`)).join(', ') + +/** Same behavior contract as the desktop BackendForm, rendered on the M* primitives. */ +function MBackendForm({ + initial, + backendNames, + mirror, + onCommit, + onCancel, +}: { + initial: StorageBackend | null + backendNames: string[] + mirror: { candidates: string[]; initialTargets: string[] } + onCommit: (backend: StorageBackend, mirrorTargets: string[]) => void + onCancel: () => void +}): React.ReactElement { + const { t } = useTranslation() + const [type, setType] = useState(initial?.type ?? 'local') + const [name, setName] = useState(initial?.name ?? '') + const [values, setValues] = useState(() => valuesOf(initial)) + const [targets, setTargets] = useState(mirror.initialTargets) + const [picker, setPicker] = useState(null) + + const fields = STORAGE_BACKEND_TYPES[type].fields as readonly StorageBackendFieldDef[] + const refOptions = backendNames.filter((candidate) => candidate !== name.trim()) + const setValue = (key: string, value: string | string[]) => setValues((prev) => ({ ...prev, [key]: value })) + + const filled = (field: StorageBackendFieldDef): boolean => { + const value = values[field.key] + if (field.kind === 'backend-ref-list') return Array.isArray(value) && value.length > 0 + return typeof value === 'string' && value.trim() !== '' + } + const duplicate = name.trim() !== (initial?.name ?? '') && backendNames.includes(name.trim()) + const canApply = name.trim() !== '' && !duplicate && fields.every((f) => !f.required || filled(f)) + + const apply = () => { + const options: Record = {} + for (const field of fields) { + const value = values[field.key] + if (field.kind === 'backend-ref-list') { + options[field.key] = Array.isArray(value) ? value : [] + continue + } + const text = typeof value === 'string' ? value : '' + if (text === '' && !field.required) continue + options[field.key] = field.kind === 'number' ? Number(text) : text + } + onCommit({ name: name.trim(), type, options } as StorageBackend, targets) + } + + return ( + + + + setName(e.target.value)} spellCheck={false} autoComplete="off" /> + + {duplicate && ( +

+ {t('storage.form.duplicateName', { name: name.trim() })} +

+ )} + {initial === null && ( + + } + onClick={() => setPicker('type')} + /> + setPicker(null)} + title={t('storage.form.type')} + options={STORAGE_BACKEND_TYPE_IDS.filter((id) => id !== 'mirror').map((id) => ({ + value: id, + label: t(`storage.type.${id}`), + }))} + value={type} + onSelect={(next) => { + setType(next as StorageBackendTypeId) + setValues({}) + }} + /> + + )} + + {fields.map((field) => { + const value = values[field.key] + if (field.kind === 'backend-ref') { + const current = typeof value === 'string' ? value : '' + return ( + + } + onClick={() => setPicker(`field:${field.key}`)} + /> + setPicker(null)} + title={t(field.labelKey)} + options={refOptions.map((candidate) => ({ value: candidate, label: candidate }))} + value={current} + onSelect={(next) => setValue(field.key, next)} + /> + + ) + } + if (field.kind === 'backend-ref-list') { + const selected = Array.isArray(value) ? value : [] + return ( + +
+ {refOptions.map((candidate) => ( +
+ {candidate} + + setValue( + field.key, + checked ? [...selected, candidate] : selected.filter((existing) => existing !== candidate), + ) + } + /> +
+ ))} +
+ + ) + } + const text = typeof value === 'string' ? value : '' + const shared = { + value: text, + onChange: (e: React.ChangeEvent) => setValue(field.key, e.target.value), + placeholder: field.defaultValue !== undefined ? String(field.defaultValue) : '', + spellCheck: false, + autoComplete: 'off', + } + return ( + + {field.kind === 'secret' ? ( + + ) : ( + + )} + + ) + })} + + +
+ {mirror.candidates + .filter((candidate) => candidate !== name.trim()) + .map((candidate) => ( +
+ {candidate} + + setTargets(checked ? [...targets, candidate] : targets.filter((existing) => existing !== candidate)) + } + /> +
+ ))} +
+ {targets.length > 0 && ( +

+ {t('storage.mirror.latencyNote')} +

+ )} + + +
+ + {t('storage.form.apply')} + + + {t('storage.form.cancel')} + +
+ + ) +} + +export default function MAdminStoragePanel(): React.ReactElement { + const { t, locale } = useTranslation() + const toast = useToast() + const admin = useStorageAdmin(t('common.error'), t('storage.saveConflict')) + const [editing, setEditing] = useState<{ + initial: StorageBackend | null + originalName: string | null + mirror: { candidates: string[]; initialTargets: string[] } + } | null>(null) + const [confirmRemove, setConfirmRemove] = useState<{ name: string; degenerate: boolean } | null>(null) + const [categoryPicker, setCategoryPicker] = useState(null) + const [syncPrompt, setSyncPrompt] = useState(null) + const [migratePrompt, setMigratePrompt] = useState(null) + const [migrationQueue, setMigrationQueue] = useState([]) + // Set by `save` right before it calls admin.save(): the pre-save mirror + // target count per row name. Consumed (and cleared) by the effect below the + // first time `admin.state` changes afterward — never touched otherwise, so + // unrelated state changes (the backfill poll included) are no-ops here. + const pendingPromptCheck = useRef | null>(null) + // Synchronous single-flight lock for the queue effect below: `admin.state` + // only reflects a just-started migration once startMigration's awaited + // refreshState() resolves, so `setMigrationQueue(rest)` re-firing the + // effect synchronously (still against the pre-POST `admin.state`) would + // otherwise dequeue and POST the next candidate before the server has + // confirmed the first — a ref (not state) so the guard is visible on that + // very next synchronous re-render, not just after a state-driven one. + const migrationStartInFlight = useRef(false) + + useEffect(() => { + if (!pendingPromptCheck.current || !admin.state) return + const before = pendingPromptCheck.current + pendingPromptCheck.current = null + const { rows: afterRows } = foldBackends(admin.state, settingsDocumentOf(admin.state)) + const grown = afterRows.find((r) => r.mirrorTargets.length > (before.get(r.name) ?? 0)) + if (grown) setSyncPrompt(grown.name) + }, [admin.state]) + + // Queued category migrations run strictly sequentially: once a slot opens + // (no migration currently running), dequeue the next candidate and start + // it. Depends on admin.state so a poll landing a terminal status re-fires + // this without any user interaction. The in-flight lock closes the race + // where setMigrationQueue(rest) re-fires this effect synchronously while + // admin.state still shows the pre-POST world — without it, the next + // candidate could dequeue and POST before the first is server-confirmed. + // Also waits out a running BACKFILL — the server's one-storage-job-at-a-time + // rule spans backfills and migrations alike, so starting while one runs + // would 409, and the queued candidate would be lost (no retry). + useEffect(() => { + if (migrationQueue.length === 0 || !admin.state) return + if (migrationStartInFlight.current) return + if (admin.state.migrations.some((m) => m.status === 'running')) return + if (admin.state.backfills.some((b) => b.status === 'running')) return + const [next, ...rest] = migrationQueue + migrationStartInFlight.current = true + setMigrationQueue(rest) + void admin.startMigration(next!.category, next!.toWire).then((error) => { + migrationStartInFlight.current = false + if (error) toast.error(error) + }) + }, [migrationQueue, admin.state]) + + if (admin.loading) { + return ( + +

{t('storage.loading')}

+ + ) + } + if (!admin.state || !admin.draft) { + return ( + +

{admin.loadError || t('common.error')}

+ + ) + } + const { state, draft } = admin + + const backendNames = [...new Set([...state.backends.map((b) => b.name), ...draft.backends.map((b) => b.name)])] + const { rows, degenerate } = foldBackends(state, draft) + const effective = effectiveCategoryMap(state, draft) + const usageSums = usageByBackend(state, draft) + + const startEdit = (row: FoldedBackendRow) => { + setEditing({ + initial: row.backend, + originalName: row.name, + mirror: { + candidates: replicaCandidates(rows, row.name, row.mirrorTargets), + initialTargets: row.mirrorTargets, + }, + }) + } + + const commitBackend = (backend: StorageBackend, mirrorTargets: string[]) => { + const renamedFrom = editing?.originalName && editing.originalName !== backend.name ? editing.originalName : null + // Widened to StorageConfig: these edit helpers are version-blind (they + // return plain StorageConfig), and admin.setDraft re-attaches the + // draft's own `version` regardless of what shape it's handed. + let next: StorageConfig | null = draft + if (renamedFrom) next = renameBackendRefs(removeBackend(next, renamedFrom), renamedFrom, backend.name) + next = upsertBackend(next, backend) + next = setMirrorTargets(state, next, backend.name, mirrorTargets) + admin.setDraft(next) + setEditing(null) + } + + const removeMessage = (name: string, isDegenerate: boolean): string => { + const row = rows.find((r) => r.name === name) + const assigned = isDegenerate + ? degenerate.find((d) => d.backend.name === name)?.categories ?? [] + : row?.categories ?? [] + const usedAsReplicaBy = isDegenerate ? [] : replicaOfPrimaries(draft, name) + return [ + t('storage.remove.body', { name }), + assigned.length > 0 ? t('storage.remove.stillAssigned', { categories: categoryNames(t, assigned) }) : '', + usedAsReplicaBy.length > 0 ? t('storage.remove.usedAsReplicaBy', { primaries: usedAsReplicaBy.join(', ') }) : '', + ] + .filter(Boolean) + .join(' ') + } + + const setCategory = (category: StorageCategory, primaryName: string) => { + const target = adoptedMirrorFor(draft, primaryName)?.name ?? primaryName + // The admin state's category record is exhaustive by schema contract. + const stateEntry = state.categories[category]! + const categories = { ...draft.categories } + if (stateEntry.source === 'default' && target === stateEntry.backend) { + delete categories[category] + } else { + categories[category] = target + } + admin.setDraft({ ...draft, categories }) + } + + // Snapshot the LAST-CONFIRMED (pre-save `state`'s own fold, not the + // in-progress `draft`) mirror target counts; the effect above compares + // them against the fresh post-save state to detect newly-added + // replicas. Folding `draft` here would already include the unsaved + // edit and mask the very growth this is meant to detect. + const snapshotMirrorTargets = (): Map => { + const { rows: beforeRows } = foldBackends(state, settingsDocumentOf(state)) + return new Map(beforeRows.map((r) => [r.name, r.mirrorTargets.length])) + } + + const doPlainSave = async () => { + pendingPromptCheck.current = snapshotMirrorTargets() + if (await admin.save()) toast.success(t('storage.saved')) + else pendingPromptCheck.current = null + } + + const moveAndSave = async (candidates: MigrationCandidate[]) => { + pendingPromptCheck.current = snapshotMirrorTargets() + const ok = await admin.save(stripCategories(draft, state, candidates.map((c) => c.category))) + if (ok) { + setMigrationQueue(candidates) + toast.success(t('storage.saved')) + } else { + pendingPromptCheck.current = null + } + setMigratePrompt(null) + } + + const routeOnlySave = async () => { + setMigratePrompt(null) + await doPlainSave() + } + + const save = async () => { + const candidates = computeMigrationCandidates(draft, state) + if (candidates.length > 0) { + setMigratePrompt(candidates) + return + } + await doPlainSave() + } + + const handleCancelMigration = async (m: StorageMigrationStatus) => { + const error = await admin.cancelMigration(m.category) + if (error) toast.error(error) + } + + const handleRefreshStats = async () => { + const error = await admin.refreshStats() + if (error) toast.error(error) + } + + const handleStartBackfill = async (row: FoldedBackendRow) => { + if (!row.mirrorName) return + setSyncPrompt((prev) => (prev === row.name ? null : prev)) + const error = await admin.startBackfill(row.mirrorName) + if (error) toast.error(error) + } + + const handleCancelBackfill = async (row: FoldedBackendRow) => { + if (!row.mirrorName) return + const error = await admin.cancelBackfill(row.mirrorName) + if (error) toast.error(error) + } + + return ( +
+ + + {state.health.replicaFailures.length === 0 ? ( +

{t('storage.health.allClear')}

+ ) : ( +
+ {state.health.replicaFailures.map((failure) => ( +

+ {t('storage.health.failureLine', { + op: failure.op, + key: failure.key, + backend: failure.backend, + error: failure.error, + })}{' '} + · {relativeTime(failure.at, locale)} +

+ ))} +
+ )} + {state.seedFilePresent && ( +

{t('storage.health.seedFile')}

+ )} + + + + +
+

+ {state.usage + ? t('storage.usage.computed', { age: relativeTime(state.usage.computedAt, locale) }) + : t('storage.usage.never')} +

+ + {state.usage ? t('storage.usage.refresh') : t('storage.usage.compute')} + +
+
+ {rows.map((row) => { + const resultKey = row.mirrorName ?? row.name + const result = admin.testResults[resultKey] + // row.mirrorName only exists when foldBackends adopted a draft mirror + // for this row, so the draft lookup below cannot miss. + const testCandidate = row.mirrorName + ? draft.backends.find((b) => b.name === row.mirrorName)! + : row.backend + const rowUsage = usageSums?.[row.name] + const backfill = row.mirrorName ? state.backfills.find((b) => b.backend === row.mirrorName) : undefined + return ( +
+
+ {row.name} + + {t(`storage.type.${row.type}`)} + + + {t(`storage.source.${row.source}`)} + +
+

+ {row.categories.length > 0 + ? t('storage.backends.usedBy', { categories: categoryNames(t, row.categories) }) + : t('storage.backends.unused')} +

+ {rowUsage && ( +

+ {t('storage.usage.line', { objects: String(rowUsage.objects), size: formatBytes(rowUsage.bytes) })} + {row.name === 'uploads-local' && state.usage!.legacyPhotos.objects > 0 + ? ` (${t('storage.usage.legacyNote')})` + : ''} +

+ )} + {row.mirrorTargets.length > 0 && ( +

+ {t('storage.mirror.mirroredTo', { targets: row.mirrorTargets.join(', ') })} +

+ )} + {row.replicaOf.length > 0 && ( +

+ {t('storage.mirror.replicaOf', { primaries: row.replicaOf.join(', ') })} +

+ )} + {row.source === 'env' && ( +

{t('storage.backends.envReadOnly')}

+ )} +
+ admin.test(testCandidate)}> + {t('storage.actions.test')} + + {row.source !== 'env' && ( + startEdit(row)}> + {t('storage.actions.edit')} + + )} + {row.source === 'settings' && ( + setConfirmRemove({ name: row.name, degenerate: false })}> + {t('storage.actions.remove')} + + )} +
+ {result === 'running' ? ( +

{t('storage.test.running')}

+ ) : ( + result && ( +
+

+ {result.ok ? t('storage.test.ok') : t('storage.test.failed')} +

+ {result.targets.map((target) => ( +

+ {target.ok ? '✓' : '✗'} {target.name} + {target.error ? ` — ${target.error}` : ''} +

+ ))} +
+ ) + )} + {row.mirrorTargets.length > 0 && ( +
+ {backfill?.status === 'running' ? ( + <> +

+ {t('storage.sync.running', { done: String(backfill.done), total: String(backfill.total) })} +

+

+ {t('storage.sync.counts', { + copied: String(backfill.copied), + skipped: String(backfill.skipped), + failed: String(backfill.failed), + })} +

+
+ handleCancelBackfill(row)}> + {t('storage.sync.cancel')} + +
+ + ) : backfill?.status === 'done' ? ( +

+ {t('storage.sync.done', { + copied: String(backfill.copied), + deleted: String(backfill.deleted), + failed: String(backfill.failed), + })} +

+ ) : backfill?.status === 'cancelled' ? ( +

{t('storage.sync.cancelled')}

+ ) : backfill?.status === 'error' ? ( +

+ {t('storage.sync.error', { error: backfill.error ?? '' })} +

+ ) : syncPrompt !== row.name ? ( +
+ handleStartBackfill(row)}> + {t('storage.sync.now')} + +
+ ) : null} +
+ )} + {syncPrompt === row.name && ( +
+

{t('storage.sync.prompt')}

+
+ handleStartBackfill(row)}> + {t('storage.sync.now')} + + setSyncPrompt(null)}> + {t('storage.sync.dismiss')} + +
+
+ )} +
+ ) + })} + + {degenerate.map(({ backend, reason }) => { + const result = admin.testResults[backend.name] + const primary = backend.type === 'mirror' ? backend.options.primary : '' + return ( +
+
+ {backend.name} + + {t('storage.type.mirror')} + +
+

+ {t(`storage.mirror.degenerate.${reason}`, { primary })} +

+
+ admin.test(backend)}> + {t('storage.actions.test')} + + setConfirmRemove({ name: backend.name, degenerate: true })}> + {t('storage.actions.remove')} + +
+ {result === 'running' ? ( +

{t('storage.test.running')}

+ ) : ( + result && ( +
+

+ {result.ok ? t('storage.test.ok') : t('storage.test.failed')} +

+ {result.targets.map((target) => ( +

+ {target.ok ? '✓' : '✗'} {target.name} + {target.error ? ` — ${target.error}` : ''} +

+ ))} +
+ ) + )} +
+ ) + })} + + {state.migrations.map((m) => ( +
+

{t(`storage.category.${m.category}`)}

+ {m.status === 'running' ? ( + <> +

+ {t('storage.migrate.running', { + category: t(`storage.category.${m.category}`), + done: String(m.done), + total: String(m.total), + })} +

+
+ void handleCancelMigration(m)}> + {t('storage.migrate.cancel')} + +
+ + ) : m.status === 'done' ? ( + <> +

+ {t('storage.migrate.done', { copied: String(m.copied), skipped: String(m.skipped) })} +

+ {m.failed > 0 && ( +

+ {t('storage.migrate.doneFailures', { failed: String(m.failed) })} +

+ )} + {m.reclaimable && ( +

+ {t('storage.migrate.reclaimable', { + objects: String(m.reclaimable.objects), + size: formatBytes(m.reclaimable.bytes), + from: m.from, + })} +

+ )} + + ) : m.status === 'failed' ? ( +

+ {t('storage.migrate.failed', { error: m.error ?? '' })} +

+ ) : ( +

{t('storage.migrate.cancelled')}

+ )} +
+ ))} +
+ {editing === null && ( +
+ + setEditing({ + initial: null, + originalName: null, + mirror: { candidates: replicaCandidates(rows, null), initialTargets: [] }, + }) + } + > + {t('storage.backends.add')} + +
+ )} + + + {editing !== null && ( + setEditing(null)} + /> + )} + + + +
+ {STORAGE_CATEGORIES.map((category) => { + // The admin state's category record is exhaustive by schema contract. + const stateEntry = state.categories[category]! + const selectedPrimary = primaryNameOf(state, draft, effective[category]) + const changed = selectedPrimary !== primaryNameOf(state, draft, stateEntry.backend) + const viaMirror = effective[category] !== selectedPrimary + return ( + + {t(`storage.category.${category}`)}{' '} + + {category} + + + } + hint={t(`storage.categoryDesc.${category}`)} + > + {state.usage?.categories[category] && ( +

+ {t('storage.usage.line', { + objects: String(state.usage.categories[category]!.objects), + size: formatBytes(state.usage.categories[category]!.bytes), + })} +

+ )} +
+ } + onClick={() => setCategoryPicker(category)} + /> +
+ {changed && ( +

+ {t('storage.categories.reassignWarning')} +

+ )} + {viaMirror && CACHE_CATEGORIES.includes(category) && ( +

+ {t('storage.mirror.cacheWarning')} +

+ )} + + ) + })} +
+ setCategoryPicker(null)} + title={categoryPicker ? t(`storage.category.${categoryPicker}`) : ''} + options={rows.map((row) => ({ value: row.name, label: row.name }))} + value={categoryPicker ? primaryNameOf(state, draft, effective[categoryPicker]) : ''} + onSelect={(name) => { + if (categoryPicker) setCategory(categoryPicker, name) + }} + /> + + +
+ + {t('storage.save')} + + {admin.dirty && {t('storage.unsaved')}} +
+ {migratePrompt && ( +
+ +

{t('storage.migrate.promptTitle')}

+ {migratePrompt.map((c) => ( +

+ {c.objects === null + ? t('storage.migrate.promptLineUnknown', { + category: t(`storage.category.${c.category}`), + from: c.from, + to: c.to, + }) + : t('storage.migrate.promptLine', { + category: t(`storage.category.${c.category}`), + objects: String(c.objects), + size: formatBytes(c.bytes ?? 0), + from: c.from, + to: c.to, + })} +

+ ))} +
+ void moveAndSave(migratePrompt)}> + {t('storage.migrate.move')} + + void routeOnlySave()}> + {t('storage.migrate.routeOnly')} + +
+ +
+ )} + {admin.saveError && ( +
+

{admin.saveError}

+ {/* A 409 pins the draft to a version the server moved past — saving + again can never clear it, so offer the one action that can. */} + {admin.saveConflict && ( + void admin.discardDraft()}> + {t('storage.discardAndReload')} + + )} +
+ )} + + setConfirmRemove(null)} + title={t('storage.remove.title')} + message={confirmRemove ? removeMessage(confirmRemove.name, confirmRemove.degenerate) : ''} + confirmLabel={t('storage.actions.remove')} + cancelLabel={t('storage.form.cancel')} + danger + onConfirm={() => { + if (confirmRemove) { + admin.setDraft( + confirmRemove.degenerate + ? removeBackend(draft, confirmRemove.name) + : removeBackendAndMirrors(state, draft, confirmRemove.name), + ) + } + setConfirmRemove(null) + }} + /> +
+ ) +} diff --git a/client/src/pages/AdminPage.test.tsx b/client/src/pages/AdminPage.test.tsx index 1cd09f0f4..bffee1972 100644 --- a/client/src/pages/AdminPage.test.tsx +++ b/client/src/pages/AdminPage.test.tsx @@ -69,6 +69,10 @@ vi.mock('../components/Admin/DefaultUserSettingsTab', () => ({ default: () =>
, })); +vi.mock('../components/Admin/storage/AdminStoragePanel', () => ({ + default: () =>
, +})); + beforeEach(() => { resetAllStores(); }); @@ -1631,4 +1635,19 @@ describe('AdminPage', () => { expect(screen.queryByText('Files')).not.toBeInTheDocument(); }); }); + + it('FE-PAGE-ADMIN-STOR-001: the Storage tab renders the storage panel', async () => { + seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin() }); + render(); + const tab = await screen.findByRole('button', { name: /^storage$/i }); + fireEvent.click(tab); + expect(screen.getByTestId('admin-storage-panel')).toBeInTheDocument(); + }); + + it('FE-PAGE-ADMIN-STOR-002: managed mode hides the Storage tab (hoster-level config)', async () => { + seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin(), managed: true }); + render(); + await screen.findByRole('button', { name: /^users$/i }); + expect(screen.queryByRole('button', { name: /^storage$/i })).not.toBeInTheDocument(); + }); }); diff --git a/client/src/pages/AdminPage.tsx b/client/src/pages/AdminPage.tsx index 9fc153137..16e31f8a6 100644 --- a/client/src/pages/AdminPage.tsx +++ b/client/src/pages/AdminPage.tsx @@ -12,7 +12,8 @@ import PackingTemplateManager from '../components/Admin/PackingTemplateManager' import AuditLogPanel from '../components/Admin/AuditLogPanel' import AdminMcpTokensPanel from '../components/Admin/AdminMcpTokensPanel' import AdminPluginsPanel from '../components/Admin/AdminPluginsPanel' -import { Users, Map, Briefcase, Shield, FileText, SlidersHorizontal, UserCog, Puzzle, Blocks, Settings as SettingsIcon, Bell, Database, ScrollText, KeyRound, GitBranch, Bug } from 'lucide-react' +import AdminStoragePanel from '../components/Admin/storage/AdminStoragePanel' +import { Users, Map, Briefcase, Shield, FileText, SlidersHorizontal, UserCog, Puzzle, Blocks, Settings as SettingsIcon, Bell, Database, ScrollText, KeyRound, GitBranch, Bug, HardDrive } from 'lucide-react' import PageSidebar, { type PageSidebarTab } from '../components/Layout/PageSidebar' import { useAdmin } from './admin/useAdmin' import AdminUpdateBanner from './admin/AdminUpdateBanner' @@ -55,6 +56,9 @@ function AdminPageDesktop(): React.ReactElement { { id: 'settings', label: t('admin.tabs.settings'), icon: SettingsIcon, group: gConfig }, { id: 'addons', label: t('admin.tabs.addons'), icon: Puzzle, group: gConfig }, { id: 'plugins', label: t('admin.tabs.plugins'), icon: Blocks, group: gConfig }, + // Storage backends and their credentials are hoster-level configuration — + // the server refuses the whole surface in managed mode (MANAGED_FORBIDDEN). + ...(managed ? [] : [{ id: 'storage', label: t('admin.tabs.storage'), icon: HardDrive, group: gConfig }]), { id: 'notifications', label: t('admin.tabs.notifications'), icon: Bell, group: gIntegration }, ...(mcpEnabled ? [{ id: 'mcp-tokens', label: t('admin.tabs.mcpTokens'), icon: KeyRound, group: gIntegration }] : []), // Releases and update cadence belong to whoever operates the install. @@ -180,6 +184,8 @@ function AdminPageDesktop(): React.ReactElement { {activeTab === 'plugins' && } + {activeTab === 'storage' && } + {activeTab === 'github' && } {activeTab === 'defaults' && } diff --git a/client/src/utils/formatBytes.test.ts b/client/src/utils/formatBytes.test.ts new file mode 100644 index 000000000..1ff19a358 --- /dev/null +++ b/client/src/utils/formatBytes.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest' +import { formatBytes } from './formatBytes' + +describe('formatBytes', () => { + it('FE-UTIL-BYTES-001: scales B→TB with one decimal above bytes', () => { + expect(formatBytes(0)).toBe('0 B') + expect(formatBytes(512)).toBe('512 B') + expect(formatBytes(3_174)).toBe('3.1 KB') + expect(formatBytes(2 * 1024 * 1024)).toBe('2.0 MB') + expect(formatBytes(1.5 * 1024 ** 3)).toBe('1.5 GB') + expect(formatBytes(1.2 * 1024 ** 4)).toBe('1.2 TB') + }) +}) diff --git a/client/src/utils/formatBytes.ts b/client/src/utils/formatBytes.ts new file mode 100644 index 000000000..bd00c2229 --- /dev/null +++ b/client/src/utils/formatBytes.ts @@ -0,0 +1,13 @@ +const UNITS = ['B', 'KB', 'MB', 'GB', 'TB'] as const + +/** B→TB, one decimal above bytes. (FileManager's MB-capped formatSize predates this; not consolidated here.) */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${Math.round(bytes)} B` + let value = bytes + let unit = 0 + while (value >= 1024 && unit < UNITS.length - 1) { + value /= 1024 + unit += 1 + } + return `${value.toFixed(1)} ${UNITS[unit]}` +} diff --git a/client/src/utils/relativeTime.test.ts b/client/src/utils/relativeTime.test.ts new file mode 100644 index 000000000..698524800 --- /dev/null +++ b/client/src/utils/relativeTime.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; +import { relativeTime } from './relativeTime'; + +describe('relativeTime', () => { + const now = 1_700_000_000_000; + + it('FE-UTIL-RELTIME-001: formats seconds, minutes, hours and days', () => { + expect(relativeTime(now - 30_000, 'en', now)).toBe('30 seconds ago'); + expect(relativeTime(now - 120_000, 'en', now)).toBe('2 minutes ago'); + expect(relativeTime(now - 2 * 3_600_000, 'en', now)).toBe('2 hours ago'); + expect(relativeTime(now - 3 * 86_400_000, 'en', now)).toBe('3 days ago'); + }); + + it('FE-UTIL-RELTIME-002: clamps future timestamps to "now"', () => { + expect(relativeTime(now + 5_000, 'en', now)).toBe('now'); + }); + + it('FE-UTIL-RELTIME-003: localizes via Intl (de)', () => { + expect(relativeTime(now - 120_000, 'de', now)).toBe('vor 2 Minuten'); + }); +}); diff --git a/client/src/utils/relativeTime.ts b/client/src/utils/relativeTime.ts new file mode 100644 index 000000000..8c46cb9e9 --- /dev/null +++ b/client/src/utils/relativeTime.ts @@ -0,0 +1,15 @@ +const STEPS: Array<{ limit: number; div: number; unit: Intl.RelativeTimeFormatUnit }> = [ + { limit: 60_000, div: 1_000, unit: 'second' }, + { limit: 3_600_000, div: 60_000, unit: 'minute' }, + { limit: 86_400_000, div: 3_600_000, unit: 'hour' }, + { limit: Number.POSITIVE_INFINITY, div: 86_400_000, unit: 'day' }, +] + +/** "2 minutes ago" from an epoch-ms timestamp, localized. Future stamps clamp to "now". */ +export function relativeTime(at: number, locale: string, now: number = Date.now()): string { + const elapsed = now - at + if (elapsed < 1_000) return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format(0, 'second') + const step = STEPS.find((s) => elapsed < s.limit)! + const value = Math.round(elapsed / step.div) + return new Intl.RelativeTimeFormat(locale, { numeric: 'auto' }).format(-value, step.unit) +} diff --git a/client/tests/unit/mobile/admin/MAdmin.test.tsx b/client/tests/unit/mobile/admin/MAdmin.test.tsx index 0193e5e8b..5a45f7f9d 100644 --- a/client/tests/unit/mobile/admin/MAdmin.test.tsx +++ b/client/tests/unit/mobile/admin/MAdmin.test.tsx @@ -55,6 +55,9 @@ vi.mock('../../../../src/mobile/screens/admin/MAdminGitHubPanel', () => ({
{String(isPrerelease)}
), })); +vi.mock('../../../../src/mobile/screens/admin/MAdminStoragePanel', () => ({ + default: () =>
, +})); // The addon manager is reduced to the two controls MAdmin wires itself. vi.mock('../../../../src/mobile/screens/admin/MAdminAddonManager', () => ({ @@ -213,6 +216,9 @@ describe('MAdmin', () => { await openSection('Plugins'); expect(screen.getByTestId('plugins-panel')).toBeInTheDocument(); + await openSection('Storage'); + expect(screen.getByTestId('m-admin-storage-panel')).toBeInTheDocument(); + await openSection('Notifications'); expect(screen.getByTestId('notifications-section')).toBeInTheDocument(); @@ -380,4 +386,11 @@ describe('MAdmin', () => { expect(root.className).not.toContain('h-full'); expect(root.className).not.toContain('overflow-y-auto'); }); + + it('FE-MOB-ADMIN-STOR-001: managed mode hides the Storage section', async () => { + seedStore(useAuthStore, { isAuthenticated: true, user: buildAdmin(), managed: true }); + renderAdmin(); + fireEvent.click(await screen.findByRole('button', { name: 'Administration' })); + expect(screen.queryByRole('button', { name: 'Storage' })).not.toBeInTheDocument(); + }); }); diff --git a/client/tests/unit/mobile/admin/MAdminStoragePanel.test.tsx b/client/tests/unit/mobile/admin/MAdminStoragePanel.test.tsx new file mode 100644 index 000000000..12ffcda04 --- /dev/null +++ b/client/tests/unit/mobile/admin/MAdminStoragePanel.test.tsx @@ -0,0 +1,431 @@ +import { http, HttpResponse } from 'msw'; +import { describe, expect, it } from 'vitest'; +import { MASKED_SETTING_VALUE, type StorageAdminState, type StorageCategory, type StorageConfig } from '@trek/shared'; +import { server } from '../../../helpers/msw/server'; +import { fireEvent, render, screen, waitFor, within } from '../../../helpers/render'; +import { ToastContainer } from '../../../../src/components/shared/Toast'; +import MAdminStoragePanel from '../../../../src/mobile/screens/admin/MAdminStoragePanel'; + +function baseState(overrides: Partial = {}): StorageAdminState { + return { + backends: [ + { name: 'uploads-local', type: 'local', source: 'built-in', options: { root: '/data/uploads' }, categories: ['files', 'journey', 'covers', 'avatars', 'photos-google', 'photos-trek'] }, + { name: 'backups-local', type: 'local', source: 'built-in', options: { root: '/data/backups' }, categories: ['backups'] }, + { + name: 'off-box', type: 's3', source: 'settings', + options: { endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: MASKED_SETTING_VALUE, region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000 }, + categories: [], + }, + ], + categories: { + files: { backend: 'uploads-local', source: 'default' }, + journey: { backend: 'uploads-local', source: 'default' }, + covers: { backend: 'uploads-local', source: 'default' }, + avatars: { backend: 'uploads-local', source: 'default' }, + places: { backend: 'uploads-local', source: 'default' }, + 'photos-google': { backend: 'uploads-local', source: 'default' }, + 'photos-trek': { backend: 'uploads-local', source: 'default' }, + backups: { backend: 'backups-local', source: 'default' }, + }, + health: { replicaFailures: [] }, + seedFilePresent: false, + usage: null, + backfills: [], + migrations: [], + version: 0, + ...overrides, + }; +} + +function mirroredState(): StorageAdminState { + const state = baseState(); + state.backends.push({ + name: 'mirror', type: 'mirror', source: 'settings', + options: { primary: 'backups-local', replicas: ['off-box'] }, categories: ['backups'], + }); + state.categories.backups = { backend: 'mirror', source: 'settings' }; + return state; +} + +async function renderPanel(state: StorageAdminState = baseState()) { + server.use(http.get('/api/admin/storage', () => HttpResponse.json(state))); + render( + <> + + + , + ); + await waitFor(() => expect(screen.getByText('Backends')).toBeInTheDocument()); +} + +describe('MAdminStoragePanel', () => { + it('FE-MOB-MSTOR-001: renders every backend with type and source, env rows read-only', async () => { + await renderPanel(); + const row = screen.getByTestId('m-storage-backend-off-box'); + expect(within(row).getByText('S3')).toBeInTheDocument(); + expect(within(row).getByText('Settings')).toBeInTheDocument(); + expect(screen.getByText('No replica failures recorded.')).toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-002: reassigning a category via the picker sheet warns and saves through one PUT', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button')); + fireEvent.click(screen.getByRole('button', { name: 'off-box' })); // picker option; selecting closes + expect(screen.getByText(/Existing objects do not move/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned (baseState().usage === null) — the migrate + // prompt fires; this test is about the picker + PUT shape, not the + // prompt, so route new writes only (today's save behavior, unstripped). + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as { categories: Record }).categories.files).toBe('off-box'); + }); + + it('FE-MOB-MSTOR-003: a typed plaintext secret keeps Apply offered (no encryption-key gate)', async () => { + await renderPanel(); + fireEvent.click(within(screen.getByTestId('m-storage-backend-off-box')).getByRole('button', { name: 'Edit' })); + fireEvent.change(screen.getByPlaceholderText('us-east-1'), { target: { value: 'eu-west-1' } }); // sanity: form is open + const secret = screen.getByDisplayValue(MASKED_SETTING_VALUE); + fireEvent.change(secret, { target: { value: 'sk-new' } }); + expect(screen.getByRole('button', { name: 'Apply' })).toBeInTheDocument(); + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-004: a 400 renders the server message verbatim', async () => { + await renderPanel(); + server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: 'registry says no' }, { status: 400 }))); + fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button')); + fireEvent.click(screen.getByRole('button', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned — the migrate prompt fires first; route new + // writes only, matching this test's focus on the save-error rendering. + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + expect(await screen.findByText('registry says no')).toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-005: renaming onto another existing backend warns and blocks Apply', async () => { + await renderPanel(); + fireEvent.click(within(screen.getByTestId('m-storage-backend-off-box')).getByRole('button', { name: 'Edit' })); + const nameInput = screen.getByDisplayValue('off-box'); + fireEvent.change(nameInput, { target: { value: 'uploads-local' } }); + expect(screen.getByText(/A backend named uploads-local already exists/)).toBeInTheDocument(); + expect(screen.getByRole('button', { name: 'Apply' })).toBeDisabled(); + }); + + it('FE-MOB-MSTOR-006: mirrors fold — no mirror card, primary and replica decorated', async () => { + await renderPanel(mirroredState()); + expect(screen.queryByTestId('m-storage-backend-mirror')).not.toBeInTheDocument(); + const primary = screen.getByTestId('m-storage-backend-backups-local'); + expect(within(primary).getByText('Mirrored to: off-box')).toBeInTheDocument(); + const replica = screen.getByTestId('m-storage-backend-off-box'); + expect(within(replica).getByText('Replica of: backups-local')).toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-007: toggling a target on a primary synthesizes the mirror in the PUT', async () => { + let putBody: unknown; + await renderPanel(); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(baseState()); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('switch', { name: 'off-box' })); + expect(screen.getByText(/slows every upload/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('Storage configuration saved'); + const body = putBody as { backends: Array<{ name: string; type: string; options: unknown }>; categories: Record }; + expect(body.backends.find((b) => b.name === 'backups-local-mirror')!.options).toEqual({ + primary: 'backups-local', replicas: ['off-box'], + }); + expect(body.categories.backups).toBe('backups-local-mirror'); + }); + + it('FE-MOB-MSTOR-008: the category picker deals in primaries and warns on cache categories', async () => { + let putBody: unknown; + await renderPanel(mirroredState()); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json(mirroredState()); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-category-places')).getByRole('button')); + fireEvent.click(within(screen.getByRole('dialog', { name: 'Place images' })).getByRole('button', { name: 'backups-local' })); // picker option = primary name + // getByText(/re-fetchable/) is now ambiguous — the photos-google/photos-trek category + // descriptions also say "re-fetchable". The cache warning is the only role="note" on + // screen here (the mirror-targets latency note only renders while a form is open). + expect(screen.getByRole('note')).toHaveTextContent(/re-fetchable/); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + // Usage was never scanned — the migrate prompt fires first; route new + // writes only, matching this test's focus on the picker/cache-warning. + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as { categories: Record }).categories.places).toBe('mirror'); + }); + + it('FE-MOB-MSTOR-009: category rows show display name, id badge and description; photos is gone', async () => { + await renderPanel(); + const files = screen.getByTestId('m-storage-category-files'); + // The testid wraps only the select row — assert name/badge/description through the field around it. + expect(screen.getByText('Trip documents')).toBeInTheDocument(); + expect(screen.getByText('files')).toBeInTheDocument(); + expect(screen.getByText(/tickets, PDFs, booking confirmations/)).toBeInTheDocument(); + expect(within(files).getByRole('button')).toBeInTheDocument(); // row still tappable + expect(screen.queryByTestId('m-storage-category-photos')).not.toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-010: usage renders on rows and the header; never-computed shows the compute prompt', async () => { + const usage = { + computedAt: Date.now() - 3_600_000, + categories: Object.fromEntries( + (['files', 'journey', 'covers', 'avatars', 'places', 'photos-google', 'photos-trek', 'backups'] as const).map( + (c) => [c, { objects: 2, bytes: 1024 * 1024 }], + ), + ), + legacyPhotos: { objects: 0, bytes: 0 }, + }; + usage.categories.files = { objects: 3, bytes: 2048 }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + expect(screen.getByText(/Usage computed/)).toBeInTheDocument(); + expect(within(screen.getByTestId('m-storage-backend-backups-local')).getByText(/2 objects · 1\.0 MB/)).toBeInTheDocument(); + // Per-category usage line (the testid wraps only the select row, so assert by text). + expect(screen.getByText('3 objects · 2.0 KB')).toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-011: Sync now → running → done via the test poll', async () => { + let polls = 0; + await renderPanel(mirroredState()); + server.use( + http.post('/api/admin/storage/backends/mirror/backfill', () => HttpResponse.json({ started: true })), + http.get('/api/admin/storage', () => { + polls += 1; + const state = mirroredState(); + (state as StorageAdminState).backfills = + polls < 3 + ? [{ backend: 'mirror', status: 'running', done: 1, total: 4, copied: 1, skipped: 0, failed: 0, deleted: 0, startedAt: 1 }] + : [{ backend: 'mirror', status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, deleted: 2, startedAt: 1, finishedAt: 2 }]; + return HttpResponse.json(state); + }), + ); + const row = screen.getByTestId('m-storage-backend-backups-local'); + fireEvent.click(within(row).getByRole('button', { name: 'Sync now' })); + await within(row).findByText(/Syncing… 1\/4/); + await within(row).findByText(/Sync finished: 4 copied, 2 deleted, 0 failed/); + }); + + it('FE-MOB-MSTOR-012: a save that added targets raises the sync prompt', async () => { + await renderPanel(); + server.use(http.put('/api/admin/storage', async () => HttpResponse.json(mirroredState()))); + fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('switch', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText(/Existing objects are not replicated yet/); + }); + + it('FE-MOB-MSTOR-013: reassigning a populated category prompts before saving; Move strips it from the PUT and POSTs the migration after save', async () => { + let putBody: unknown; + let putCalled = false; + let migrationBody: unknown; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putCalled = true; + putBody = await request.json(); + const saved = baseState(); + saved.categories.files = { backend: 'off-box', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', async ({ request }) => { + migrationBody = await request.json(); + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button')); + fireEvent.click(screen.getByRole('button', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + + expect(await screen.findByRole('alertdialog')).toBeInTheDocument(); + expect(putCalled).toBe(false); + expect(screen.getByText(/Trip documents: 3 objects \(3\.0 KB\) from uploads-local to off-box/)).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + // files is default-sourced (uploads-local) in baseState() — stripping restores "no override". + expect((putBody as StorageConfig).categories.files).toBeUndefined(); + expect(migrationBody).toEqual({ category: 'files', to: 'off-box' }); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-014: Just route new writes saves the reassignment as-is, no migration POST', async () => { + let putBody: unknown; + let migrationPosted = false; + const usage = { + computedAt: Date.now(), + categories: { files: { objects: 3, bytes: 3072 } }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + server.use( + http.put('/api/admin/storage', async ({ request }) => { + putBody = await request.json(); + const saved = baseState(); + saved.categories.files = { backend: 'off-box', source: 'settings' }; + return HttpResponse.json(saved); + }), + http.post('/api/admin/storage/migrations', () => { + migrationPosted = true; + return HttpResponse.json({ started: true }); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button')); + fireEvent.click(screen.getByRole('button', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + + fireEvent.click(screen.getByRole('button', { name: 'Just route new writes' })); + await screen.findByText('Storage configuration saved'); + expect((putBody as StorageConfig).categories.files).toBe('off-box'); + expect(migrationPosted).toBe(false); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); + + it('FE-MOB-MSTOR-015: two candidates queue sequentially: the second POSTs only after the first turns terminal', async () => { + const usage = { + computedAt: Date.now(), + categories: { + files: { objects: 3, bytes: 3072 }, + journey: { objects: 2, bytes: 2048 }, + }, + legacyPhotos: { objects: 0, bytes: 0 }, + }; + await renderPanel({ ...baseState(), usage } as StorageAdminState); + const savedState = baseState(); + savedState.categories.files = { backend: 'off-box', source: 'settings' }; + savedState.categories.journey = { backend: 'off-box', source: 'settings' }; + const migrationPosts: Array<{ category: StorageCategory; to: string }> = []; + // Held 'running' until the test explicitly flips this — deterministic, + // unlike counting poll ticks against the 50ms test interval. + let firstDone = false; + let postCount = 0; + server.use( + http.put('/api/admin/storage', () => HttpResponse.json(savedState)), + http.post('/api/admin/storage/migrations', async ({ request }) => { + postCount += 1; + const body = (await request.json()) as { category: StorageCategory; to: string }; + migrationPosts.push(body); + // Pins the single-flight fix: the second POST must never arrive + // before the first migration is server-confirmed terminal — a + // regression here means the queue's synchronous re-fire raced ahead + // of admin.state showing the first as done. + if (postCount === 2) expect(firstDone).toBe(true); + return HttpResponse.json({ started: true }); + }), + http.get('/api/admin/storage', () => { + const state = { ...savedState }; + state.migrations = + migrationPosts.length === 0 + ? [] + : [ + { + category: migrationPosts[0]!.category, from: 'uploads-local', to: migrationPosts[0]!.to, + status: firstDone ? 'done' : 'running', done: firstDone ? 3 : 1, total: 3, + copied: firstDone ? 3 : 1, skipped: 0, failed: 0, startedAt: 1, + ...(firstDone ? { finishedAt: 2 } : {}), + }, + ]; + return HttpResponse.json(state); + }), + ); + fireEvent.click(within(screen.getByTestId('m-storage-category-files')).getByRole('button')); + fireEvent.click(within(screen.getByRole('dialog', { name: 'Trip documents' })).getByRole('button', { name: 'off-box' })); + fireEvent.click(within(screen.getByTestId('m-storage-category-journey')).getByRole('button')); + fireEvent.click(within(screen.getByRole('dialog', { name: 'Journey photos' })).getByRole('button', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByRole('alertdialog'); + fireEvent.click(screen.getByRole('button', { name: 'Move existing objects' })); + await screen.findByText('Storage configuration saved'); + + await waitFor(() => expect(postCount).toBe(1)); + expect(migrationPosts[0]!.category).toBe('files'); + // At the moment the first POST arrives, exactly one POST has happened — + // and the second must not fire while the first is still running, no + // matter how many poll ticks elapse. + expect(postCount).toBe(1); + await new Promise((r) => setTimeout(r, 200)); + expect(postCount).toBe(1); + // Once the first turns terminal, the second is dequeued — the POST + // handler's own assertion (above) pins that it never arrives early. + firstDone = true; + await waitFor(() => expect(postCount).toBe(2), { timeout: 2000 }); + expect(migrationPosts[1]!.category).toBe('journey'); + }); + + it('FE-MOB-MSTOR-016: a done migration row shows the reclaimable line; a running one shows progress + cancel wired to DELETE', async () => { + let cancelledCategory: string | null = null; + const state = baseState(); + (state as StorageAdminState).migrations = [ + { + category: 'files', from: 'uploads-local', to: 'off-box', + status: 'running', done: 2, total: 5, copied: 1, skipped: 1, failed: 0, startedAt: 1, + }, + { + category: 'journey', from: 'uploads-local', to: 'off-box', + status: 'done', done: 4, total: 4, copied: 4, skipped: 0, failed: 0, startedAt: 1, finishedAt: 2, + reclaimable: { objects: 4, bytes: 4096 }, + }, + ]; + await renderPanel(state); + server.use( + http.delete('/api/admin/storage/migrations/files', () => { + cancelledCategory = 'files'; + return HttpResponse.json({ cancelled: true }); + }), + ); + expect(screen.getByText(/Moving Trip documents… 2\/5/)).toBeInTheDocument(); + expect(screen.getByText(/Move finished: 4 copied, 0 skipped/)).toBeInTheDocument(); + expect(screen.getByText(/4 objects \(4\.0 KB\) remain on uploads-local — reclaim manually/)).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Cancel move' })); + await waitFor(() => expect(cancelledCategory).toBe('files')); + }); + + it('FE-MOB-MSTOR-017: a failed save disarms the sync-prompt snapshot', async () => { + await renderPanel(); + server.use(http.put('/api/admin/storage', () => HttpResponse.json({ error: 'nope' }, { status: 400 }))); + fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Edit' })); + fireEvent.click(screen.getByRole('switch', { name: 'off-box' })); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); + fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); + await screen.findByText('nope'); + + // The draft-only mirror (never saved) still renders its "Sync now" — + // clicking it drives a genuine admin.state refresh (via refreshState()) + // that lands a world whose mirror targets grew. With the pending-prompt + // snapshot disarmed by the failed save, the sync prompt must never appear. + server.use( + http.post('/api/admin/storage/backends/backups-local-mirror/backfill', () => HttpResponse.json({ started: true })), + http.get('/api/admin/storage', () => HttpResponse.json(mirroredState())), + ); + fireEvent.click(within(screen.getByTestId('m-storage-backend-backups-local')).getByRole('button', { name: 'Sync now' })); + await new Promise((r) => setTimeout(r, 100)); + expect(screen.queryByText(/Existing objects are not replicated yet/)).not.toBeInTheDocument(); + }); +}); diff --git a/docker-compose.minio-test.yml b/docker-compose.minio-test.yml new file mode 100644 index 000000000..9ca5c68f2 --- /dev/null +++ b/docker-compose.minio-test.yml @@ -0,0 +1,64 @@ +# Manual, license-gated live-provider test for the env-gated S3 driver +# contract suite, for local/operator use. CI runs this exact suite for +# same-repo pull requests, via the MINIO_LICENSE_KEY repo secret (see +# .github/workflows/test.yml's s3-contract job) — this compose file stays +# the manual procedure for local runs and for fork PRs, which don't get +# repo secrets and so still need it run by hand. +# +# MinIO fully dropped the community line (quay.io/minio/minio gets no +# further updates). The maintained successor is AIStor +# (quay.io/minio/aistor/minio), which requires a license key to boot at all. +# +# REQUIRED: a `minio-licence.key` file at the repo root (same directory as +# this compose file). It is git-ignored and must never be committed — obtain +# your own free license key from MinIO (https://min.io) and place it there. +# The AIStor container will not start without it. +# +# Verified empirically against RELEASE.2026-08-07T18-34-35Z: +# - MINIO_ROOT_USER / MINIO_ROOT_PASSWORD still set the root credentials +# (unchanged from community MinIO). +# - Command shape: the image entrypoint expects `minio` as its first +# argument, so `command: minio server /data --license /minio.license` +# is passed through as-is. +# - `mc ready local` works as a healthcheck (mc ships in the image; curl +# does not). +# +# 127.0.0.1 is load-bearing on both sides: aws-lite picks path-style only +# for IP/localhost hosts (stock MinIO/AIStor only speaks path-style, no +# MINIO_DOMAIN here), and the port publish below binds 127.0.0.1 only — not +# 0.0.0.0 — so this fixed-credential test instance is never reachable from +# the LAN. +# +# Procedure: +# docker compose -f docker-compose.minio-test.yml up -d +# docker compose -f docker-compose.minio-test.yml ps # wait for "healthy" +# cd server && TREK_TEST_S3_ENDPOINT=http://127.0.0.1:9000 \ +# TREK_TEST_S3_BUCKET=trek-test TREK_TEST_S3_ACCESS_KEY_ID=trekci \ +# TREK_TEST_S3_SECRET_ACCESS_KEY=trekci-secret \ +# npx vitest run tests/unit/nest/storage/s3.driver.contract.test.ts +# (optional) UI check: with the app running against this instance, open +# Admin → Storage, add an S3 backend (endpoint http://127.0.0.1:9000, +# bucket trek-test, access key trekci, secret trekci-secret), press Test +# on its row and confirm "Connection OK". +# docker compose -f docker-compose.minio-test.yml down -v +services: + minio: + # Docker Hub's minio/minio repo is archived, and quay.io/minio/minio (the + # former community line) receives no further updates — MinIO dropped it + # entirely. AIStor is the maintained successor; it requires a license + # (see header above). Pinned tag. + image: quay.io/minio/aistor/minio:RELEASE.2026-08-07T18-34-35Z + command: minio server /data --license /minio.license + ports: + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" + volumes: + - ./minio-licence.key:/minio.license:ro + environment: + - MINIO_ROOT_USER=trekci + - MINIO_ROOT_PASSWORD=trekci-secret + healthcheck: + test: ["CMD", "mc", "ready", "local"] # curl was removed from the image + interval: 2s + timeout: 5s + retries: 15 diff --git a/docker-compose.yml b/docker-compose.yml index 6709c3d30..1e5ee4d53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: volumes: - ./data:/app/data - ./uploads:/app/uploads +# - ./storage-config.json:/app/data/storage-config.json:ro # optional seed-once storage provisioning (README → Storage) restart: unless-stopped healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] diff --git a/package-lock.json b/package-lock.json index a3ef93b67..867b0d6e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -394,6 +394,404 @@ "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", "license": "MIT" }, + "node_modules/@aws-lite/client": { + "version": "0.23.7", + "resolved": "https://registry.npmjs.org/@aws-lite/client/-/client-0.23.7.tgz", + "integrity": "sha512-Qo/lq96DS8UOg1yFccolMuKG1hKnNStXVYhzVl4tZk/7m+OtANVk2JB8iuYJISc4DF7Dm3Qy+e91L6BGJ1dfkw==", + "license": "Apache-2.0", + "workspaces": [ + "plugins/acm", + "plugins/acm/types", + "plugins/apigateway", + "plugins/apigateway/types", + "plugins/apigatewaymanagementapi", + "plugins/apigatewaymanagementapi/types", + "plugins/apigatewayv2", + "plugins/apigatewayv2/types", + "plugins/cloudformation", + "plugins/cloudformation/types", + "plugins/cloudfront", + "plugins/cloudfront/types", + "plugins/cloudwatch-logs", + "plugins/cloudwatch-logs/types", + "plugins/dynamodb", + "plugins/dynamodb/types", + "plugins/iam", + "plugins/iam/types", + "plugins/lambda", + "plugins/lambda/types", + "plugins/organizations", + "plugins/organizations/types", + "plugins/rds-data", + "plugins/rds-data/types", + "plugins/route53", + "plugins/route53/types", + "plugins/s3", + "plugins/s3/types", + "plugins/sns", + "plugins/sns/types", + "plugins/sqs", + "plugins/sqs/types", + "plugins/ssm", + "plugins/ssm/types", + "plugins/sts", + "plugins/sts/types" + ], + "dependencies": { + "aws4": "^1.13.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@aws-lite/s3": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@aws-lite/s3/-/s3-0.2.7.tgz", + "integrity": "sha512-NPQZXMcHIwrcyZWrHUeoyjl3353xIoYA/6XoUu4KJs2NQKqF6xx5YO+7nXE05vSSygNSgUHetaue/x2vr2430A==", + "license": "Apache-2.0", + "engines": { + "node": ">=16" + } + }, + "node_modules/@aws-lite/s3-types": { + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@aws-lite/s3-types/-/s3-types-0.2.8.tgz", + "integrity": "sha512-e+wf4m83Qk9d+4e/on3d6dSkqC2eueL6LM7iLpZQ+U4f6Ey5tRuvirqbj8iOQcUGxJpP7UiVXAv+P61WIql/OA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/client-s3": "3" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.28", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.28.tgz", + "integrity": "sha512-VCpnmyHQ1IH49ni3LXnQj7DPr7rmcJmzYeiCkYdCcfgNtkvOj38cdcL9lapBWoItZWFACJPFJlymqC7/gem3Gw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1113.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1113.0.tgz", + "integrity": "sha512-NRqdtohoMRyWkEeeznfG1KPN08dclCbl+HFuLPB2v8qPcgoNmTFlLKl9ELiR6hsGXQ4Ur3qvRnoJVEk8ND74pg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.28", + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/credential-provider-node": "^3.972.80", + "@aws-sdk/middleware-sdk-s3": "^3.972.74", + "@aws-sdk/signature-v4-multi-region": "^3.996.45", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.8.tgz", + "integrity": "sha512-7+Kcrkvrk9lM/m7jRhHpT4jCdvzGHsuaSRbF8TdzzkY1mRzp/Ogwf9c7H29k4gGhey0BBWhCWr16+t0J61gwmg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.4", + "@aws-sdk/xml-builder": "^3.972.39", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.31.1", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.69", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.69.tgz", + "integrity": "sha512-AreCFzcB4kH2HF9031Ot0jSJr3KXvRg6e8uDeub20JEVdZU3Bv0sTq1plc7VsT3KiqutlzH7l0j50UcCWHUioA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.71", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.71.tgz", + "integrity": "sha512-A8ObcqVmDMnk4F9NozZ7JwmUu9Q4xyBJkmyq1C5U+wNM9ht9J7+EuuyabsLWXZnOoTqFaJuYBYTKf5CTipkEjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.14.tgz", + "integrity": "sha512-7c+Wti2LsERNWMfm7ySz3/6RPopFW3Nmn7s63Xpcq6R/tRuY5hpvkHA2xVgi5ukJbvok9l0IDtVEvqTtg+X7dw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/credential-provider-env": "^3.972.69", + "@aws-sdk/credential-provider-http": "^3.972.71", + "@aws-sdk/credential-provider-login": "^3.972.76", + "@aws-sdk/credential-provider-process": "^3.972.69", + "@aws-sdk/credential-provider-sso": "^3.973.13", + "@aws-sdk/credential-provider-web-identity": "^3.972.75", + "@aws-sdk/nested-clients": "^3.997.43", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.76.tgz", + "integrity": "sha512-LVixwOnEJfrrfKHeZjBA8pIMTZjNDq8ak8VpcoWUuCJDrSnBNU8POJksULMgvN089P0MXtQYH2Zs627/MK1K0g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/nested-clients": "^3.997.43", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.80", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.80.tgz", + "integrity": "sha512-bE2qh8ww4iClO1jHsBXdOE8FUgzDbdxbyorNjSCoPSkQd51k3jODItuPZfuwcLHZqDXsH+bI4AMHhqtuyR7mSg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.69", + "@aws-sdk/credential-provider-http": "^3.972.71", + "@aws-sdk/credential-provider-ini": "^3.973.14", + "@aws-sdk/credential-provider-process": "^3.972.69", + "@aws-sdk/credential-provider-sso": "^3.973.13", + "@aws-sdk/credential-provider-web-identity": "^3.972.75", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.69", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.69.tgz", + "integrity": "sha512-9kpTNdZTrcqXTfhxM7fgl9Z68ek3Fu5oe3Yf+A/pJGibEqpgZxz2tSY7SinmyCIU2PJ+ygY4FPoBBnLpocMtrQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.13", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.13.tgz", + "integrity": "sha512-Oc81qauMPzUoTnAS2YKpNwY6sY/LUyQTEeaf6yP197WMxkEBQfcKLR1MFpD7+pNTubXnfkH6gwpji+Gc7iyD2Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/nested-clients": "^3.997.43", + "@aws-sdk/token-providers": "3.1111.0", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.75", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.75.tgz", + "integrity": "sha512-YPN6uoGDgjjjeVFZrcOeCJqmB6zpXoeeNgIjqe+DexJaWqdjVfCCe+VAZwli9Z2h8KhFW8oxkO39emQ1tyz/Mw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/nested-clients": "^3.997.43", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.74.tgz", + "integrity": "sha512-2lzoV2z2QO5KJZYGOCnIZ1WVQgzMECvwuzr1xb034a++8QW4U4eGrmC2u4yg1xvNv4TLL/Uv5DLyuAiw0b9z7Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/signature-v4-multi-region": "^3.996.45", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.43", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.43.tgz", + "integrity": "sha512-bit+VpqWNyi3wHxFoTsTliNXimCSL2r2OeDTm7ZrG+YsTZ2D7ofDJ6r/t9PVBn80i6/v0X2h9Tgw6QP2MAKfPw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/signature-v4-multi-region": "^3.996.45", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.45", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.45.tgz", + "integrity": "sha512-bBuyztukzXq6plzFGHAWiQt0QXo+HL8b8lX5cFTzkez/74PtS1c0qPFCIVuHkyoT+miH2qOjAcm1/yoro2ESPA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.4", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1111.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1111.0.tgz", + "integrity": "sha512-JfljgoVtl+s3Qy21n9a7Z48uCQaOXcN74KJ3TEQfPoB293GrXFSt6HSQJF1sTZ8c/5QedEvd3NjJQMO4u9qa5A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.8", + "@aws-sdk/nested-clients": "^3.997.43", + "@aws-sdk/types": "^3.974.4", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.4.tgz", + "integrity": "sha512-dSFDNG00MEz0/xl5gxL62giLd1iYyJsTxZ1I1DOj6lC+bbgLB4TRsYClJg3b62dhXT1uATzsTNXPnC+33EJV3A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.39", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.39.tgz", + "integrity": "sha512-FTti8DS5MMWXNUWiRwXAJeYS+0GHHiMy0+7XOhcwk63ILHmfS2UFy2z/HNpZCSOJJ3P3dnWY6hfYNW3DF0nXUA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -6774,6 +7172,93 @@ "node": ">=20.0.0" } }, + "node_modules/@smithy/core": { + "version": "3.33.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.2.tgz", + "integrity": "sha512-CUGXpnPkVdjUCbix+83sWLW9VFgQOm44MDOx/ihITJMAnOZKvL8YYIc7DR9pP/tZ8CIRvMiON/TucvygqbHO3w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.2.tgz", + "integrity": "sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.11.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.11.2.tgz", + "integrity": "sha512-avwAh9HM3h2lcfjvP3zYIZGf+XVgLQ91wOJ2qoFbNpW1UZeZb33aGlhTZvtkANHfcGhJroRY64525OjfgOg30g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.2.tgz", + "integrity": "sha512-P7Ki6px6OOrxVtx8K7nLmyx4SlXUW/uTKDdMG44UHefmPGSRMBKe2v+TM59WdLcpUIrBrnuCsIqiM2MbsZjmhw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.2.tgz", + "integrity": "sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -8668,6 +9153,12 @@ "node": ">=6.0.0" } }, + "node_modules/aws4": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", + "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", + "license": "MIT" + }, "node_modules/axios": { "version": "1.18.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.0.tgz", @@ -9025,6 +9516,13 @@ "node": ">= 0.8" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "dev": true, + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", @@ -21289,6 +21787,8 @@ "name": "@trek/server", "version": "3.4.1", "dependencies": { + "@aws-lite/client": "^0.23.7", + "@aws-lite/s3": "^0.2.7", "@modelcontextprotocol/sdk": "^1.30.0", "@nestjs/common": "^11.1.24", "@nestjs/config": "^4.0.4", @@ -21335,6 +21835,7 @@ "zod": "^4.3.6" }, "devDependencies": { + "@aws-lite/s3-types": "^0.2.8", "@eslint/js": "^10.0.1", "@nestjs/testing": "^11.1.24", "@swc/core": "^1.15.40", diff --git a/server/package.json b/server/package.json index e9d81749b..24125788d 100644 --- a/server/package.json +++ b/server/package.json @@ -25,6 +25,8 @@ "test:coverage": "vitest run --coverage" }, "dependencies": { + "@aws-lite/client": "^0.23.7", + "@aws-lite/s3": "^0.2.7", "@modelcontextprotocol/sdk": "^1.30.0", "@nestjs/common": "^11.1.24", "@nestjs/config": "^4.0.4", @@ -81,6 +83,7 @@ "file-type": "^21.3.4" }, "devDependencies": { + "@aws-lite/s3-types": "^0.2.8", "@eslint/js": "^10.0.1", "@nestjs/testing": "^11.1.24", "@swc/core": "^1.15.40", diff --git a/server/scripts/migrate-encryption.ts b/server/scripts/migrate-encryption.ts index d3e113614..5f080e471 100644 --- a/server/scripts/migrate-encryption.ts +++ b/server/scripts/migrate-encryption.ts @@ -326,6 +326,68 @@ async function main() { } } } + + // --- app_settings: storage.backends --- + // Storage backend secrets live encrypted INSIDE the storage.backends JSON + // array (BACKENDS_KEY in src/nest/storage/storage-registry.service.ts is + // its own app_settings row — 'storage.categories' is a sibling row with no + // secrets), on the fields the shared type registry marks `secret` (only + // options.secretAccessKey for s3 today; src/nest/storage/storage-secrets.ts + // encrypts it with the same enc:v1: scheme as the api-key columns above). + // This script can't import storageSecretFields from @trek/shared (kept + // independent of the app, see the header note), so the field list per + // backend type is pinned here. Miss one and that backend's secret stays + // encrypted under the old key after rotation, then fails to decrypt on the + // next read (StorageBackendError) — pinned by + // tests/unit/db/migrate-encryption-parity.test.ts. + const STORAGE_BACKEND_SECRET_FIELDS: Record = { + s3: ['secretAccessKey'], + }; + { + const row = db.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as + | { value: string } + | undefined; + if (row?.value) { + let backends: unknown; + try { + backends = JSON.parse(row.value); + } catch { + result.errors.push('app_settings.storage.backends: invalid JSON — skipping'); + backends = undefined; + } + if (backends !== undefined) { + if (Array.isArray(backends)) { + let changed = false; + for (const backend of backends as Record[]) { + if (!backend || typeof backend !== 'object') continue; + const type = typeof backend.type === 'string' ? backend.type : undefined; + const fields = type ? STORAGE_BACKEND_SECRET_FIELDS[type] : undefined; + if (!fields || fields.length === 0) continue; + const options = backend.options; + if (!options || typeof options !== 'object') continue; + const name = typeof backend.name === 'string' ? backend.name : '?'; + for (const field of fields) { + const raw = (options as Record)[field]; + if (typeof raw !== 'string' || !raw) continue; + const newVal = migrateApiKeyValue(raw, `app_settings.storage.backends[${name}].${field}`); + if (newVal !== null) { + (options as Record)[field] = newVal; + changed = true; + } + } + } + if (changed) { + db.prepare("UPDATE app_settings SET value = ? WHERE key = 'storage.backends'").run( + JSON.stringify(backends), + ); + } + } else { + result.errors.push('app_settings.storage.backends: not an array — skipping'); + } + } + } + } + // --- settings: per-user encrypted keys --- const encryptedSettingKeys = ['webhook_url', 'ntfy_token', 'mapbox_access_token']; const settingRows = db.prepare( diff --git a/server/src/app-config/README.md b/server/src/app-config/README.md index 3fa08fd14..e07d8be4a 100644 --- a/server/src/app-config/README.md +++ b/server/src/app-config/README.md @@ -68,9 +68,12 @@ PORT, HOST, TRUST_PROXY, SESSION_DURATION(_REMEMBER), MCP_SESSION_TTL, MCP_MAX_SESSION_PER_USER, MCP_SSE_KEEPALIVE, TREK_PLUGIN_RPC_*/LOG_*/MAX_RSS_MB, TREK_PLUGIN_REGISTRY_URL, TREK_WIKI_DIR*, TREK_PLACE_PHOTO_DIR, BACKUP_*, TRANSIT_API_URL, LOG_LEVEL*, ALLOW_INTERNAL_NETWORK*, DEFAULT_LANGUAGE, -TREK_DB_FILE, TREK_DB_JOURNAL_MODE, TREK_DB_SYNCHRONOUS, ENCRYPTION_KEY. +TREK_DB_FILE, TREK_DB_JOURNAL_MODE, TREK_DB_SYNCHRONOUS, ENCRYPTION_KEY**. (* frozen today because the consuming module captures it at import; tests that override these set them at file top, before the SUT import.) +(** ENCRYPTION_KEY is boot-stable for the cipher key material itself — +`src/config.ts` resolves and freezes it at process start, per the exemption +below.) **Runtime-toggled** (read live on every access via `readEnv()` / `RuntimeEnvService`; tests mutate these mid-lifetime): diff --git a/server/src/app-config/parsers.ts b/server/src/app-config/parsers.ts index d4709c244..2d75e4db3 100644 --- a/server/src/app-config/parsers.ts +++ b/server/src/app-config/parsers.ts @@ -46,6 +46,13 @@ export function positiveIntOr(raw: string | undefined, fallback: number): number return Number.isFinite(n) && n > 0 ? n : fallback; } +/** Integer ≥ 0 (zero is meaningful — e.g. retries disabled), else the fallback. */ +export function nonNegativeIntOr(raw: string | undefined, fallback: number): number { + if (raw === undefined || raw.trim() === '') return fallback; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n >= 0 ? n : fallback; +} + /** Comma-split + trim, null when unset — the websocket ALLOWED_ORIGINS variant (no empty-entry filter). */ export function csvList(raw: string | undefined): string[] | null { return raw ? raw.split(',').map((o) => o.trim()) : null; diff --git a/server/src/bootstrap.ts b/server/src/bootstrap.ts index 70b5d2d9e..02debeff5 100644 --- a/server/src/bootstrap.ts +++ b/server/src/bootstrap.ts @@ -16,6 +16,7 @@ import { validateBodyContracts } from './nest/common/validate-body-contracts'; import { validateRouteGuards } from './nest/common/validate-route-guards'; import { validateManagedRoutes } from './nest/common/validate-managed-routes'; import { TrekWsAdapter } from './nest/realtime/trek-ws.adapter'; +import { StorageService } from './nest/storage/storage.service'; /** * Builds the unified TREK NestJS application that serves the ENTIRE surface — the @@ -84,7 +85,10 @@ export async function buildApp(): Promise { // config instead of reading process.env itself. const http = app.get>(httpConfig.KEY); applyGlobalMiddleware(instance, { http }); - applyPlatformUploads(instance); + // Same pre-init consumption bridge as httpConfig above: the StorageService + // instance is resolvable before init, and the handlers only *register* here — + // per-request resolution runs after app.init() completed the registry load. + applyPlatformUploads(instance, app.get(StorageService)); // The SDK discovery router (+ its addon gate). Container-built so its deps // are injected (same pre-init consumption bridge as httpConfig above), but // applied here as a PATHLESS app.use: the SDK router matches absolute diff --git a/server/src/db/migrations.ts b/server/src/db/migrations.ts index ac2840b5d..27d1d700a 100644 --- a/server/src/db/migrations.ts +++ b/server/src/db/migrations.ts @@ -3895,8 +3895,8 @@ function runMigrations(db: Database.Database): void { // its day are on the same trip. The table carries no trip_id and its two // foreign keys only ask that the ids exist, so pairs that never belonged // together could accumulate; the writer refuses them now, and this clears - // whatever an older build let through. Appended LAST — the array is - // index-addressed against schema_version. + // whatever an older build let through. This slot keeps the index it shipped + // with on dev — the array is index-addressed against schema_version. () => { db.exec(` DELETE FROM reservation_day_positions @@ -4067,6 +4067,22 @@ function runMigrations(db: Database.Database): void { if (byName) update.run(byName, row.id); } }, + // Storage slice 2 — collab note attachments historically stored 'files/' + // in trip_files.filename while the file manager stored bare names in the same + // column; the storage layer addresses objects as category + bare name, so + // normalize the legacy rows. substr is 1-indexed: 7 drops the six chars of + // 'files/'. The LIKE guard makes it a no-op on already-bare rows. + // + // Appended LAST again, and for the same reason as the note this replaces: + // the array is index-addressed against schema_version, so a slot that has + // shipped in dev keeps the index it shipped with and anything from this + // branch goes after it. A database that already ran this migration at its + // pre-merge index replays it harmlessly (the LIKE guard) but skips whatever + // now occupies that index. Acceptable only because this branch has never + // been published; never do this with a released slot. + () => { + db.exec("UPDATE trip_files SET filename = substr(filename, 7) WHERE filename LIKE 'files/%'"); + }, ]; if (currentVersion < migrations.length) { diff --git a/server/src/index.ts b/server/src/index.ts index 017f8d52c..f8d458883 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,28 +10,16 @@ import http from 'node:http'; import type { INestApplication } from '@nestjs/common'; import { buildApp, getHttpServer } from './bootstrap'; -// Create upload and data directories on startup. -// Every uploads subdir the app writes to must be listed here (#1762): a dir -// created lazily on first upload needs write permission on the uploads mount -// point itself at request time, which fails with EACCES on Docker hosts whose -// bind-mounted uploads dir isn't writable by the runtime user — while every -// feature writing into an already-existing subdir keeps working. Creating them -// at boot turns that into one loud startup failure instead of a stray 500. -// Keep in sync with the `mkdir -p` list in the Dockerfile (guarded by -// tests/unit/uploads-dirs.test.ts). -const uploadsDir = path.join(__dirname, '../uploads'); -const photosDir = path.join(uploadsDir, 'photos'); -const filesDir = path.join(uploadsDir, 'files'); -const coversDir = path.join(uploadsDir, 'covers'); -const avatarsDir = path.join(uploadsDir, 'avatars'); -const journeyDir = path.join(uploadsDir, 'journey'); -const placesDir = path.join(uploadsDir, 'places'); -const backupsDir = path.join(__dirname, '../data/backups'); +// data/tmp is the driver-agnostic global scratch dir (restore-upload spool, +// mirror stream staging) and stays boot-created here. Driver-owned roots — the +// uploads/ category tree and data/backups — are ensured by LocalDriver.init on +// every storage-registry load (boot and reload(), storage-registry.service.ts), +// which keeps the #1762 EACCES guarantee: a non-writable bind mount still +// fails loudly at startup, inside app.init(), instead of as a stray 500 on +// first upload. The Dockerfile `mkdir -p` list is pinned to the registry's +// category prefixes by tests/unit/uploads-dirs.test.ts. const tmpDir = path.join(__dirname, '../data/tmp'); - -[uploadsDir, photosDir, filesDir, coversDir, avatarsDir, journeyDir, placesDir, backupsDir, tmpDir].forEach(dir => { - if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); -}); +if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir, { recursive: true }); import { getAppUrl, getMcpSafeUrl, readEnv } from './app-config'; diff --git a/server/src/middleware/globalMiddleware.ts b/server/src/middleware/globalMiddleware.ts index e5ee7611e..cb7652005 100644 --- a/server/src/middleware/globalMiddleware.ts +++ b/server/src/middleware/globalMiddleware.ts @@ -6,6 +6,41 @@ import cookieParser from 'cookie-parser'; import { readEnv, type AppEnv } from '../app-config'; import { logDebug, logWarn, logError } from '../nest/audit/audit-log.logger'; +/** + * Field names redacted from request-log query/body dumps (case-insensitive — + * lookup lowercases the key first). `secretaccesskey` covers the S3 storage + * backend's secret field (storage-admin PUT bodies land in the same debug + * log line as any other request body). + * + * Exported (module-level, not per-request) so it can be unit-tested directly + * without threading LOG_LEVEL through the real logger pipeline. + */ +export const SENSITIVE_KEYS = new Set([ + 'password', + 'new_password', + 'current_password', + 'token', + 'jwt', + 'authorization', + 'cookie', + 'client_secret', + 'mfa_token', + 'code', + 'smtp_pass', + 'secretaccesskey', +]); + +/** Deep-redacts every key in `SENSITIVE_KEYS` (case-insensitive) from a request-log value. */ +export function redact(value: unknown): unknown { + if (!value || typeof value !== 'object') return value; + if (Array.isArray(value)) return (value as unknown[]).map(redact); + const out: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + out[k] = SENSITIVE_KEYS.has(k.toLowerCase()) ? '[REDACTED]' : redact(v); + } + return out; +} + /** * The global request pipeline shared by the legacy Express app and the NestJS * instance. Both mount the *exact same* config so a request hitting a migrated @@ -172,18 +207,7 @@ export function applyGlobalMiddleware( app.use(cookieParser()); - // Request logging with sensitive field redaction - const SENSITIVE_KEYS = new Set(['password', 'new_password', 'current_password', 'token', 'jwt', 'authorization', 'cookie', 'client_secret', 'mfa_token', 'code', 'smtp_pass']); - const redact = (value: unknown): unknown => { - if (!value || typeof value !== 'object') return value; - if (Array.isArray(value)) return (value as unknown[]).map(redact); - const out: Record = {}; - for (const [k, v] of Object.entries(value as Record)) { - out[k] = SENSITIVE_KEYS.has(k.toLowerCase()) ? '[REDACTED]' : redact(v); - } - return out; - }; - + // Request logging with sensitive field redaction (SENSITIVE_KEYS/redact above) app.use((req: Request, res: Response, next: NextFunction) => { if (req.path === '/api/health') return next(); const startedAt = Date.now(); diff --git a/server/src/nest/README.md b/server/src/nest/README.md index d65a2890b..55ba83b31 100644 --- a/server/src/nest/README.md +++ b/server/src/nest/README.md @@ -125,6 +125,27 @@ tools, which never pass through an HTTP guard. In the five domains piloted for answer 404 to anonymous callers, so the addon check must beat the 401. - `common/demo-write.ts` — `isDemoWriteBlocked`, the demo-mode upload block. A function and not a guard on purpose; see the gotcha below. +- `storage/` — the storage abstraction and its admin surface. `storage.types.ts` + defines the driver contract (opaque keys, no rename/append — S3 constraints + baked in) plus the `ServedCategory` seam: `STORAGE_CATEGORIES` (shared, 8, + configurable) + the legacy `photos` directory that is served and backed up + but not configurable. `drivers/` (local, s3, mirror) all pass one contract + suite (`tests/unit/nest/storage/storage-driver.contract.ts`). + `storage-registry.service.ts` merges built-ins ← env (`place-photos-local`) + ← `app_settings` rows, validates semantics (references, no mirror nesting), + falls back to last-good/defaults at boot but NEVER on admin writes + (`preview()` runs the real build first), seeds once from + `data/storage-config.json`, and hot-swaps on `reload()`. + `storage.service.ts` is the facade every byte-path uses. + `storage-admin.{controller,service}.ts` (+ `storage-secrets.ts`, + `storage-probe.ts`) carry `api/admin/storage`: masked GET of the effective + world, full-document PUT (unmask → preview → encrypt → one transaction → + reload → audit), and per-target test probes with ephemeral drivers. Secrets + live `enc:v1:`-encrypted inside the `storage.backends` JSON (the implicit + key covers installs without an explicit `ENCRYPTION_KEY`). The client + presents mirrors as replicas-on-primary; the wire keeps explicit mirror + backends. `storage-jobs.service.ts` runs backfill and category-migration + jobs (one at a time, TTL'd statuses). - `memories/photo-provider.ts` + `providers/` — the PhotoProvider seam (#584). Dispatch to a photo backend was a `switch (photo.provider)` in three places (`photo-resolver.service.ts` twice, `journey-public.controller.ts` once, where diff --git a/server/src/nest/app.module.ts b/server/src/nest/app.module.ts index 4d4a88bc7..4cda70887 100644 --- a/server/src/nest/app.module.ts +++ b/server/src/nest/app.module.ts @@ -46,6 +46,7 @@ import { TripInviteModule } from './trip-invite/trip-invite.module'; import { TransitModule } from './transit/transit.module'; import { FeedsModule } from './feeds/feeds.module'; import { SettingsModule } from './settings/settings.module'; +import { StorageModule } from './storage/storage.module'; import { BackupModule } from './backup/backup.module'; import { BookingImportModule } from './booking-import/booking-import.module'; import { ReservationImportModule } from './reservation-import/reservation-import.module'; @@ -74,7 +75,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module'; * migrated. */ @Module({ - imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule], + imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule], providers: [ // Default-deny: a route is authenticated unless it carries @Public() or // @OptionalAuth(), or declares its own @UseGuards chain. Protection used to diff --git a/server/src/nest/audit/audit.service.ts b/server/src/nest/audit/audit.service.ts index 8e6fe9746..58ff13546 100644 --- a/server/src/nest/audit/audit.service.ts +++ b/server/src/nest/audit/audit.service.ts @@ -18,6 +18,11 @@ const ACTION_LABELS: Record = { 'admin.user_delete': 'deleted user', 'admin.plugin_retrust': "re-trusted a plugin's author signing key", 'admin.invite_create': 'created invite', + 'admin.storage_update': 'updated storage configuration', + 'admin.storage_test': 'tested a storage backend', + 'admin.storage_backfill': 'started a storage sync', + 'admin.storage_backfill_cancel': 'cancelled a storage sync', + 'admin.storage_stats_refresh': 'refreshed storage usage stats', 'immich.private_ip_configured': 'configured Immich with private IP', }; diff --git a/server/src/nest/auth/auth.controller.ts b/server/src/nest/auth/auth.controller.ts index c1074fb1b..e63a732df 100644 --- a/server/src/nest/auth/auth.controller.ts +++ b/server/src/nest/auth/auth.controller.ts @@ -17,15 +17,13 @@ import { import { FileInterceptor } from '@nestjs/platform-express'; import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; import type { Request, Response } from 'express'; import path from 'path'; -import fs from 'fs'; -import { v4 as uuid } from 'uuid'; import { AuthService } from './auth.service'; import { TokenService } from '../tokens/token.service'; import { UserProfileService } from './user-profile.service'; -import { avatarDir } from './auth.helpers'; +import { StorageService } from '../storage/storage.service'; import { ChangePasswordDto, MapsKeyUpdateDto, @@ -49,21 +47,17 @@ import { ManagedForbidden } from '../common/managed'; const WINDOW = 15 * 60 * 1000; const ALLOWED_AVATAR_EXTS = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; -const AVATAR_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { if (!fs.existsSync(avatarDir)) fs.mkdirSync(avatarDir, { recursive: true }); cb(null, avatarDir); }, - filename: (_req, file, cb) => cb(null, uuid() + path.extname(file.originalname)), - }), - limits: { fileSize: 5 * 1024 * 1024 }, - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - if (!file.mimetype.startsWith('image/') || !ALLOWED_AVATAR_EXTS.includes(ext)) { - const err: Error & { statusCode?: number } = new Error('Only image files (jpg, png, gif, webp) are allowed'); - err.statusCode = 400; - return cb(err, false); - } - cb(null, true); - }, +export const MAX_AVATAR_SIZE = 5 * 1024 * 1024; +// Consumed by auth.module.ts's MulterModule factory; the storage engine (spool +// destination + UUID filename) comes from the storage upload factory. +export const AVATAR_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + if (!file.mimetype.startsWith('image/') || !ALLOWED_AVATAR_EXTS.includes(ext)) { + const err: Error & { statusCode?: number } = new Error('Only image files (jpg, png, gif, webp) are allowed'); + err.statusCode = 400; + return cb(err); + } + cb(null, true); }; /** @@ -77,7 +71,7 @@ const AVATAR_UPLOAD = { @Controller('api/auth') @UseGuards(JwtAuthGuard) export class AuthController { - constructor(private readonly auth: AuthService, private readonly profile: UserProfileService, private readonly tokens: TokenService, private readonly rl: RateLimitService, private readonly audit: AuditService, private readonly env: RuntimeEnvService) {} + constructor(private readonly auth: AuthService, private readonly profile: UserProfileService, private readonly tokens: TokenService, private readonly rl: RateLimitService, private readonly audit: AuditService, private readonly env: RuntimeEnvService, private readonly storage: StorageService) {} private limit(bucket: string, req: Request, max: number): void { if (!this.rl.check(bucket, req.ip || 'unknown', max, WINDOW, Date.now())) { @@ -182,7 +176,7 @@ export class AuthController { @Post('avatar') @HttpCode(200) - @UseInterceptors(FileInterceptor('avatar', AVATAR_UPLOAD)) + @UseInterceptors(FileInterceptor('avatar')) async avatar(@CurrentUser() user: User, @UploadedFile() file: Express.Multer.File | undefined) { if (isDemoWriteBlocked(this.env, user.email)) { throw new HttpException(DEMO_WRITE_ERROR, 403); @@ -190,6 +184,9 @@ export class AuthController { if (!file) { throw new HttpException({ error: 'No image uploaded' }, 400); } + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before the DB row references the final path. + await this.storage.put('avatars', file.filename, { tmpPath: file.path }); return this.profile.saveAvatar(user.id, file.filename); } diff --git a/server/src/nest/auth/auth.helpers.ts b/server/src/nest/auth/auth.helpers.ts index 5b34bb11b..e3a76811a 100644 --- a/server/src/nest/auth/auth.helpers.ts +++ b/server/src/nest/auth/auth.helpers.ts @@ -1,7 +1,5 @@ import bcrypt from 'bcryptjs'; import crypto from 'crypto'; -import path from 'path'; -import fs from 'fs'; import { User } from '../../types'; import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; @@ -38,13 +36,6 @@ export const ADMIN_SETTINGS_KEYS = [ 'passkey_login', 'webauthn_rp_id', 'webauthn_origins', ]; -// One directory deeper than the legacy src/services location, so the extra -// '../' keeps resolving to server/uploads/avatars (same anchor the multer -// diskStorage in auth.controller.ts uses). Import-time mkdir on purpose -// (legacy parity): the upload dir must exist before the first request. -export const avatarDir = path.join(__dirname, '../../../uploads/avatars'); -if (!fs.existsSync(avatarDir)) fs.mkdirSync(avatarDir, { recursive: true }); - // --------------------------------------------------------------------------- // Helpers (exported for route-level use where needed) // --------------------------------------------------------------------------- diff --git a/server/src/nest/auth/auth.module.ts b/server/src/nest/auth/auth.module.ts index 059add436..2542227b8 100644 --- a/server/src/nest/auth/auth.module.ts +++ b/server/src/nest/auth/auth.module.ts @@ -18,6 +18,12 @@ import { MailerModule } from '../notifications/mailer/mailer.module'; import { PermissionsModule } from '../permissions/permissions.module'; import { TripMembershipModule } from '../trip-membership/trip-membership.module'; import { EphemeralTokenModule } from './ephemeral-token.module'; +import { MulterModule } from '@nestjs/platform-express'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { AVATAR_FILE_FILTER, MAX_AVATAR_SIZE } from './auth.controller'; +import { AllowedFileTypesModule } from '../files/allowed-file-types.module'; /** * Auth module — public flows (login/register/reset/mfa-verify/logout) and the @@ -43,7 +49,20 @@ import { EphemeralTokenModule } from './ephemeral-token.module'; * through auth.bridge.ts. */ @Module({ - imports: [EphemeralTokenModule, RateLimitModule, AuditModule, PermissionsModule, TripMembershipModule, MailerModule, AppConfigModule, TokensModule, BudgetModule], + imports: [ + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: (storage: StorageService) => + buildStorageUploadOptions(storage, { + category: 'avatars', + maxSize: MAX_AVATAR_SIZE, + fileFilter: AVATAR_FILE_FILTER, + }), + }), + StorageModule, + AllowedFileTypesModule, + EphemeralTokenModule, RateLimitModule, AuditModule, PermissionsModule, TripMembershipModule, MailerModule, AppConfigModule, TokensModule, BudgetModule], controllers: [AuthPublicController, AuthController, PasskeyController], providers: [AuthService, UserProfileService, RegistrationInvitesService, PasskeyService, UserCleanupService, WebauthnConfigService, AuthMcp], exports: [AuthService, RegistrationInvitesService, PasskeyService, UserCleanupService], diff --git a/server/src/nest/auth/auth.service.ts b/server/src/nest/auth/auth.service.ts index 31ce4528a..a18dab305 100644 --- a/server/src/nest/auth/auth.service.ts +++ b/server/src/nest/auth/auth.service.ts @@ -31,13 +31,13 @@ import { TripMembershipService } from '../trip-membership/trip-membership.servic import { WebauthnConfigService } from './webauthn-config.service'; import { setAuthCookie, clearAuthCookie } from '../common/cookie'; import { MailerService } from '../notifications/mailer/mailer.service'; +import { AllowedFileTypesService } from '../files/allowed-file-types.service'; import { getAppUrl } from '../../app-config'; import { ADMIN_SETTINGS_KEYS, BCRYPT_COST, DUMMY_PASSWORD_HASH, EMAIL_REGEX, - avatarDir, generateBackupCodes, hashBackupCodeBcrypt, mask_stored_api_key, @@ -128,6 +128,7 @@ export class AuthService { private readonly userCleanup: UserCleanupService, private readonly mailer: MailerService, private readonly tokens: EphemeralTokenService, + private readonly allowedFileTypes: AllowedFileTypesService, ) {} // Cookie @@ -290,7 +291,11 @@ export class AuthService { oidc_configured: oidcConfigured, oidc_display_name: oidcConfigured ? (oidcDisplayName || 'SSO') : undefined, require_mfa: requireMfaRow?.value === 'true', - allowed_file_types: this.db.get<{ value: string }>("SELECT value FROM app_settings WHERE key = 'allowed_file_types'")?.value || 'jpg,jpeg,png,gif,webp,heic,pdf,doc,docx,xls,xlsx,txt,csv', + // The canonical live-read: same query + DEFAULT_ALLOWED_EXTENSIONS + // fallback the upload filters use, so the client's picker and the + // server's acceptance can never drift (the historical inline copy here + // dropped pkpass, pkpasses, md and markdown). + allowed_file_types: this.allowedFileTypes.get(), // Whether the configuration belongs to whoever operates this install // rather than to its admin. The client uses it to stop offering settings // the server would refuse anyway — it is an honesty flag for the UI, never diff --git a/server/src/nest/auth/user-profile.service.ts b/server/src/nest/auth/user-profile.service.ts index b3b44b897..8276e7a36 100644 --- a/server/src/nest/auth/user-profile.service.ts +++ b/server/src/nest/auth/user-profile.service.ts @@ -1,11 +1,10 @@ import { Injectable } from '@nestjs/common'; -import path from 'path'; -import fs from 'fs'; import { readEnv, getAppUrl } from '../../app-config'; import { DatabaseService } from '../database/database.service'; +import { StorageService } from '../storage/storage.service'; import { decrypt_api_key, maybe_encrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { avatarUrl } from '../common/avatarUrl'; -import { EMAIL_REGEX, avatarDir, mask_stored_api_key } from './auth.helpers'; +import { EMAIL_REGEX, mask_stored_api_key } from './auth.helpers'; import { splitManagedKeys, MANAGED_LOCKED_PROFILE_KEYS } from '../common/managed'; import { INSTANCE_API_KEY_NAMES, @@ -33,7 +32,10 @@ import { User } from '../../types'; */ @Injectable() export class UserProfileService { - constructor(private readonly db: DatabaseService) {} + constructor( + private readonly db: DatabaseService, + private readonly storage: StorageService, + ) {} /** * On a centrally administered install the three key columns belong to the @@ -281,12 +283,12 @@ export class UserProfileService { async saveAvatar(userId: number, filename: string) { const current = this.db.get<{ avatar: string | null }>('SELECT avatar FROM users WHERE id = ?', userId); // Only a locally uploaded file has something to clean up. An OIDC picture URL - // (#1399) has no file on disk, so skip the rm — path.join on a URL is meaningless. + // (#1399) has no storage object, so skip the delete entirely. if (current?.avatar && !/^https:\/\//i.test(current.avatar)) { - // Fire-and-forget: leftover files are harmless; the DB update is - // the source of truth for which avatar is current. - const oldPath = path.join(avatarDir, current.avatar); - await fs.promises.rm(oldPath, { force: true }).catch(() => {}); + // Fire-and-forget parity: leftover objects are harmless; the DB update is + // the source of truth for which avatar is current. The catch also + // swallows a hostile stored value the central key validation rejects. + await this.storage.delete('avatars', current.avatar).catch(() => {}); } this.db.run('UPDATE users SET avatar = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', filename, userId); @@ -297,10 +299,9 @@ export class UserProfileService { async deleteAvatar(userId: number) { const current = this.db.get<{ avatar: string | null }>('SELECT avatar FROM users WHERE id = ?', userId); - // An OIDC picture URL (#1399) has no local file — only rm an uploaded one. + // An OIDC picture URL (#1399) has no storage object — only delete an uploaded one. if (current?.avatar && !/^https:\/\//i.test(current.avatar)) { - const filePath = path.join(avatarDir, current.avatar); - await fs.promises.rm(filePath, { force: true }).catch(() => {}); + await this.storage.delete('avatars', current.avatar).catch(() => {}); } this.db.run('UPDATE users SET avatar = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ?', userId); return { success: true }; diff --git a/server/src/nest/backup/auto-backup.job.ts b/server/src/nest/backup/auto-backup.job.ts index e74f490c7..bdcaef623 100644 --- a/server/src/nest/backup/auto-backup.job.ts +++ b/server/src/nest/backup/auto-backup.job.ts @@ -3,6 +3,7 @@ import { readEnv } from '../../app-config'; import { logInfo, logError } from '../audit/audit-log.logger'; import { BackupService } from './backup.service'; import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { StorageService } from '../storage/storage.service'; import { parseAutoBackupBody } from './backup.impl'; import { buildCronExpression, @@ -24,6 +25,7 @@ export class AutoBackupJob implements OnApplicationBootstrap { constructor( private readonly backup: BackupService, private readonly registrar: CronRegistrarService, + private readonly storage: StorageService, ) {} onApplicationBootstrap(): void { @@ -66,7 +68,7 @@ export class AutoBackupJob implements OnApplicationBootstrap { const settings = loadSettings(); if (settings.keep_days > 0) { - cleanupOldBackups(settings.keep_days); + await cleanupOldBackups(this.storage, settings.keep_days); } } diff --git a/server/src/nest/backup/auto-backup.settings.ts b/server/src/nest/backup/auto-backup.settings.ts index 176c574b0..960e389fe 100644 --- a/server/src/nest/backup/auto-backup.settings.ts +++ b/server/src/nest/backup/auto-backup.settings.ts @@ -1,16 +1,19 @@ import path from 'node:path'; import fs from 'node:fs'; import { logInfo, logError } from '../audit/audit-log.logger'; +import type { StorageService } from '../storage/storage.service'; /** * Auto-backup settings and retention — the pure half of the auto-backup cron * (moved from src/scheduler.ts). Stays a plain module beside backup.impl.ts - * for the same reason that does: file I/O against data/backup-settings.json - * and data/backups, no container state. AutoBackupJob owns the scheduling. + * for the same reason that does: file I/O against data/backup-settings.json, + * no container state. AutoBackupJob owns the scheduling and passes its + * injected StorageService into cleanupOldBackups — retention addresses the + * backups category through the facade so expired archives leave mirror + * replicas too. */ const dataDir = path.join(__dirname, '../../../data'); -const backupsDir = path.join(dataDir, 'backups'); const settingsFile = path.join(dataDir, 'backup-settings.json'); export const VALID_INTERVALS = ['hourly', 'daily', 'weekly', 'monthly']; @@ -70,16 +73,16 @@ function autoBackupTimestampMs(filename: string): number | null { return Number.isNaN(ms) ? null : ms; } -export function cleanupOldBackups(keepDays: number, now: number = Date.now()): void { +export async function cleanupOldBackups(storage: StorageService, keepDays: number, now: number = Date.now()): Promise { try { const cutoff = now - keepDays * 24 * 60 * 60 * 1000; - const files = fs.readdirSync(backupsDir).filter(f => f.startsWith('auto-backup-') && f.endsWith('.zip')); - for (const file of files) { - const filePath = path.join(backupsDir, file); - const ageMs = autoBackupTimestampMs(file) ?? fs.statSync(filePath).mtimeMs; + for await (const obj of storage.list('backups')) { + if (obj.key.includes('/')) continue; // list() recurses; retention is top-level-only like the readdir it replaces + if (!obj.key.startsWith('auto-backup-') || !obj.key.endsWith('.zip')) continue; // manual backup-*.zip is never auto-deleted + const ageMs = autoBackupTimestampMs(obj.key) ?? obj.mtimeMs; if (ageMs < cutoff) { - fs.unlinkSync(filePath); - logInfo(`Auto-Backup old backup deleted: ${file}`); + await storage.delete('backups', obj.key); // fans out to mirror replicas too + logInfo(`Auto-Backup old backup deleted: ${obj.key}`); } } } catch (err: unknown) { diff --git a/server/src/nest/backup/backup.controller.ts b/server/src/nest/backup/backup.controller.ts index 61d61d09e..b270d0080 100644 --- a/server/src/nest/backup/backup.controller.ts +++ b/server/src/nest/backup/backup.controller.ts @@ -27,19 +27,10 @@ import { CurrentUser } from '../auth/current-user.decorator'; import { AutoBackupSettingsDto } from './backup.dto'; import { getClientIp } from '../audit/client-ip'; import { AuditService } from '../audit/audit.service'; -import { getUploadTmpDir, MAX_BACKUP_UPLOAD_SIZE } from './backup.impl'; +import { StorageNotFoundError } from '../storage/storage.types'; import { ManagedForbidden, isManagedBlocked, MANAGED_FORBIDDEN_ERROR } from '../common/managed'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -const UPLOAD = { - dest: getUploadTmpDir(), - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - if (file.originalname.endsWith('.zip')) return cb(null, true); - cb(new Error('Only ZIP files allowed'), false); - }, - limits: { fileSize: MAX_BACKUP_UPLOAD_SIZE }, -}; - /** * /api/backup — admin-only database backup management (list, create, download, * restore from a stored or uploaded zip, auto-backup settings, delete). @@ -60,9 +51,9 @@ export class BackupController { ) {} @Get('list') - list() { + async list() { try { - return { backups: this.backup.listBackups() }; + return { backups: await this.backup.listBackups() }; } catch { throw new HttpException({ error: 'Error loading backups' }, 500); } @@ -84,14 +75,23 @@ export class BackupController { } @Get('download/:filename') - download(@Param('filename') filename: string, @Res() res: Response): void { + async download(@Param('filename') filename: string, @Res() res: Response): Promise { if (!this.backup.isValidBackupFilename(filename)) { throw new HttpException({ error: 'Invalid filename' }, 400); } - if (!this.backup.backupFileExists(filename)) { + if (!(await this.backup.backupFileExists(filename))) { throw new HttpException({ error: 'Backup not found' }, 404); } - res.download(this.backup.backupFilePath(filename), filename); + try { + await this.backup.sendBackupToResponse(filename, res); + } catch (err) { + // The pre-check above owns the normal miss; this only covers a delete + // racing between the check and the send. + if (err instanceof StorageNotFoundError) { + throw new HttpException({ error: 'Backup not found' }, 404); + } + throw err; + } } @ManagedForbidden('a restore replaces database and uploads, and the operator owns the recovery point') @@ -101,11 +101,11 @@ export class BackupController { if (!this.backup.isValidBackupFilename(filename)) { throw new HttpException({ error: 'Invalid filename' }, 400); } - if (!this.backup.backupFileExists(filename)) { + if (!(await this.backup.backupFileExists(filename))) { throw new HttpException({ error: 'Backup not found' }, 404); } try { - const result = await this.backup.restoreFromZip(this.backup.backupFilePath(filename)); + const result = await this.backup.restoreBackup(filename); if (!result.success) { throw new HttpException({ error: result.error }, result.status || 400); } @@ -123,7 +123,7 @@ export class BackupController { ) @Post('upload-restore') @HttpCode(200) // Express answers upload-restore with res.json (200). - @UseInterceptors(FileInterceptor('backup', UPLOAD)) + @UseInterceptors(FileInterceptor('backup')) async uploadRestore(@CurrentUser() user: User, @UploadedFile() file: Express.Multer.File | undefined, @Req() req: Request) { // Checked here rather than in the guard: a guard runs before the multipart // parser, so throwing there leaves the body unread and the client sees an @@ -177,14 +177,14 @@ export class BackupController { } @Delete(':filename') - remove(@CurrentUser() user: User, @Param('filename') filename: string, @Req() req: Request) { + async remove(@CurrentUser() user: User, @Param('filename') filename: string, @Req() req: Request) { if (!this.backup.isValidBackupFilename(filename)) { throw new HttpException({ error: 'Invalid filename' }, 400); } - if (!this.backup.backupFileExists(filename)) { + if (!(await this.backup.backupFileExists(filename))) { throw new HttpException({ error: 'Backup not found' }, 404); } - this.backup.deleteBackup(filename); + await this.backup.deleteBackup(filename); this.audit.writeAudit({ userId: user.id, action: 'backup.delete', resource: filename, ip: getClientIp(req) }); return { success: true }; } diff --git a/server/src/nest/backup/backup.impl.ts b/server/src/nest/backup/backup.impl.ts index c78f28ad6..16e91804f 100644 --- a/server/src/nest/backup/backup.impl.ts +++ b/server/src/nest/backup/backup.impl.ts @@ -1,6 +1,7 @@ import archiver from 'archiver'; import unzipper from 'unzipper'; import path from 'path'; +import { pipeline } from 'node:stream/promises'; import { readEnv } from '../../app-config'; import fs from 'fs'; import Database from 'better-sqlite3'; @@ -10,14 +11,15 @@ import { invalidatePermissionsCache } from '../permissions/permissions-cache'; import { pluginsCodeRoot, pluginsDataRoot } from '../plugins/paths'; import { stageExtractedPluginTrees, applyStagedRestoreNow } from '../plugins/plugin-backup'; import { snapshotAllPluginDataDbs } from '../plugins/host/plugin-data.service'; +import type { Response } from 'express'; +import type { StorageService } from '../storage/storage.service'; +import { StorageInvalidKeyError } from '../storage/storage.types'; // --------------------------------------------------------------------------- // Paths // --------------------------------------------------------------------------- const dataDir = path.join(__dirname, '../../../data'); -const backupsDir = path.join(dataDir, 'backups'); -const uploadsDir = path.join(__dirname, '../../../uploads'); // Compressed upload cap for restore archives. Defaults to 500 MB, raisable via // BACKUP_UPLOAD_LIMIT_MB for instances whose backups (uploads/ included) grow @@ -36,10 +38,6 @@ export const MAX_BACKUP_DECOMPRESSED_SIZE = backupEnv.maxDecompressedMb * 1024 * // Helpers // --------------------------------------------------------------------------- -export function ensureBackupsDir(): void { - if (!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true }); -} - export function formatSize(bytes: number): string { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; @@ -80,12 +78,19 @@ export function isValidBackupFilename(filename: string): boolean { return /^(?:auto-)?backup-[\w-]+\.zip$/.test(filename); } -export function backupFilePath(filename: string): string { - return path.join(backupsDir, filename); +export function backupFileExists(storage: StorageService, filename: string): Promise { + return storage.exists('backups', filename); } -export function backupFileExists(filename: string): boolean { - return fs.existsSync(path.join(backupsDir, filename)); +/** + * The codebase's only res.download becomes the storage equivalent: root-relative + * sendFile via sendToResponse, with res.download's attachment header rebuilt by + * hand (filenames are regex-gated ASCII — no encoding cases). + */ +export function sendBackupToResponse(storage: StorageService, filename: string, res: Response): Promise { + return storage.sendToResponse('backups', filename, res, { + disposition: `attachment; filename="${filename}"`, + }); } // --------------------------------------------------------------------------- @@ -122,27 +127,32 @@ export interface BackupInfo { created_at: string; } -export function listBackups(): BackupInfo[] { - ensureBackupsDir(); - return fs.readdirSync(backupsDir) - .filter(f => f.endsWith('.zip')) - .map(filename => { - const filePath = path.join(backupsDir, filename); - const stat = fs.statSync(filePath); - return { - filename, - size: stat.size, - sizeText: formatSize(stat.size), - created_at: stat.mtime.toISOString(), - }; - }) - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); +export async function listBackups(storage: StorageService): Promise { + const backups: BackupInfo[] = []; + for await (const obj of storage.list('backups')) { + // storage.list() recurses; the legacy readdir was single-level. Nested keys + // (a restore-* staging tree when data and uploads map to the same dir) and + // non-zip files must not surface. + if (obj.key.includes('/') || !obj.key.endsWith('.zip')) continue; + backups.push({ + filename: obj.key, + size: obj.size, + sizeText: formatSize(obj.size), + created_at: new Date(obj.mtimeMs).toISOString(), + }); + } + return backups.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); } // --------------------------------------------------------------------------- // Create backup // --------------------------------------------------------------------------- +/** The categories a backup archives — everything else under uploads/ is a + * re-derivable cache (photos-google, photos-trek) or not uploads at all + * (backups). Restore's rehydration walks the same list. */ +export const BACKUP_UPLOAD_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places', 'photos'] as const; + /** * Writes a full backup zip and returns its BackupInfo. * @@ -151,27 +161,71 @@ export function listBackups(): BackupInfo[] { * only auto-backup-*.zip, and the admin panel badges them as automatic. Manual * backups keep the default. */ -export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'): Promise { - ensureBackupsDir(); - +export async function createBackup(storage: StorageService, prefix: 'backup' | 'auto-backup' = 'backup'): Promise { const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); const filename = `${prefix}-${timestamp}.zip`; - const outputPath = path.join(backupsDir, filename); - // The scratch names carry the prefix too: a scheduled run and a manual one that - // start in the same second would otherwise share a snapshot path, and the first - // to finish would delete the other's staging copy mid-archive. - const pdataSnap = path.join(backupsDir, `.plugins-snap-${prefix}-${timestamp}`); - const dbSnap = path.join(backupsDir, `.travel-snap-${prefix}-${timestamp}.db`); + // All staging lives in the backups backend's own spool: same volume as the + // destination (the put commit stays an atomic rename) and crash leftovers are + // reaped by LocalDriver's boot spool-cleanup. The scratch names carry the + // prefix too: a scheduled run and a manual one that start in the same second + // would otherwise share a snapshot path, and the first to finish would delete + // the other's staging copy mid-archive. + const spoolDir = storage.spoolDirFor('backups'); + const zipSpool = path.join(spoolDir, `zip-build-${prefix}-${timestamp}`); + const pdataSnap = path.join(spoolDir, `plugins-snap-${prefix}-${timestamp}`); + const dbSnap = path.join(spoolDir, `travel-snap-${prefix}-${timestamp}.db`); + // Per-backup staging for uploads with no local path (a remote/S3 primary, or + // a local path that vanished between listing and archiving — see + // getLocalPathOrNull). Same spool as the rest of the build, same cleanup. + const stagingDir = path.join(spoolDir, `staging-${prefix}-${timestamp}`); try { try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) {} + // Enumerate the archived categories up front (the archiver reads entries + // lazily during finalize(), so the promise executor below must stay + // synchronous). Everything NOT in BACKUP_UPLOAD_CATEGORIES is excluded by + // construction: the re-derivable caches (photos-google in both + // TREK_PLACE_PHOTO_DIR modes, photos-trek) and backups itself are never + // enumerated, which is what makes the same-dir-misconfig guard (#1358) + // structural instead of pattern-based. + const uploadEntries: { absPath: string; name: string }[] = []; + for (const category of BACKUP_UPLOAD_CATEGORIES) { + for await (const obj of storage.list(category)) { + // In mode A the google/trek caches nest under the photos/ prefix — the + // category walk would sweep them back in without this skip. + if (category === 'photos' && (obj.key.startsWith('google/') || obj.key.startsWith('trek/'))) continue; + // Local path available (exists on disk right now — the fail-safe half + // of getLocalPathOrNull's contract) → push it directly: zero-copy, the + // default-install path. Otherwise (a remote/S3 primary, or a local + // path that vanished between the list() and here) stream the object + // into this backup's own staging dir so archiver has a real file to + // read lazily during finalize() — the temp file withLocalFile would + // have produced is gone by the time archiver gets to it. + const localPath = await storage.getLocalPathOrNull(category, obj.key); + if (localPath !== null) { + uploadEntries.push({ absPath: localPath, name: `uploads/${category}/${obj.key}` }); + continue; + } + const stagedPath = path.join(stagingDir, category, obj.key); + fs.mkdirSync(path.dirname(stagedPath), { recursive: true }); + const { stream } = await storage.getStream(category, obj.key); + await pipeline(stream, fs.createWriteStream(stagedPath)); + uploadEntries.push({ absPath: stagedPath, name: `uploads/${category}/${obj.key}` }); + } + } + await new Promise((resolve, reject) => { - const output = fs.createWriteStream(outputPath); + const output = fs.createWriteStream(zipSpool); const archive = archiver('zip', { zlib: { level: 9 } }); output.on('close', resolve); archive.on('error', reject); + // archiver emits 'warning' (not 'error') for entries it couldn't + // stat/read — a stale staged path, a permission error — and by default + // just skips them, silently dropping bytes from the backup. Fail the + // backup instead: a dropped entry must never pass as a success. + archive.on('warning', reject); archive.pipe(output); @@ -205,28 +259,7 @@ export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'): archive.file(encKeyPath, { name: '.encryption_key' }); } - if (fs.existsSync(uploadsDir)) { - // Exclude the place-photo and trek-memory caches: both are re-derivable - // (re-fetched on demand, keyed on stable ids) and would otherwise dominate - // backup size. Restores self-heal — the cache dirs are recreated at startup. - // - // Also exclude backups/ and restore-*/: these live under data/, not uploads/, - // but when an install maps data and uploads to the SAME directory (a - // misconfiguration, but a catastrophic one) the glob would otherwise sweep - // every prior backup zip into the new archive — each run embedding all - // previous runs, so size compounds without bound (see issue #1358). Ignoring - // them keeps the backup bounded regardless of how the volumes are mounted. - archive.glob( - '**/*', - { - cwd: uploadsDir, - ignore: ['photos/google/**', 'photos/trek/**', 'backups/**', 'restore-*/**'], - nodir: true, - dot: true, - }, - { prefix: 'uploads' }, - ); - } + for (const entry of uploadEntries) archive.file(entry.absPath, { name: entry.name }); // Plugin data — each plugin's own SQLite file and any blobs. This is the ONLY // copy of the user data a plugin holds, so it belongs in the backup. Checkpoint @@ -262,23 +295,29 @@ export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'): archive.finalize(); }); - const stat = fs.statSync(outputPath); + // The commit — and, under a mirror backend, the replica fan-out point. + await storage.put('backups', filename, { tmpPath: zipSpool }); + const stat = await storage.stat('backups', filename); + if (!stat) throw new Error(`Backup vanished after commit: ${filename}`); return { filename, size: stat.size, sizeText: formatSize(stat.size), - created_at: stat.birthtime.toISOString(), + created_at: new Date(stat.mtimeMs).toISOString(), }; } catch (err: unknown) { console.error('Backup error:', err); - if (fs.existsSync(outputPath)) fs.unlinkSync(outputPath); throw err; } finally { - // The snapshots were staging copies for the archiver only; drop them once streaming is - // done (the await above resolves on the output stream's 'close', so the archive is - // complete). + // put commits by rename, so on success the zip spool file is already gone; + // on failure these clean the half-built staging. The destination needs no + // unlink anymore — nothing lands there until put succeeds. (The await on + // the build promise resolves on the output stream's 'close', so the + // snapshots are no longer being read.) + fs.rmSync(zipSpool, { force: true }); fs.rmSync(pdataSnap, { recursive: true, force: true }); fs.rmSync(dbSnap, { force: true }); + fs.rmSync(stagingDir, { recursive: true, force: true }); } } @@ -292,7 +331,51 @@ export interface RestoreResult { status?: number; } -export async function restoreFromZip(zipPath: string): Promise { +/** Restore a zip that already sits in the backups store, reading it through + * the storage facade (primary-local in v1; a remote backend downloads to + * tempDir via withLocalFile — the seam is in place, resumability is not). */ +export function restoreBackup(storage: StorageService, filename: string): Promise { + return storage.withLocalFile('backups', filename, (zipPath) => restoreFromZip(storage, zipPath)); +} + +const isBackupCategory = (dir: string): dir is (typeof BACKUP_UPLOAD_CATEGORIES)[number] => + (BACKUP_UPLOAD_CATEGORIES as readonly string[]).includes(dir); + +/** + * Per-entry storage.put replaces the old wipe-and-cpSync (and with it the + * realpathSync symlinked-uploads workaround — the driver resolves its own + * root at init). Entries that cannot map to a storage key — an unknown + * top-level dir, or dot-segments from old `dot: true` archives — are skipped + * with a warning (2026-08-17 decision): new archives never contain them, and + * the category mapping stays structural in both directions. + */ +async function rehydrateUploads(storage: StorageService, extractedUploads: string): Promise { + const walk = (dir: string): string[] => + fs.readdirSync(dir, { withFileTypes: true }).flatMap((e) => { + const p = path.join(dir, e.name); + return e.isDirectory() ? walk(p) : e.isFile() ? [p] : []; + }); + for (const absPath of walk(extractedUploads)) { + const rel = path.relative(extractedUploads, absPath).split(path.sep).join('/'); + const slash = rel.indexOf('/'); + const top = slash === -1 ? '' : rel.slice(0, slash); + if (slash === -1 || !isBackupCategory(top)) { + console.warn(`Restore: skipping upload entry outside a storage category: ${rel}`); + continue; + } + try { + await storage.put(top, rel.slice(slash + 1), { tmpPath: absPath }); + } catch (err) { + if (err instanceof StorageInvalidKeyError) { + console.warn(`Restore: skipping upload entry with an invalid key: ${rel}`); + continue; + } + throw err; + } + } +} + +export async function restoreFromZip(storage: StorageService, zipPath: string): Promise { const extractDir = path.join(dataDir, `restore-${Date.now()}`); let reinitFailed: unknown = null; try { @@ -404,41 +487,6 @@ export async function restoreFromZip(zipPath: string): Promise { if (fs.existsSync(extractedEncKey)) { fs.copyFileSync(extractedEncKey, path.join(dataDir, '.encryption_key')); } - - const extractedUploads = path.join(extractDir, 'uploads'); - if (fs.existsSync(extractedUploads)) { - for (const sub of fs.readdirSync(uploadsDir)) { - const subPath = path.join(uploadsDir, sub); - if (fs.statSync(subPath).isDirectory()) { - for (const file of fs.readdirSync(subPath)) { - try { fs.unlinkSync(path.join(subPath, file)); } catch (e) {} - } - } - } - // Copy into the real directory behind uploadsDir. In Docker, uploadsDir - // (/app/server/uploads) is a symlink to the mounted /app/uploads volume; - // cpSync(dereference:false) would otherwise try to overwrite the symlink - // node with a directory and throw ERR_FS_CP_DIR_TO_NON_DIR. realpathSync - // is a no-op when uploadsDir is a plain directory (dev/non-Docker). - fs.cpSync(extractedUploads, fs.realpathSync(uploadsDir), { recursive: true, force: true }); - } - - // Plugin trees can't be swapped while the runtime holds their DBs open, so stage - // them beside the live trees, then ask the runtime to quiesce its plugins and apply - // the swap NOW. If the runtime isn't up (plugins disabled / restore during boot), - // the staging waits for the boot reconcile — with nothing running, no data diverges. - // Best-effort: a staging error must not fail an otherwise-good core restore. - try { - stageExtractedPluginTrees(extractDir); - // Quiesce regardless of whether trees were staged: the restored travel.db carries - // a different `plugins` table, so any plugin still running with its pre-restore - // identity/grants is now a ghost — invisible in the restored UI, unstoppable short - // of a process restart. applyStagedRestoreNow closes those handles; the tree swap - // it also performs is a no-op when nothing was staged (e.g. an older archive). - await applyStagedRestoreNow(); - } catch (e) { - console.error('Restore: staging plugin trees failed:', e); - } } finally { // Reopening the DB must always run (even if the copy above threw) so the // process is never left without a connection. Capture a reopen failure @@ -458,6 +506,54 @@ export async function restoreFromZip(zipPath: string): Promise { invalidatePermissionsCache(); } + if (!reinitFailed) { + // The registry reads storage.* app_settings through the DB handle that + // was just closed and reopened above — reload it now, AFTER reinitialize() + // and BEFORE any byte moves, so rehydrated uploads land where the RESTORED + // config says rather than the stale pre-restore one (audit #4). Skipped + // entirely when reopen failed: with no live DB handle the registry has + // nothing to read, and the restore is already reported as "restart + // required" below — rehydrating into a stale/guessed config would be worse. + storage.reloadConfig(); + + const extractedUploads = path.join(extractDir, 'uploads'); + if (fs.existsSync(extractedUploads)) { + // Parity with the legacy wipe: it unlinked one level deep only (nested + // files — journey/thumbs, photos/google — survived until overwritten by + // the copy) and swallowed per-file errors. + for (const category of BACKUP_UPLOAD_CATEGORIES) { + for await (const obj of storage.list(category)) { + if (obj.key.includes('/')) continue; + await storage.delete(category, obj.key).catch(() => { /* best-effort, as the old unlink loop was */ }); + } + } + await rehydrateUploads(storage, extractedUploads); + } + } + + // Plugin trees can't be swapped while the runtime holds their DBs open, so stage + // them beside the live trees, then ask the runtime to quiesce its plugins and apply + // the swap NOW. If the runtime isn't up (plugins disabled / restore during boot), + // the staging waits for the boot reconcile — with nothing running, no data diverges. + // Best-effort: a staging error must not fail an otherwise-good core restore. Runs + // UNCONDITIONALLY, even when reinitFailed — unlike reload/rehydration this is pure + // filesystem staging with no DB dependency (plugin-backup.ts), so a failed reopen + // must not cost the archive's plugin data: extractDir is unlinked right below, and + // an un-staged tree there would be gone for good with no recovery path. + try { + stageExtractedPluginTrees(extractDir); + // Quiesce regardless of whether trees were staged: the restored travel.db carries + // a different `plugins` table, so any plugin still running with its pre-restore + // identity/grants is now a ghost — invisible in the restored UI, unstoppable short + // of a process restart. applyStagedRestoreNow closes those handles; the tree swap + // it also performs is a no-op when nothing was staged (e.g. an older archive). It + // degrades gracefully when the DB isn't reopened, same as any other best-effort + // failure here. + await applyStagedRestoreNow(); + } catch (e) { + console.error('Restore: staging plugin trees failed:', e); + } + fs.rmSync(extractDir, { recursive: true, force: true }); if (reinitFailed) { console.error('Restore: database reopen failed after file swap:', reinitFailed); @@ -482,15 +578,7 @@ export async function restoreFromZip(zipPath: string): Promise { // Delete backup // --------------------------------------------------------------------------- -export function deleteBackup(filename: string): void { - const filePath = path.join(backupsDir, filename); - fs.unlinkSync(filePath); +export function deleteBackup(storage: StorageService, filename: string): Promise { + return storage.delete('backups', filename); } -// --------------------------------------------------------------------------- -// Upload config (multer dest) -// --------------------------------------------------------------------------- - -export function getUploadTmpDir(): string { - return path.join(dataDir, 'tmp/'); -} diff --git a/server/src/nest/backup/backup.module.ts b/server/src/nest/backup/backup.module.ts index 432430cd1..89e92d9d1 100644 --- a/server/src/nest/backup/backup.module.ts +++ b/server/src/nest/backup/backup.module.ts @@ -1,13 +1,45 @@ import { Module } from '@nestjs/common'; +import { MulterModule } from '@nestjs/platform-express'; +import type { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface'; import { BackupController } from './backup.controller'; import { BackupService } from './backup.service'; import { AutoBackupJob } from './auto-backup.job'; import { AuditModule } from '../audit/audit.module'; import { SchedulingModule } from '../scheduling/scheduling.module'; import { AppConfigModule } from '../app-config/app-config.module'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { MAX_BACKUP_UPLOAD_SIZE } from './backup.impl'; + +/** + * Multer options for the restore upload. The uploaded zip is a restore INPUT, + * not a stored object — it gets no storage category and spools to the + * driver-agnostic scratch dir (data/tmp), where the handler consumes and + * unlinks it. Resolved through DI so no path is computed at module load. + */ +export function buildBackupUploadOptions(storage: StorageService): MulterOptions { + return { + dest: storage.tempDir(), + fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { + if (file.originalname.endsWith('.zip')) return cb(null, true); + cb(new Error('Only ZIP files allowed'), false); + }, + limits: { fileSize: MAX_BACKUP_UPLOAD_SIZE }, + }; +} @Module({ - imports: [AppConfigModule, AuditModule, SchedulingModule], + imports: [ + AppConfigModule, + AuditModule, + SchedulingModule, + StorageModule, + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: buildBackupUploadOptions, + }), + ], controllers: [BackupController], providers: [BackupService, AutoBackupJob], }) diff --git a/server/src/nest/backup/backup.service.ts b/server/src/nest/backup/backup.service.ts index e112a85c1..9b5a3b580 100644 --- a/server/src/nest/backup/backup.service.ts +++ b/server/src/nest/backup/backup.service.ts @@ -1,4 +1,6 @@ import { Injectable } from '@nestjs/common'; +import type { Response } from 'express'; +import { StorageService } from '../storage/storage.service'; import * as svc from './backup.impl'; /** @@ -13,14 +15,17 @@ import * as svc from './backup.impl'; */ @Injectable() export class BackupService { - listBackups() { return svc.listBackups(); } - createBackup(prefix?: 'backup' | 'auto-backup') { return svc.createBackup(prefix); } - restoreFromZip(zipPath: string) { return svc.restoreFromZip(zipPath); } - deleteBackup(filename: string) { return svc.deleteBackup(filename); } + constructor(private readonly storage: StorageService) {} + + listBackups() { return svc.listBackups(this.storage); } + createBackup(prefix?: 'backup' | 'auto-backup') { return svc.createBackup(this.storage, prefix); } + restoreFromZip(zipPath: string) { return svc.restoreFromZip(this.storage, zipPath); } + restoreBackup(filename: string) { return svc.restoreBackup(this.storage, filename); } + deleteBackup(filename: string) { return svc.deleteBackup(this.storage, filename); } isValidBackupFilename(filename: string) { return svc.isValidBackupFilename(filename); } - backupFilePath(filename: string) { return svc.backupFilePath(filename); } - backupFileExists(filename: string) { return svc.backupFileExists(filename); } + backupFileExists(filename: string) { return svc.backupFileExists(this.storage, filename); } + sendBackupToResponse(filename: string, res: Response) { return svc.sendBackupToResponse(this.storage, filename, res); } checkRateLimit(key: string, maxAttempts: number, windowMs: number) { return svc.checkRateLimit(key, maxAttempts, windowMs); } get rateWindow() { return svc.BACKUP_RATE_WINDOW; } diff --git a/server/src/nest/collab/collab.controller.ts b/server/src/nest/collab/collab.controller.ts index 526cab011..a8a54bb7e 100644 --- a/server/src/nest/collab/collab.controller.ts +++ b/server/src/nest/collab/collab.controller.ts @@ -15,12 +15,11 @@ import { UseInterceptors, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; import path from 'path'; -import fs from 'fs'; -import { v4 as uuidv4 } from 'uuid'; import type { User } from '../../types'; import { CollabService } from './collab.service'; +import { StorageService } from '../storage/storage.service'; import { CollabNoteCreateDto, CollabNoteUpdateDto, @@ -34,24 +33,18 @@ import { CurrentUser } from '../auth/current-user.decorator'; import { RequirePermission, TripAccessGuard } from '../permissions/trip-access.guard'; import { BLOCKED_EXTENSIONS } from '../files/files.constants'; -const MAX_NOTE_FILE_SIZE = 50 * 1024 * 1024; -const filesDir = path.join(__dirname, '../../../uploads/files'); -const NOTE_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { if (!fs.existsSync(filesDir)) fs.mkdirSync(filesDir, { recursive: true }); cb(null, filesDir); }, - filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`), - }), - limits: { fileSize: MAX_NOTE_FILE_SIZE }, - defParamCharset: 'utf8', // parity with legacy routes/collab.ts — preserve non-ASCII original filenames - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg') || file.mimetype.includes('html') || file.mimetype.includes('javascript')) { - const err: Error & { statusCode?: number } = new Error('File type not allowed'); - err.statusCode = 400; - return cb(err, false); - } - cb(null, true); - }, +export const MAX_NOTE_FILE_SIZE = 50 * 1024 * 1024; +// Consumed by collab.module.ts's MulterModule factory; the rest of the multer +// options (spool destination, filename, limits) come from the storage upload +// factory. +export const collabNoteFileFilter: Options['fileFilter'] = (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg') || file.mimetype.includes('html') || file.mimetype.includes('javascript')) { + const err: Error & { statusCode?: number } = new Error('File type not allowed'); + err.statusCode = 400; + return cb(err); + } + cb(null, true); }; /** @@ -78,7 +71,10 @@ const NOTE_UPLOAD = { // caller then sees ECONNRESET instead of the 404). Same as files.controller.ts. @UseGuards(JwtAuthGuard) export class CollabController { - constructor(private readonly collab: CollabService) {} + constructor( + private readonly collab: CollabService, + private readonly storage: StorageService, + ) {} private requireTrip(tripId: string, user: User) { const trip = this.collab.verifyTripAccess(tripId, user.id); @@ -139,8 +135,8 @@ export class CollabController { @UseGuards(TripAccessGuard) @RequirePermission('collab_edit') @Delete('notes/:id') - deleteNote(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) { - if (!this.collab.deleteNote(tripId, id)) { + async deleteNote(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) { + if (!(await this.collab.deleteNote(tripId, id))) { throw new HttpException({ error: 'Note not found' }, 404); } this.collab.broadcast(tripId, 'collab:note:deleted', { noteId: Number(id) }, socketId); @@ -148,8 +144,8 @@ export class CollabController { } @Post('notes/:id/files') - @UseInterceptors(FileInterceptor('file', NOTE_UPLOAD)) - addNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) { + @UseInterceptors(FileInterceptor('file')) + async addNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) { const trip = this.requireTrip(tripId, user); if (!this.collab.canUploadFiles(trip, user)) { throw new HttpException({ error: 'No permission to upload files' }, 403); @@ -157,6 +153,9 @@ export class CollabController { if (!file) { throw new HttpException({ error: 'No file uploaded' }, 400); } + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before the DB row references the final path. + await this.storage.put('files', file.filename, { tmpPath: file.path }); const result = this.collab.addNoteFile(tripId, id, file); if (!result) { throw new HttpException({ error: 'Note not found' }, 404); @@ -168,8 +167,8 @@ export class CollabController { @UseGuards(TripAccessGuard) @RequirePermission('collab_edit') @Delete('notes/:id/files/:fileId') - deleteNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Param('fileId') fileId: string, @Headers('x-socket-id') socketId?: string) { - if (!this.collab.deleteNoteFile(tripId, id, fileId)) { + async deleteNoteFile(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Param('fileId') fileId: string, @Headers('x-socket-id') socketId?: string) { + if (!(await this.collab.deleteNoteFile(tripId, id, fileId))) { throw new HttpException({ error: 'File not found' }, 404); } this.collab.broadcast(tripId, 'collab:note:updated', { note: this.collab.getFormattedNoteById(tripId, id) }, socketId); diff --git a/server/src/nest/collab/collab.mcp.ts b/server/src/nest/collab/collab.mcp.ts index 7a4bec645..b6c4893c8 100644 --- a/server/src/nest/collab/collab.mcp.ts +++ b/server/src/nest/collab/collab.mcp.ts @@ -145,7 +145,7 @@ export class CollabMcp { if (this.auth.isDemoUser(ctx.userId)) return demoDenied(); if (!this.collab.verifyTripAccess(tripId, ctx.userId)) return noAccess(); if (!this.guards.hasTripPermission('collab_edit', tripId, ctx.userId)) return permissionDenied(); - const deleted = this.collab.deleteNote(tripId, noteId); + const deleted = await this.collab.deleteNote(tripId, noteId); if (!deleted) return errorResult('Note not found.'); this.guards.safeBroadcast(tripId, 'collab:note:deleted', { noteId }); return ok({ success: true }); diff --git a/server/src/nest/collab/collab.module.ts b/server/src/nest/collab/collab.module.ts index a8f3b440d..dcb148297 100644 --- a/server/src/nest/collab/collab.module.ts +++ b/server/src/nest/collab/collab.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; -import { CollabController } from './collab.controller'; +import { MulterModule } from '@nestjs/platform-express'; +import { CollabController, collabNoteFileFilter, MAX_NOTE_FILE_SIZE } from './collab.controller'; import { CollabService } from './collab.service'; import { CollabRpc } from './collab.rpc'; import { PluginGuardsModule } from '../plugins/host/plugin-guards.module'; @@ -10,9 +11,25 @@ import { AuthModule } from '../auth/auth.module'; import { AddonsModule } from '../addons/addons.module'; import { NotificationsModule } from '../notifications/notifications.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; @Module({ - imports: [McpSharedModule, NotificationsModule, PermissionsModule, AuthModule, RealtimeModule, PluginGuardsModule, AddonsModule], + imports: [ + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: (storage: StorageService) => + buildStorageUploadOptions(storage, { + category: 'files', + maxSize: MAX_NOTE_FILE_SIZE, + defParamCharset: 'utf8', // parity with legacy routes/collab.ts — preserve non-ASCII original filenames + fileFilter: collabNoteFileFilter, + }), + }), + StorageModule, + McpSharedModule, NotificationsModule, PermissionsModule, AuthModule, RealtimeModule, PluginGuardsModule, AddonsModule], controllers: [CollabController], providers: [CollabService, CollabMcp, CollabRpc], // For in-container consumers (CollabRpc). diff --git a/server/src/nest/collab/collab.service.ts b/server/src/nest/collab/collab.service.ts index 5b55f9e5b..efd1b2d1b 100644 --- a/server/src/nest/collab/collab.service.ts +++ b/server/src/nest/collab/collab.service.ts @@ -1,5 +1,4 @@ import path from 'path'; -import fs from 'fs'; import { Injectable } from '@nestjs/common'; import { DatabaseService, type TripAccess } from '../database/database.service'; import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared'; @@ -9,6 +8,7 @@ import { avatarUrl } from '../common/avatarUrl'; import { checkSsrf, createPinnedDispatcher } from '../../utils/ssrfGuard'; import type { CollabNote, CollabPoll, CollabMessage, TripFile, User } from '../../types'; import { NotificationsService } from '../notifications/notifications.service'; +import { StorageService } from '../storage/storage.service'; type Trip = TripAccess; @@ -69,6 +69,7 @@ export class CollabService { private readonly permissions: PermissionsService, private readonly realtime: RealtimeService, private readonly notifications: NotificationsService, + private readonly storage: StorageService, ) {} verifyTripAccess(tripId: string | number, userId: number) { @@ -193,16 +194,17 @@ export class CollabService { return this.formatNote(note); } - deleteNote(tripId: string | number, noteId: string | number): boolean { + async deleteNote(tripId: string | number, noteId: string | number): Promise { const existing = this.db.get('SELECT id FROM collab_notes WHERE id = ? AND trip_id = ?', noteId, tripId); if (!existing) return false; - // Clean up attached files from disk (unlink-first is intentional — a - // failed row delete leaves dangling rows, never orphaned files). + // Clean up attached objects first (delete-first is intentional — a failed + // row delete leaves dangling rows, never orphaned files). basename() + // tolerates any legacy 'files/'-prefixed row the boot migration has not + // seen. const noteFiles = this.db.all('SELECT id, filename FROM trip_files WHERE note_id = ?', noteId); for (const f of noteFiles) { - const filePath = path.join(__dirname, '../../../uploads', f.filename); - try { fs.unlinkSync(filePath); } catch { /* ignore */ } + await this.storage.delete('files', path.basename(f.filename)).catch(() => { /* ignore */ }); } this.db.transaction(() => { this.db.run('DELETE FROM trip_files WHERE note_id = ?', noteId); @@ -221,7 +223,7 @@ export class CollabService { const result = this.db.run( 'INSERT INTO trip_files (trip_id, note_id, filename, original_name, file_size, mime_type) VALUES (?, ?, ?, ?, ?, ?)', - tripId, noteId, `files/${file.filename}`, file.originalname, file.size, file.mimetype + tripId, noteId, file.filename, file.originalname, file.size, file.mimetype ); const saved = this.db.get('SELECT * FROM trip_files WHERE id = ?', result.lastInsertRowid)!; @@ -234,7 +236,7 @@ export class CollabService { return this.formatNote(note); } - deleteNoteFile(tripId: string | number, noteId: string | number, fileId: string | number): boolean { + async deleteNoteFile(tripId: string | number, noteId: string | number, fileId: string | number): Promise { // Scope to the trip — like every sibling collab op — so a caller authorized for THEIR // trip can't delete a note-file that belongs to someone else's trip (IDOR). trip_files // carries trip_id, so this ties the deleted object to the URL's :tripId the controller @@ -242,8 +244,7 @@ export class CollabService { const file = this.db.get('SELECT * FROM trip_files WHERE id = ? AND note_id = ? AND trip_id = ?', fileId, noteId, tripId); if (!file) return false; - const filePath = path.join(__dirname, '../../../uploads', file.filename); - try { fs.unlinkSync(filePath); } catch { /* ignore */ } + await this.storage.delete('files', path.basename(file.filename)).catch(() => { /* ignore */ }); this.db.run('DELETE FROM trip_files WHERE id = ?', fileId); return true; diff --git a/server/src/nest/collections/collections.controller.ts b/server/src/nest/collections/collections.controller.ts index e1f85a729..98de918c1 100644 --- a/server/src/nest/collections/collections.controller.ts +++ b/server/src/nest/collections/collections.controller.ts @@ -18,11 +18,8 @@ import { import { FileInterceptor } from '@nestjs/platform-express'; import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -import type { Request } from 'express'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; import path from 'path'; -import fs from 'fs'; -import { v4 as uuidv4 } from 'uuid'; import type { User } from '../../types'; import { CollectionsService } from './collections.service'; import { AddonGuard } from '../addons/addon.guard'; @@ -30,7 +27,8 @@ import { RequireAddon } from '../addons/require-addon.decorator'; import { ADDON_IDS } from '../../addons'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { CurrentUser } from '../auth/current-user.decorator'; -import { PLACE_IMAGE_UPLOAD } from '../common/place-image-upload'; +import { PLACE_IMAGE_FILE_FILTER } from '../common/place-image-upload'; +import { StorageService } from '../storage/storage.service'; import { placeImageUrl } from '../places/place-image'; import { CollectionCreateDto, @@ -56,23 +54,16 @@ import { } from './collections.dto'; import { PlaceRatingDto } from '../places/places.dto'; -const MAX_COVER_SIZE = 20 * 1024 * 1024; -const coversDir = path.join(__dirname, '../../../uploads/covers'); -const COVER_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { - if (!fs.existsSync(coversDir)) fs.mkdirSync(coversDir, { recursive: true }); - cb(null, coversDir); - }, - filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`), - }), - limits: { fileSize: MAX_COVER_SIZE }, - fileFilter: (_req: Request, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; - if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true); - else cb(new Error('Only jpg, png, gif, webp images allowed'), false); - }, +export const MAX_COVER_SIZE = 20 * 1024 * 1024; +// Duplicated on purpose from trips.controller.ts (historical parity — no +// drive-by consolidation). Quirk preserved: a plain Error without statusCode +// maps to 500, not 400. Passed inline on the cover route; the storage engine +// comes from collections.module.ts's storage-upload factory options. +const COVER_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; + if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true); + else cb(new Error('Only jpg, png, gif, webp images allowed')); }; /** @@ -89,7 +80,7 @@ const COVER_UPLOAD = { @UseGuards(AddonGuard, JwtAuthGuard) @RequireAddon(ADDON_IDS.COLLECTIONS, 'Collections') export class CollectionsController { - constructor(private readonly collections: CollectionsService, private readonly env: RuntimeEnvService) {} + constructor(private readonly collections: CollectionsService, private readonly env: RuntimeEnvService, private readonly storage: StorageService) {} // ── Lists ───────────────────────────────────────────────────────────────── @Get() @@ -130,8 +121,8 @@ export class CollectionsController { @Post('places/delete-many') @HttpCode(200) - deleteMany(@CurrentUser() user: User, @Body() body: CollectionDeleteManyDto, @Headers('x-socket-id') socketId?: string) { - return { deleted: this.collections.deletePlacesMany(user.id, body.ids, socketId) }; + async deleteMany(@CurrentUser() user: User, @Body() body: CollectionDeleteManyDto, @Headers('x-socket-id') socketId?: string) { + return { deleted: await this.collections.deletePlacesMany(user.id, body.ids, socketId) }; } @Post('places/status-many') @@ -184,8 +175,8 @@ export class CollectionsController { @Post('places/:pid/image') @HttpCode(200) - @UseInterceptors(FileInterceptor('image', PLACE_IMAGE_UPLOAD)) - uploadPlaceImage( + @UseInterceptors(FileInterceptor('image', { fileFilter: PLACE_IMAGE_FILE_FILTER })) + async uploadPlaceImage( @CurrentUser() user: User, @Param('pid') pid: string, @UploadedFile() file: Express.Multer.File | undefined, @@ -195,12 +186,15 @@ export class CollectionsController { throw new HttpException(DEMO_WRITE_ERROR, 403); } if (!file) throw new HttpException({ error: 'No image uploaded' }, 400); + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before the DB row references the final path. + await this.storage.put('places', file.filename, { tmpPath: file.path }); return this.collections.setPlaceImage(user.id, Number(pid), placeImageUrl(file.filename), socketId); } @Delete('places/:pid') - deletePlace(@CurrentUser() user: User, @Param('pid') pid: string, @Headers('x-socket-id') socketId?: string) { - this.collections.deletePlace(user.id, Number(pid), socketId); + async deletePlace(@CurrentUser() user: User, @Param('pid') pid: string, @Headers('x-socket-id') socketId?: string) { + await this.collections.deletePlace(user.id, Number(pid), socketId); return { success: true }; } @@ -360,12 +354,15 @@ export class CollectionsController { } @Post(':id/cover') - @UseInterceptors(FileInterceptor('cover', COVER_UPLOAD)) - uploadCover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) { + @UseInterceptors(FileInterceptor('cover', { fileFilter: COVER_FILE_FILTER })) + async uploadCover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined, @Headers('x-socket-id') socketId?: string) { if (isDemoWriteBlocked(this.env, user.email)) { throw new HttpException(DEMO_WRITE_ERROR, 403); } if (!file) throw new HttpException({ error: 'No image uploaded' }, 400); + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before the DB row references the final path. + await this.storage.put('covers', file.filename, { tmpPath: file.path }); const coverUrl = `/uploads/covers/${file.filename}`; return this.collections.setCollectionCover(user.id, Number(id), coverUrl, socketId); } diff --git a/server/src/nest/collections/collections.mcp.ts b/server/src/nest/collections/collections.mcp.ts index 23e454a50..1d6e15597 100644 --- a/server/src/nest/collections/collections.mcp.ts +++ b/server/src/nest/collections/collections.mcp.ts @@ -211,7 +211,7 @@ export class CollectionsMcp { }) async updateCollectionPlace({ placeId, ...body }: { placeId: number } & CollectionPlaceUpdateRequest, ctx: McpContext) { const demo = this.denyDemo(ctx.userId); if (demo) return demo; - try { return ok({ place: this.collections.updatePlace(ctx.userId, placeId, body) }); } catch (err) { return fail(err); } + try { return ok({ place: await this.collections.updatePlace(ctx.userId, placeId, body) }); } catch (err) { return fail(err); } } @Tool({ @@ -253,7 +253,7 @@ export class CollectionsMcp { }) async deleteCollectionPlace({ placeId }: { placeId: number }, ctx: McpContext) { const demo = this.denyDemo(ctx.userId); if (demo) return demo; - try { this.collections.deletePlace(ctx.userId, placeId); return ok({ success: true }); } catch (err) { return fail(err); } + try { await this.collections.deletePlace(ctx.userId, placeId); return ok({ success: true }); } catch (err) { return fail(err); } } @Tool({ diff --git a/server/src/nest/collections/collections.module.ts b/server/src/nest/collections/collections.module.ts index ee569a326..4f0468c55 100644 --- a/server/src/nest/collections/collections.module.ts +++ b/server/src/nest/collections/collections.module.ts @@ -9,12 +9,33 @@ import { AddonsModule } from '../addons/addons.module'; import { PermissionsModule } from '../permissions/permissions.module'; import { AuthModule } from '../auth/auth.module'; import { NotificationsModule } from '../notifications/notifications.module'; +import { MulterModule } from '@nestjs/platform-express'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { MAX_COVER_SIZE } from './collections.controller'; /** Collections domain (saved-places library). Registered in AppModule. * Exports CollectionsService for in-container consumers (PluginsModule's * RPC host deps factory). */ @Module({ - imports: [NotificationsModule, AddonsModule, PermissionsModule, AuthModule, AppConfigModule, PluginGuardsModule], + imports: [ + // Two upload routes, two categories: the per-file resolver routes each part + // to the right spool. Fieldname routing is exact — FileInterceptor('cover')/ + // FileInterceptor('image') reject any other part with LIMIT_UNEXPECTED_FILE + // before the storage callbacks run. Both routes pass their fileFilter + // inline (cover keeps its plain-Error 500 quirk; image its statusCode 400). + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: (storage: StorageService) => + buildStorageUploadOptions(storage, { + category: (_req, file) => (file.fieldname === 'image' ? 'places' : 'covers'), + maxSize: MAX_COVER_SIZE, // same 20 MB cap for covers and place images + }), + }), + StorageModule, + NotificationsModule, AddonsModule, PermissionsModule, AuthModule, AppConfigModule, PluginGuardsModule], controllers: [CollectionsController], providers: [CollectionsService, CollectionsMcp, CollectionsRpc], exports: [CollectionsService], diff --git a/server/src/nest/collections/collections.rpc.ts b/server/src/nest/collections/collections.rpc.ts index 93bf1424e..8504bfa4a 100644 --- a/server/src/nest/collections/collections.rpc.ts +++ b/server/src/nest/collections/collections.rpc.ts @@ -83,11 +83,14 @@ export class CollectionsRpc { } @PluginMethod('collections.deletePlace', { permission: 'db:write:collections' }) - deletePlace(params: Record, ctx: PluginRpcContext): unknown { + async deletePlace(params: Record, ctx: PluginRpcContext): Promise { const userId = this.requireCollectionsUser(ctx, 'writes'); const placeId = num(params.placeId, 'placeId'); this.requireCollectionsAddon(); - this.mapCollectionError(() => this.collections.deletePlace(userId, placeId, undefined)); + // deletePlace is async (it deletes the underlying storage object): await it so a + // refusal actually reaches the plugin as RESOURCE_FORBIDDEN/BAD_PARAMS instead of + // being dropped as an unhandled rejection while this returns {deleted: true} anyway. + await this.mapCollectionError(() => this.collections.deletePlace(userId, placeId, undefined)); return { deleted: true }; } @@ -102,10 +105,17 @@ export class CollectionsRpc { this.guards.requireAddon(ADDON_IDS.COLLECTIONS, 'collections'); } - /** The service's status-tagged errors, mapped onto the RPC error taxonomy. */ - private mapCollectionError(fn: () => T): T { + /** + * The service's status-tagged errors, mapped onto the RPC error taxonomy. + * + * Async-aware: `deletePlace` is async (it deletes the underlying storage object), + * while `create`/`update`/`savePlace`/`copyToTrip` stay sync. Awaiting inside always + * works for both — a sync throw from `fn()` is caught by this function's own + * try/catch same as an awaited rejection — so every caller goes through one path. + */ + private async mapCollectionError(fn: () => T | Promise): Promise { try { - return fn(); + return await fn(); } catch (e) { const status = (e as { status?: number })?.status; const msg = e instanceof Error ? e.message : 'collection error'; diff --git a/server/src/nest/collections/collections.service.ts b/server/src/nest/collections/collections.service.ts index 967835628..51f3a0349 100644 --- a/server/src/nest/collections/collections.service.ts +++ b/server/src/nest/collections/collections.service.ts @@ -1,10 +1,10 @@ -import fs from 'fs'; import path from 'path'; import { Injectable } from '@nestjs/common'; import { DatabaseService } from '../database/database.service'; import { PermissionsService } from '../permissions/permissions.service'; import { RealtimeService } from '../realtime/realtime.service'; import { reclaimPlaceImage } from '../places/place-image'; +import { StorageService } from '../storage/storage.service'; import { COORD_DEDUP_TOLERANCE, externalIdsOf, @@ -47,17 +47,6 @@ function serializeLinks(links: CollectionLink[] | undefined): string | null { return links && links.length ? JSON.stringify(links) : null; } -/** - * Reclaim a replaced cover file. Path-confined to uploads/covers (mirrors - * tripService.deleteOldCover — kept local so this service doesn't pull the - * trips import graph). Collection + trip covers share the same directory. - */ -function deleteOldCollectionCover(coverImage: string | null | undefined): void { - if (!coverImage) return; - const coversDir = path.resolve(__dirname, '../../../uploads/covers'); - const resolved = path.resolve(path.join(coversDir, path.basename(coverImage))); - if (resolved.startsWith(coversDir + path.sep) && fs.existsSync(resolved)) fs.unlinkSync(resolved); -} // --------------------------------------------------------------------------- // Errors — thrown as plain Errors carrying a status; TrekExceptionFilter maps @@ -96,8 +85,24 @@ export class CollectionsService { private readonly permissions: PermissionsService, private readonly realtime: RealtimeService, private readonly notifications: NotificationsService, + private readonly storage: StorageService, ) {} + /** + * Reclaim a replaced cover object (mirrors tripService.deleteOldCover — kept + * local so this service doesn't pull the trips import graph). Collection + + * trip covers share the 'covers' category; basename() tolerates the stored + * /uploads/covers/ URL form and any external URL the client saved + * (central key validation rejects hostile values; the catch swallows them + * like the old containment guard did). + */ + private async deleteOldCollectionCover(coverImage: string | null | undefined): Promise { + if (!coverImage) return; + await this.storage.delete('covers', path.basename(coverImage)).catch(() => { + /* external URL or already gone */ + }); + } + // ------------------------------------------------------------------------- // Visibility — a user may see/edit a collection if they own it OR are an // accepted member. Every read/write goes through assertAccess. @@ -374,11 +379,11 @@ export class CollectionsService { } /** Set (or clear) a list's cover image, reclaiming the previous file. */ - setCollectionCover(userId: number, id: number, coverUrl: string | null, socketId?: string): Collection { + async setCollectionCover(userId: number, id: number, coverUrl: string | null, socketId?: string): Promise { this.assertCanEdit(userId, id); const prev = this.db.get<{ cover_image: string | null }>('SELECT cover_image FROM collections WHERE id = ?', id)?.cover_image ?? null; this.db.run('UPDATE collections SET cover_image = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', coverUrl, id); - if (prev && prev !== coverUrl) deleteOldCollectionCover(prev); + if (prev && prev !== coverUrl) await this.deleteOldCollectionCover(prev); this.notifyCollectionUsers(id, socketId, 'collections:updated'); const col = this.getCollectionRow(id); return { ...col, is_owner: col.owner_id === userId }; @@ -649,7 +654,7 @@ export class CollectionsService { return { copied, skipped }; } - updatePlace(userId: number, placeId: number, body: CollectionPlaceUpdateRequest, socketId?: string): CollectionPlace { + async updatePlace(userId: number, placeId: number, body: CollectionPlaceUpdateRequest, socketId?: string): Promise { const currentCollection = this.collectionIdOfPlace(placeId); this.assertCanEdit(userId, currentCollection); @@ -701,7 +706,7 @@ export class CollectionsService { }); if (body.image_url !== undefined && prevImage !== (body.image_url ?? null)) { - reclaimPlaceImage(prevImage); + await reclaimPlaceImage(this.storage, prevImage); } this.notifyCollectionUsers(currentCollection, socketId, 'collections:updated'); @@ -737,16 +742,16 @@ export class CollectionsService { return this.getPlaceById(placeId); } - deletePlace(userId: number, placeId: number, socketId?: string): void { + async deletePlace(userId: number, placeId: number, socketId?: string): Promise { const collectionId = this.collectionIdOfPlace(placeId); this.assertCanDelete(userId, collectionId); const image = this.db.get<{ image_url: string | null }>('SELECT image_url FROM collection_places WHERE id = ?', placeId)?.image_url ?? null; this.db.run('DELETE FROM collection_places WHERE id = ?', placeId); // CASCADE drops tags. NO photo-cache reclaim. - reclaimPlaceImage(image); + await reclaimPlaceImage(this.storage, image); this.notifyCollectionUsers(collectionId, socketId, 'collections:updated'); } - deletePlacesMany(userId: number, ids: number[], socketId?: string): number[] { + async deletePlacesMany(userId: number, ids: number[], socketId?: string): Promise { // All-or-nothing since the post-fold quirk pass: every id is resolved and // permission-checked BEFORE any delete (the relocated legacy interleaved // checks with deletes, so a mid-list 403/404 left earlier deletes committed), @@ -766,7 +771,7 @@ export class CollectionsService { deleted.push(id); } }); - for (const image of images) reclaimPlaceImage(image); + for (const image of images) await reclaimPlaceImage(this.storage, image); touched.forEach(cid => this.notifyCollectionUsers(cid, socketId, 'collections:updated')); return deleted; } @@ -875,12 +880,12 @@ export class CollectionsService { } /** Set (or clear) a saved place's custom thumbnail, reclaiming the previous upload. */ - setPlaceImage(userId: number, placeId: number, imageUrl: string | null, socketId?: string): CollectionPlace { + async setPlaceImage(userId: number, placeId: number, imageUrl: string | null, socketId?: string): Promise { const collectionId = this.collectionIdOfPlace(placeId); this.assertCanEdit(userId, collectionId); const prev = this.db.get<{ image_url: string | null }>('SELECT image_url FROM collection_places WHERE id = ?', placeId)?.image_url ?? null; this.db.run('UPDATE collection_places SET image_url = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?', imageUrl, placeId); - if (prev !== imageUrl) reclaimPlaceImage(prev); + if (prev !== imageUrl) await reclaimPlaceImage(this.storage, prev); this.notifyCollectionUsers(collectionId, socketId, 'collections:updated'); return this.getPlaceById(placeId); } diff --git a/server/src/nest/common/crypto/apiKeyCrypto.ts b/server/src/nest/common/crypto/apiKeyCrypto.ts index 96eccea5c..ca6e0b3df 100644 --- a/server/src/nest/common/crypto/apiKeyCrypto.ts +++ b/server/src/nest/common/crypto/apiKeyCrypto.ts @@ -41,3 +41,8 @@ export function maybe_encrypt_api_key(value: unknown) { return encrypt_api_key(trimmed); } +/** True exactly for values carrying the enc:v1: envelope produced by encrypt_api_key. */ +export function is_encrypted_api_key(value: unknown): value is string { + return typeof value === 'string' && value.startsWith(ENCRYPTED_PREFIX); +} + diff --git a/server/src/nest/common/place-image-upload.ts b/server/src/nest/common/place-image-upload.ts index 61aacb21f..b8f304a4a 100644 --- a/server/src/nest/common/place-image-upload.ts +++ b/server/src/nest/common/place-image-upload.ts @@ -1,38 +1,24 @@ -import fs from 'fs'; import path from 'path'; -import { diskStorage } from 'multer'; -import { v4 as uuidv4 } from 'uuid'; -import type { Request } from 'express'; -import { PLACE_IMAGES_DIR } from '../places/place-image'; +import type { Options } from 'multer'; -const MAX_PLACE_IMAGE_SIZE = 20 * 1024 * 1024; // 20 MB — same cap as covers. +export const MAX_PLACE_IMAGE_SIZE = 20 * 1024 * 1024; // 20 MB — same cap as covers. /** - * Multer config for the custom place-image upload (#1136). Mirrors the collection - * COVER_UPLOAD: server-chosen UUID filename, image-only filter, written to the - * dedicated uploads/places dir. Shared by the trip-place and collection-place - * upload endpoints. + * fileFilter for the custom place-image upload (#1136), shared by the + * trip-place and collection-place upload endpoints (passed inline on each + * route; the storage engine — spool destination + UUID filename — comes from + * the owning module's storage-upload factory options). */ -export const PLACE_IMAGE_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { - if (!fs.existsSync(PLACE_IMAGES_DIR)) fs.mkdirSync(PLACE_IMAGES_DIR, { recursive: true }); - cb(null, PLACE_IMAGES_DIR); - }, - filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`), - }), - limits: { fileSize: MAX_PLACE_IMAGE_SIZE }, - fileFilter: (_req: Request, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; - if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) { - cb(null, true); - } else { - // Carry statusCode so TrekExceptionFilter maps the rejection to a 400 rather - // than a 500 (same contract as the avatar upload's fileFilter). - const err: Error & { statusCode?: number } = new Error('Only jpg, png, gif, webp images allowed'); - err.statusCode = 400; - cb(err, false); - } - }, +export const PLACE_IMAGE_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; + if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) { + cb(null, true); + } else { + // Carry statusCode so TrekExceptionFilter maps the rejection to a 400 rather + // than a 500 (same contract as the avatar upload's fileFilter). + const err: Error & { statusCode?: number } = new Error('Only jpg, png, gif, webp images allowed'); + err.statusCode = 400; + cb(err); + } }; diff --git a/server/src/nest/common/trek-exception.filter.ts b/server/src/nest/common/trek-exception.filter.ts index 6b8910788..1ace82dcd 100644 --- a/server/src/nest/common/trek-exception.filter.ts +++ b/server/src/nest/common/trek-exception.filter.ts @@ -22,6 +22,22 @@ export class TrekExceptionFilter implements ExceptionFilter { catch(exception: unknown, host: ArgumentsHost): void { const res = host.switchToHttp().getResponse(); + // 0. A response already streaming to the client (headers sent) cannot be + // rewritten with a JSON error envelope — this fires when a storage + // stream (or any other in-flight write) errors after + // sendToResponse()/pipeline() has already flushed headers, e.g. a + // client abort mid-download. Writing here would throw + // ERR_HTTP_HEADERS_SENT or corrupt a response the client is mid-read + // on. Log it first — this is the ONLY place a post-header failure is + // ever recorded, since destroying the socket below skips every other + // branch's logging (including case 2's `status >= 500` log) — then + // destroy the socket so the connection doesn't hang open. + if (res.headersSent) { + console.error('Unhandled error after headers sent:', exception); + res.destroy(); + return; + } + // 1. Raw multer errors that slipped past @nestjs/platform-express's // transformException (it leaves codes it does not recognise untouched). // Legacy: LIMIT_FILE_SIZE -> 413, everything else -> 400, body { error: message }. diff --git a/server/src/nest/common/validate-managed-routes.ts b/server/src/nest/common/validate-managed-routes.ts index 22f6a0b14..8351ed9ad 100644 --- a/server/src/nest/common/validate-managed-routes.ts +++ b/server/src/nest/common/validate-managed-routes.ts @@ -79,6 +79,14 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [ 'PluginsController.link', 'PluginsController.reload', 'PluginsController.upload', + 'StorageAdminController.backfillCancel', + 'StorageAdminController.backfillStart', + 'StorageAdminController.get', + 'StorageAdminController.migrationCancel', + 'StorageAdminController.migrationStart', + 'StorageAdminController.statsRefresh', + 'StorageAdminController.test', + 'StorageAdminController.update', ]; /** diff --git a/server/src/nest/files/files-download.controller.ts b/server/src/nest/files/files-download.controller.ts index 96fdfbd9a..2f96ab0a4 100644 --- a/server/src/nest/files/files-download.controller.ts +++ b/server/src/nest/files/files-download.controller.ts @@ -1,9 +1,9 @@ import { Controller, Get, HttpException, Param, Req, Res } from '@nestjs/common'; import type { Request, Response } from 'express'; import path from 'path'; -import fs from 'fs'; import { FilesService } from './files.service'; import { Public } from '../auth/public.decorator'; +import { StorageService } from '../storage/storage.service'; /** * GET /api/trips/:tripId/files/:id/download — authenticated file download. @@ -11,15 +11,25 @@ import { Public } from '../auth/public.decorator'; * Deliberately NOT behind the JwtAuthGuard: it accepts a cookie, a Bearer header * OR a one-shot `?token=` query param (so links can be opened directly), all via * the legacy authenticateDownload helper. Byte-identical to the legacy route: - * 401 token, 404 trip/file, 403 path traversal, .pkpass served inline for Wallet. + * 401 token, 404 trip/file, .pkpass served inline for Wallet. Bytes come from + * storage.sendToResponse, whose local branch is the root-relative res.sendFile + * form this route needs under the ExpressAdapter. */ @Public('authenticates itself: the download link carries a short-lived ?token the client mints') @Controller('api/trips/:tripId/files') export class FilesDownloadController { - constructor(private readonly files: FilesService) {} + constructor( + private readonly files: FilesService, + private readonly storage: StorageService, + ) {} @Get(':id/download') - download(@Req() req: Request, @Res() res: Response, @Param('tripId') tripId: string, @Param('id') id: string): void { + async download( + @Req() req: Request, + @Res() res: Response, + @Param('tripId') tripId: string, + @Param('id') id: string, + ): Promise { const auth = this.files.authenticateDownload(req); if ('error' in auth) { throw new HttpException({ error: auth.error }, auth.status); @@ -35,11 +45,10 @@ export class FilesDownloadController { throw new HttpException({ error: 'File not found' }, 404); } - const { resolved, safe } = this.files.resolveFilePath(file.filename); - if (!safe) { - throw new HttpException({ error: 'Forbidden' }, 403); - } - if (!fs.existsSync(resolved)) { + // basename() tolerates a stray prefixed row (legacy rows sometimes stored + // a relative path); the storage layer's key validation refuses the rest. + const name = path.basename(file.filename); + if (!(await this.storage.exists('files', name).catch(() => false))) { throw new HttpException({ error: 'File not found' }, 404); } @@ -47,22 +56,20 @@ export class FilesDownloadController { // (iOS/macOS) hands them to Wallet instead of downloading as a blob. A // `.pkpasses` bundle (a ZIP of multiple passes) is a distinct type with its // own plural MIME type — without it Wallet won't offer to add the passes. + const ext = path.extname(name).toLowerCase(); const walletMime = - path.extname(resolved).toLowerCase() === '.pkpass' + ext === '.pkpass' ? 'application/vnd.apple.pkpass' - : path.extname(resolved).toLowerCase() === '.pkpasses' + : ext === '.pkpasses' ? 'application/vnd.apple.pkpasses' : null; - if (walletMime) { - res.setHeader('Content-Type', walletMime); - res.setHeader('Content-Disposition', `inline; filename="${path.basename(file.original_name || resolved)}"`); - } - - // Serve with an explicit { root } + basename rather than an absolute path: - // under the Nest ExpressAdapter, res.sendFile(absolutePath) resolves the - // file relative to the (rewritten) req.url and fails with a spurious - // "Not Found", whereas the root-relative form streams correctly. The - // resolveFilePath guard above already pins this to the uploads dir. - res.sendFile(path.basename(resolved), { root: path.dirname(resolved) }); + await this.storage.sendToResponse( + 'files', + name, + res, + walletMime + ? { contentType: walletMime, disposition: `inline; filename="${path.basename(file.original_name || name)}"` } + : undefined, + ); } } diff --git a/server/src/nest/files/files.controller.ts b/server/src/nest/files/files.controller.ts index 38caecebb..34f2c0767 100644 --- a/server/src/nest/files/files.controller.ts +++ b/server/src/nest/files/files.controller.ts @@ -18,55 +18,45 @@ import { import { FileInterceptor } from '@nestjs/platform-express'; import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; import path from 'path'; import fs from 'fs'; -import { v4 as uuidv4 } from 'uuid'; import type { User } from '../../types'; +import { StorageService } from '../storage/storage.service'; import { FilesService } from './files.service'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { CurrentUser } from '../auth/current-user.decorator'; import { TripAccessGuard } from '../permissions/trip-access.guard'; import type { TripAccess } from '../database/database.service'; import { Trip } from '../permissions/trip.decorator'; -import { MAX_FILE_SIZE, MAX_VIDEO_SIZE, BLOCKED_EXTENSIONS, filesDir, isVideoExtension } from './files.constants'; +import { MAX_FILE_SIZE, BLOCKED_EXTENSIONS, isVideoExtension } from './files.constants'; import { FileUploadDto, FileUpdateDto, FileLinkDto } from './files.dto'; import { AllowedFileTypesService } from './allowed-file-types.service'; /** - * Trip-file upload options, built from the container. + * Trip-file upload filter, built from the container. * - * A factory rather than a module-scope literal because the fileFilter reads the - * operator's allowed-extension list at request time, and that list lives in the - * database. It used to reach it through files.bridge.ts, which built a second - * FilesService outside the injector; MulterModule.registerAsync gives the - * closure the real one. + * A factory rather than a module-scope literal because it reads the operator's + * allowed-extension list at request time, and that list lives in the database. + * The rest of the multer options come from the storage upload factory + * (buildStorageUploadOptions in files.module.ts); this closure is passed + * through untouched. */ -export function buildFilesUploadOptions(allowedTypes: AllowedFileTypesService) { - return { - storage: diskStorage({ - destination: (_req, _file, cb) => { if (!fs.existsSync(filesDir)) fs.mkdirSync(filesDir, { recursive: true }); cb(null, filesDir); }, - filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`), - }), - // Allow up to the video cap; non-video files are still held to MAX_FILE_SIZE by - // the per-type guard in the upload handler (#823). - limits: { fileSize: MAX_VIDEO_SIZE }, - defParamCharset: 'utf8', // parity with legacy routes/files.ts — preserve non-ASCII original filenames - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - const reject = () => { - // i18n key — the client resolves it via t() (see translateApiError). - const err: Error & { statusCode?: number } = new Error('files.uploadErrorType'); - err.statusCode = 400; - cb(err, false); - }; - if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg')) return reject(); - const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase()); - const fileExt = ext.replace('.', ''); - // Video is accepted as media regardless of the admin doc-types allowlist (#823). - if (allowed.includes(fileExt) || isVideoExtension(fileExt) || (allowed.includes('*') && !BLOCKED_EXTENSIONS.includes(ext))) return cb(null, true); - reject(); - }, +export function filesUploadFileFilter(allowedTypes: AllowedFileTypesService): Options['fileFilter'] { + return (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + const reject = () => { + // i18n key — the client resolves it via t() (see translateApiError). + const err: Error & { statusCode?: number } = new Error('files.uploadErrorType'); + err.statusCode = 400; + cb(err); + }; + if (BLOCKED_EXTENSIONS.includes(ext) || file.mimetype.includes('svg')) return reject(); + const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase()); + const fileExt = ext.replace('.', ''); + // Video is accepted as media regardless of the admin doc-types allowlist (#823). + if (allowed.includes(fileExt) || isVideoExtension(fileExt) || (allowed.includes('*') && !BLOCKED_EXTENSIONS.includes(ext))) return cb(null, true); + reject(); }; } @@ -97,6 +87,7 @@ export class FilesController { constructor( private readonly files: FilesService, private readonly env: RuntimeEnvService, + private readonly storage: StorageService, ) {} @@ -117,14 +108,14 @@ export class FilesController { @Post() @UseInterceptors(FileInterceptor('file')) - upload( + async upload( @CurrentUser() user: User, @Param('tripId') tripId: string, @UploadedFile() file: Express.Multer.File | undefined, @Body() body: FileUploadDto, @Headers('x-socket-id') socketId?: string, ) { - // multer (diskStorage) has already written the upload by the time we get here, + // multer (diskStorage) has already spooled the upload by the time we get here, // so every rejection below must remove the orphaned bytes — otherwise a 404/403 // leaves up to the 500 MB video cap on disk (#823). const cleanup = () => { if (file?.path) { try { fs.unlinkSync(file.path); } catch { /* best-effort */ } } }; @@ -162,6 +153,14 @@ export class FilesController { cleanup(); throw err; } + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before anything references the final path. + try { + await this.storage.put('files', file.filename, { tmpPath: file.path }); + } catch (err) { + cleanup(); + throw err; + } const created = this.files.createFile(tripId, file, user.id, { place_id: body.place_id, description: body.description, diff --git a/server/src/nest/files/files.module.ts b/server/src/nest/files/files.module.ts index 1788267c2..0c2a7f7c5 100644 --- a/server/src/nest/files/files.module.ts +++ b/server/src/nest/files/files.module.ts @@ -11,15 +11,28 @@ import { EphemeralTokenModule } from '../auth/ephemeral-token.module'; import { MulterModule } from '@nestjs/platform-express'; import { AllowedFileTypesModule } from './allowed-file-types.module'; import { AllowedFileTypesService } from './allowed-file-types.service'; -import { buildFilesUploadOptions } from './files.controller'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { filesUploadFileFilter } from './files.controller'; +import { MAX_VIDEO_SIZE } from './files.constants'; @Module({ imports: [ MulterModule.registerAsync({ - imports: [AllowedFileTypesModule], - inject: [AllowedFileTypesService], - useFactory: (allowedTypes: AllowedFileTypesService) => buildFilesUploadOptions(allowedTypes), + imports: [StorageModule, AllowedFileTypesModule], + inject: [StorageService, AllowedFileTypesService], + useFactory: (storage: StorageService, allowedTypes: AllowedFileTypesService) => + buildStorageUploadOptions(storage, { + category: 'files', + // Allow up to the video cap; non-video files are still held to + // MAX_FILE_SIZE by the per-type guard in the upload handler (#823). + maxSize: MAX_VIDEO_SIZE, + defParamCharset: 'utf8', // parity with legacy routes/files.ts — preserve non-ASCII original filenames + fileFilter: filesUploadFileFilter(allowedTypes), + }), }), + StorageModule, EphemeralTokenModule, PermissionsModule, AppConfigModule, RealtimeModule, PluginGuardsModule], controllers: [FilesController, FilesDownloadController], providers: [FilesService, FilesRpc], diff --git a/server/src/nest/files/files.rpc.ts b/server/src/nest/files/files.rpc.ts index de235b402..8a3a785e2 100644 --- a/server/src/nest/files/files.rpc.ts +++ b/server/src/nest/files/files.rpc.ts @@ -1,6 +1,6 @@ -import fsMod from 'node:fs'; import pathMod from 'node:path'; import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; import { PluginController, PluginMethod } from '../plugins/host/rpc-kit/decorators'; import { PluginGuards } from '../plugins/host/plugin-guards.service'; import { BadParams, ForbiddenResource } from '../plugins/host/rpc-errors'; @@ -10,8 +10,10 @@ import { RealtimeService } from '../realtime/realtime.service'; import { DatabaseService } from '../database/database.service'; import { readEnv } from '../../app-config'; import { isDemoEmail } from '../common/demo'; -import { BLOCKED_EXTENSIONS, filesDir } from './files.constants'; +import { BLOCKED_EXTENSIONS } from './files.constants'; import { FilesService } from './files.service'; +import { StorageService } from '../storage/storage.service'; +import { StorageNotFoundError, StorageInvalidKeyError, type ObjectStat } from '../storage/storage.types'; /** Files use three separate rights, one per operation, unlike every other domain. */ const UPLOAD_ACTION = 'file_upload'; @@ -46,9 +48,10 @@ type CreateInput = { * sensitive than its filename. And the three write operations use three distinct * rights (file_upload / file_edit / file_delete) rather than one domain action. * - * The upload path is the only one in the plugin surface that touches disk, so the - * extension is checked against the central blocklist and the size is capped before - * anything is written, and a demo user is refused outright. + * This surface holds no direct fs access: both byte-paths — read and write — go + * through the storage layer, so `ctx.files` behaves identically whether the backend + * is local disk, S3, or a mirrored pair. The extension blocklist and size cap still + * run before the put, and a demo user is refused outright. */ @PluginController() export class FilesRpc { @@ -57,6 +60,7 @@ export class FilesRpc { private readonly realtime: RealtimeService, private readonly db: DatabaseService, private readonly guards: PluginGuards, + private readonly storage: StorageService, ) {} @PluginMethod('files.list', { permission: 'db:read:files' }) @@ -74,7 +78,8 @@ export class FilesRpc { /** * Size-capped BEFORE the read, so a 500MB video cannot be pulled through the IPC - * pipe as ~667MB of base64. The read itself runs off the event loop: 10MB of + * pipe as ~667MB of base64. The bytes come from the storage layer, so this read + * works on remote backends too. The read itself runs off the event loop: 10MB of * readFile plus base64 on the host thread would stall every other plugin RPC and * every HTTP request for its duration. */ @@ -84,10 +89,35 @@ export class FilesRpc { if ((file.file_size ?? 0) > CONTENT_MAX) { throw new BadParams(`file too large to read (>${CONTENT_MAX} bytes); use the download UI`); } - const { resolved, safe } = this.files.resolveFilePath(file.filename); - if (!safe) throw new ForbiddenResource('file path is not accessible'); - const buf = await fsMod.promises.readFile(resolved); - if (buf.length > CONTENT_MAX) throw new BadParams('file too large to read'); + let stream: Readable; + let stat: ObjectStat; + try { + ({ stream, stat } = await this.storage.getStream('files', pathMod.basename(file.filename))); + } catch (err) { + if (err instanceof StorageNotFoundError || err instanceof StorageInvalidKeyError) { + throw new ForbiddenResource('file path is not accessible'); + } + throw err; + } + // Re-checked against the OBJECT, not the DB row: file_size can drift. + if (stat.size > CONTENT_MAX) { + stream.destroy(); + throw new BadParams(`file too large to read (>${CONTENT_MAX} bytes); use the download UI`); + } + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of stream) { + const part = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + total += part.length; + // A driver whose stat under-reports must not push an oversized payload + // through the IPC pipe: abort as soon as the running total crosses the cap. + if (total > CONTENT_MAX) { + stream.destroy(); + throw new BadParams('file too large to read'); + } + chunks.push(part); + } + const buf = Buffer.concat(chunks); return { name: file.original_name, mimetype: file.mime_type ?? 'application/octet-stream', @@ -97,7 +127,7 @@ export class FilesRpc { } @PluginMethod('files.create', { permission: 'db:write:files' }) - create(params: Record, ctx: PluginRpcContext): unknown { + async create(params: Record, ctx: PluginRpcContext): Promise { const tripId = num(params.tripId, 'tripId'); const actor = this.guards.requireActor(ctx, 'file'); const input = asPayload(params.input); @@ -116,7 +146,7 @@ export class FilesRpc { return this.writeFile(tripId, input as unknown as CreateInput, actor); } - private writeFile(tripId: number, input: CreateInput, actingUserId: number): unknown { + private async writeFile(tripId: number, input: CreateInput, actingUserId: number): Promise { // Mirrors the REST upload guard: a demo user must not write bytes to the shared // demo instance, not even through a plugin's db:write:files. The email is only // resolved when demo mode is actually on, so self-hosted installs pay nothing. @@ -138,8 +168,12 @@ export class FilesRpc { }); if (foreign) throw new ForbiddenResource(`${foreign} does not belong to trip ${tripId}`); const filename = `${randomUUID()}${ext}`; - fsMod.mkdirSync(filesDir, { recursive: true }); - fsMod.writeFileSync(pathMod.join(filesDir, filename), buf); + // Same order as the REST upload (files.controller.ts): the object is + // committed to storage before anything references it — a put failure can + // orphan a blob at worst, never create a row pointing at missing bytes. + await this.storage.put('files', filename, Readable.from(buf), { + contentType: input.mimetype || 'application/octet-stream', + }); const file = this.files.createFile( tripId, { filename, originalname: original, size: buf.length, mimetype: input.mimetype || 'application/octet-stream' }, diff --git a/server/src/nest/files/files.service.ts b/server/src/nest/files/files.service.ts index 33b52768b..ba0544697 100644 --- a/server/src/nest/files/files.service.ts +++ b/server/src/nest/files/files.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import path from 'path'; -import fs from 'fs'; import type { Request } from 'express'; import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared'; import { RealtimeService } from '../realtime/realtime.service'; @@ -10,7 +9,8 @@ import { EphemeralTokenService } from '../auth/ephemeral-token.service'; import { verifyJwtAndLoadUser } from '../auth/jwt-verify'; import type { User, TripFile } from '../../types'; import { DatabaseService, type TripAccess } from '../database/database.service'; -import { DEFAULT_ALLOWED_EXTENSIONS, filesDir } from './files.constants'; +import { DEFAULT_ALLOWED_EXTENSIONS } from './files.constants'; +import { StorageService } from '../storage/storage.service'; type Trip = TripAccess; type FilePermission = 'file_upload' | 'file_edit' | 'file_delete'; @@ -45,9 +45,9 @@ export interface FileLink { * DatabaseService's canAccessTrip (the legacy verifyTripAccess re-export was a * plain wrapper over the same helper); the file_* permissions, path-resolution * guard, download-token auth and WebSocket broadcasts are unchanged. The - * load-time constants live in files.constants.ts; the one out-of-DI consumer - * path (module-scope multer configs) reaches getAllowedExtensions through - * files.bridge.ts. + * load-time constants live in files.constants.ts; the admin allowed-types + * live-read is AllowedFileTypesService (the single query owner since the + * storage slice-2 consolidation). * * Two deliberate post-migration fixes over the legacy behavior: createFileLink * no longer swallows insert errors, and updateFile coerces an empty-string @@ -60,6 +60,7 @@ export class FilesService { private readonly permissions: PermissionsService, private readonly realtime: RealtimeService, private readonly tokens: EphemeralTokenService, + private readonly storage: StorageService, ) {} verifyTripAccess(tripId: string | number, userId: number) { @@ -74,13 +75,6 @@ export class FilesService { this.realtime.broadcast(tripId, event, payload, socketId); } - getAllowedExtensions(): string { - try { - const row = this.db.get<{ value: string }>("SELECT value FROM app_settings WHERE key = 'allowed_file_types'"); - return row?.value || DEFAULT_ALLOWED_EXTENSIONS; - } catch { return DEFAULT_ALLOWED_EXTENSIONS; } - } - // --------------------------------------------------------------------------- // Token-based download auth & safe path resolution (used by the unguarded // download route) @@ -112,14 +106,6 @@ export class FilesService { return { error: 'Authentication required', status: 401 }; } - resolveFilePath(filename: string): { resolved: string; safe: boolean } { - const safeName = path.basename(filename); - const filePath = path.join(filesDir, safeName); - const resolved = path.resolve(filePath); - const safe = resolved.startsWith(path.resolve(filesDir)); - return { resolved, safe }; - } - // --------------------------------------------------------------------------- // Trip-scoped link validation // --------------------------------------------------------------------------- @@ -251,14 +237,12 @@ export class FilesService { } async permanentDeleteFile(file: TripFile): Promise { - const { resolved } = this.resolveFilePath(file.filename); - // `force: true` swallows ENOENT, replacing the prior existsSync+unlink - // double-call that blocked the event loop twice per deletion. Only - // drop the DB row when the on-disk unlink either succeeded or the - // file was already gone — otherwise a permission / ENOSPC failure - // would orphan the bytes on disk with no DB pointer left to clean it. + // storage.delete is idempotent on a missing object (the old rm force:true + // contract). Only drop the DB row when the delete either succeeded or the + // object was already gone — otherwise a permission / ENOSPC failure + // would orphan the bytes with no DB pointer left to clean them. try { - await fs.promises.rm(resolved, { force: true }); + await this.storage.delete('files', path.basename(file.filename)); } catch (e) { console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e); throw e; @@ -273,9 +257,8 @@ export class FilesService { // and a retry via the single-file delete path can try again. const successfullyUnlinked: number[] = []; await Promise.all(trashed.map(async (file) => { - const { resolved } = this.resolveFilePath(file.filename); try { - await fs.promises.rm(resolved, { force: true }); + await this.storage.delete('files', path.basename(file.filename)); successfullyUnlinked.push(Number(file.id)); } catch (e) { console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e); diff --git a/server/src/nest/journey/journey-domain.service.ts b/server/src/nest/journey/journey-domain.service.ts index e59a9d82d..6e5dc8355 100644 --- a/server/src/nest/journey/journey-domain.service.ts +++ b/server/src/nest/journey/journey-domain.service.ts @@ -1468,22 +1468,22 @@ export class JourneyDomainService { deleteGalleryPhoto( journeyPhotoId: number, userId: number, - ): { photo_id: number; file_path?: string | null } | null { + ): { photo_id: number; file_path?: string | null; thumbnail_path?: string | null } | null { const row = this.db.prepare('SELECT * FROM journey_photos WHERE id = ?').get(journeyPhotoId) as | { id: number; journey_id: number; photo_id: number } | undefined; if (!row) return null; if (!this.canEdit(row.journey_id, userId)) return null; - const trekRow = this.db.prepare('SELECT file_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as - | { file_path?: string; provider?: string } + const trekRow = this.db.prepare('SELECT file_path, thumbnail_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as + | { file_path?: string; thumbnail_path?: string; provider?: string } | undefined; // cascade on journey_entry_photos.journey_photo_id handles junction cleanup this.db.prepare('DELETE FROM journey_photos WHERE id = ?').run(journeyPhotoId); this.photos.deleteIfOrphan(row.photo_id); - return { photo_id: row.photo_id, file_path: trekRow?.file_path ?? null }; + return { photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, thumbnail_path: trekRow?.thumbnail_path ?? null }; } setPhotoProvider(photoId: number, provider: string, assetId: string, ownerId: number) { @@ -1533,21 +1533,21 @@ export class JourneyDomainService { deletePhoto( photoId: number, userId: number, - ): { id: number; photo_id: number; file_path?: string | null; journey_id: number } | null { + ): { id: number; photo_id: number; file_path?: string | null; thumbnail_path?: string | null; journey_id: number } | null { const row = this.db.prepare('SELECT id, journey_id, photo_id FROM journey_photos WHERE id = ?').get(photoId) as | { id: number; journey_id: number; photo_id: number } | undefined; if (!row) return null; if (!this.canEdit(row.journey_id, userId)) return null; - const trekRow = this.db.prepare('SELECT file_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as - | { file_path?: string; provider?: string } + const trekRow = this.db.prepare('SELECT file_path, thumbnail_path, provider FROM trek_photos WHERE id = ?').get(row.photo_id) as + | { file_path?: string; thumbnail_path?: string; provider?: string } | undefined; this.db.prepare('DELETE FROM journey_photos WHERE id = ?').run(photoId); this.photos.deleteIfOrphan(row.photo_id); - return { id: row.id, photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, journey_id: row.journey_id }; + return { id: row.id, photo_id: row.photo_id, file_path: trekRow?.file_path ?? null, thumbnail_path: trekRow?.thumbnail_path ?? null, journey_id: row.journey_id }; } // ── Contributors ───────────────────────────────────────────────────────── diff --git a/server/src/nest/journey/journey-public.controller.ts b/server/src/nest/journey/journey-public.controller.ts index aeff8486f..fff8c467f 100644 --- a/server/src/nest/journey/journey-public.controller.ts +++ b/server/src/nest/journey/journey-public.controller.ts @@ -1,8 +1,8 @@ import { Controller, Get, HttpException, Param, Res } from '@nestjs/common'; import type { Response } from 'express'; import path from 'node:path'; -import fs from 'node:fs'; import { JourneyService } from './journey.service'; +import { StorageService } from '../storage/storage.service'; import { Public } from '../auth/public.decorator'; /** @@ -17,7 +17,10 @@ import { Public } from '../auth/public.decorator'; @Public('share-token validated: the whole point is a link that works without an account') @Controller('api/public/journey') export class JourneyPublicController { - constructor(private readonly journey: JourneyService) {} + constructor( + private readonly journey: JourneyService, + private readonly storage: StorageService, + ) {} @Get(':token') get(@Param('token') token: string) { @@ -54,17 +57,18 @@ export class JourneyPublicController { const wantThumb = kind === 'thumbnail' ? 'thumbnail' : 'original'; if (provider === 'local') { - // Local journey assets are flat filenames; use basename() and confine the - // resolved path to the journey upload directory. - const journeyDir = path.resolve(__dirname, '../../../uploads/journey'); - const resolved = path.resolve(path.join(journeyDir, path.basename(assetId))); - if (!resolved.startsWith(journeyDir + path.sep) || !fs.existsSync(resolved)) { + // Local journey assets are flat filenames. basename() kept for shape; + // containment now lives in central storage key validation + // (storage-keys.ts), which subsumes the old journeyDir startsWith guard. + const name = path.basename(assetId); + if (!(await this.storage.exists('journey', name).catch(() => false))) { throw new HttpException({ error: 'Not found' }, 404); } res.set('Cache-Control', 'public, max-age=86400'); - // Root-relative, see photo-resolver.service.ts — an absolute path 404s under - // the Nest ExpressAdapter. The guard above already confined `resolved`. - res.sendFile(path.basename(resolved), { root: path.dirname(resolved) }); + // The storage layer serves it, whichever driver is configured. dev's fix + // for the absolute-path 404 under the Nest ExpressAdapter does not apply + // here any more: nothing on this path calls sendFile. + await this.storage.sendToResponse('journey', name, res); return; } diff --git a/server/src/nest/journey/journey.controller.ts b/server/src/nest/journey/journey.controller.ts index 5c979f612..2f00865ca 100644 --- a/server/src/nest/journey/journey.controller.ts +++ b/server/src/nest/journey/journey.controller.ts @@ -16,11 +16,14 @@ import { UseInterceptors, } from '@nestjs/common'; import { FileInterceptor, FilesInterceptor, FileFieldsInterceptor } from '@nestjs/platform-express'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; +import type { Request } from 'express'; import path from 'node:path'; import fs from 'node:fs'; import crypto from 'node:crypto'; import type { User } from '../../types'; +import { StorageService } from '../storage/storage.service'; +import { journeyThumbName } from '../memories/thumbnail.service'; import { JourneyService } from './journey.service'; import { JourneyBookService } from './journey-book.service'; import { PhotoCaptureBackfillService } from '../memories/photo-capture-backfill.service'; @@ -39,73 +42,73 @@ import { import { isVideoMime, isVideoExtension, MAX_VIDEO_SIZE } from '../files/files.constants'; import { AllowedFileTypesService } from '../files/allowed-file-types.service'; -const uploadsBase = path.join(__dirname, '../../../uploads/journey'); /** - * Journey image upload options, built from the container. - * - * Same reason as the trip-file factory: the fileFilter reads the operator's - * allowed-extension list at request time. Note this config carries NO - * defParamCharset, unlike the trip-file one; that difference is deliberate and - * predates the move, so the two are not folded into one shared object. + * One filename hook for all four journey upload routes (consumed by + * journey.module.ts's storage-upload factory). Field routing is exact: the + * image routes only ever admit 'photos'/'cover' fields and the video route + * only 'video'/'poster' — any other part dies in multer with + * LIMIT_UNEXPECTED_FILE before this hook runs. */ -export function buildJourneyImageUploadOptions(allowedTypes: AllowedFileTypesService) { - return { - storage: diskStorage({ - destination: (_req, _file, cb) => { if (!fs.existsSync(uploadsBase)) fs.mkdirSync(uploadsBase, { recursive: true }); cb(null, uploadsBase); }, - filename: (_req, file, cb) => cb(null, `${crypto.randomUUID()}${path.extname(file.originalname).toLowerCase() || '.jpg'}`), - }), - limits: { fileSize: 20 * 1024 * 1024 }, - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) { - const err: Error & { statusCode?: number } = new Error('Only image files are allowed'); - err.statusCode = 400; - return cb(err, false); - } - const ext = path.extname(file.originalname).toLowerCase().replace('.', ''); - const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase()); - if (!allowed.includes('*') && !allowed.includes(ext)) { - const err: Error & { statusCode?: number } = new Error(`File type .${ext} is not allowed`); - err.statusCode = 400; - return cb(err, false); - } - cb(null, true); - }, +export const journeyUploadFilename = (_req: Request, file: Express.Multer.File): string => { + // The poster is ALWAYS stored as .jpg, never the client-supplied extension: + // otherwise a poster declared image/* but named x.html / x.js would land on + // disk with that extension and be served inline same-origin (stored XSS, + // reachable via the public share proxy). The video extension is validated by + // the fileFilter, so it is safe to keep. + const ext = + file.fieldname === 'poster' ? '.jpg' + : file.fieldname === 'video' ? (path.extname(file.originalname).toLowerCase() || '.mp4') + : (path.extname(file.originalname).toLowerCase() || '.jpg'); + return `${crypto.randomUUID()}${ext}`; +}; + +/** + * Journey image upload filter, built from the container. + * + * Same reason as the trip-file factory: it reads the operator's + * allowed-extension list at request time. The module's multer options carry NO + * defParamCharset, unlike the trip-file ones; that difference is deliberate + * and predates the move. + */ +export function journeyImageFileFilter(allowedTypes: AllowedFileTypesService): Options['fileFilter'] { + return (_req, file, cb) => { + if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) { + const err: Error & { statusCode?: number } = new Error('Only image files are allowed'); + err.statusCode = 400; + return cb(err); + } + const ext = path.extname(file.originalname).toLowerCase().replace('.', ''); + const allowed = allowedTypes.get().split(',').map((e) => e.trim().toLowerCase()); + if (!allowed.includes('*') && !allowed.includes(ext)) { + const err: Error & { statusCode?: number } = new Error(`File type .${ext} is not allowed`); + err.statusCode = 400; + return cb(err); + } + cb(null, true); }; } // Gallery video upload (#823): one video plus an optional client-captured poster // image, written to the same uploads/journey store. Larger cap than images since // phone clips are big; videos are stored as-is and streamed with HTTP Range. -const VIDEO_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { if (!fs.existsSync(uploadsBase)) fs.mkdirSync(uploadsBase, { recursive: true }); cb(null, uploadsBase); }, - filename: (_req, file, cb) => { - // The poster is ALWAYS stored as .jpg, never the client-supplied extension: - // otherwise a poster declared image/* but named x.html / x.js would land on - // disk with that extension and be served inline same-origin (stored XSS, - // reachable via the public share proxy). The video extension is validated by - // the fileFilter, so it is safe to keep. - const ext = file.fieldname === 'poster' ? '.jpg' : (path.extname(file.originalname).toLowerCase() || '.mp4'); - cb(null, `${crypto.randomUUID()}${ext}`); - }, - }), - limits: { fileSize: MAX_VIDEO_SIZE }, - fileFilter: (_req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const reject = (msg: string) => { - const err: Error & { statusCode?: number } = new Error(msg); - err.statusCode = 400; - cb(err, false); - }; - if (file.fieldname === 'poster') { - if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) return reject('Poster must be an image'); - return cb(null, true); - } - // 'video' field - const ext = path.extname(file.originalname).toLowerCase().replace('.', ''); - if (!isVideoMime(file.mimetype)) return reject('Only video files are allowed'); - if (!isVideoExtension(ext)) return reject(`Video type .${ext} is not allowed`); - cb(null, true); - }, +// Passed inline on the video route; the storage engine (spool destination + +// journeyUploadFilename) is inherited from the module options via the +// interceptor's shallow merge. +const VIDEO_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => { + const reject = (msg: string) => { + const err: Error & { statusCode?: number } = new Error(msg); + err.statusCode = 400; + cb(err); + }; + if (file.fieldname === 'poster') { + if (!file.mimetype.startsWith('image/') || file.mimetype.includes('svg')) return reject('Poster must be an image'); + return cb(null, true); + } + // 'video' field + const ext = path.extname(file.originalname).toLowerCase().replace('.', ''); + if (!isVideoMime(file.mimetype)) return reject('Only video files are allowed'); + if (!isVideoExtension(ext)) return reject(`Video type .${ext} is not allowed`); + cb(null, true); }; /** @@ -125,10 +128,30 @@ const VIDEO_UPLOAD = { export class JourneyController { constructor( private readonly journey: JourneyService, + private readonly storage: StorageService, private readonly books: JourneyBookService, private readonly captureBackfill: PhotoCaptureBackfillService, ) {} + /** + * Commit spooled multer files to their final uploads/journey location. On any + * failure: best-effort removal of both spool leftovers and already-committed + * names, then rethrow (500) — so a DB row never references a missing file. + */ + private async commitJourneyUploads(files: Express.Multer.File[]): Promise { + try { + for (const f of files) await this.storage.put('journey', f.filename, { tmpPath: f.path }); + } catch (err) { + for (const f of files) { + if (f.path) { try { fs.unlinkSync(f.path); } catch { /* best-effort */ } } + } + for (const f of files) { + await this.storage.delete('journey', f.filename).catch(() => {}); + } + throw err; + } + } + // The add call carries only an asset id, so when and where the picture was taken // are fetched from the provider afterwards rather than trusted from the client // (#1614). Detached: a slow or unreachable provider must not hold up the add. @@ -193,6 +216,9 @@ export class JourneyController { if (!files?.length) { throw new HttpException({ error: 'No files uploaded' }, 400); } + // Commit to final storage BEFORE the addPhoto loop: the Immich mirror below + // reads each file at its final uploads/journey path. + await this.commitJourneyUploads(files); const results: unknown[] = []; for (const file of files) { const relativePath = `journey/${file.filename}`; @@ -274,24 +300,51 @@ export class JourneyController { } @Delete('photos/:photoId') - deletePhoto(@CurrentUser() user: User, @Param('photoId') photoId: string) { + async deletePhoto(@CurrentUser() user: User, @Param('photoId') photoId: string) { const photo = this.journey.deletePhoto(Number(photoId), user.id); if (!photo) { throw new HttpException({ error: 'Photo not found' }, 404); } - if (photo.file_path) { - try { fs.unlinkSync(path.join(__dirname, '../../../uploads', photo.file_path)); } catch { /* file already gone */ } - } + await this.deletePhotoObject(photo.file_path); + await this.reclaimDerived(photo.file_path, photo.thumbnail_path); return { success: true }; } + /** + * Best-effort removal of a deleted photo's bytes. file_path rows are the + * uploads-relative 'journey/' form every writer stores; anything else + * (provider rows have null) has no local object. + */ + private async deletePhotoObject(filePath: string | null | undefined): Promise { + if (!filePath) return; + const name = filePath.startsWith('journey/') ? filePath.slice('journey/'.length) : null; + if (!name) return; + await this.storage.delete('journey', name).catch(() => { /* file already gone */ }); + } + + /** + * Delete a photo's derived artifacts with it (spec In-scope fix #2): the + * hash-named thumbnail derivable from file_path, and whatever + * thumbnail_path recorded — a generated thumb (usually the same object) or + * a video's uploaded poster. Both best-effort, both idempotent. + */ + private async reclaimDerived(filePath: string | null | undefined, thumbnailPath: string | null | undefined): Promise { + if (filePath?.startsWith('journey/')) { + await this.storage.delete('journey', journeyThumbName(filePath)).catch(() => { /* already gone */ }); + } + if (thumbnailPath?.startsWith('journey/')) { + await this.storage.delete('journey', thumbnailPath.slice('journey/'.length)).catch(() => { /* already gone */ }); + } + } + // ── Gallery (prefix /:id/gallery — before /:id) ───────────────────────── @Post(':id/gallery/photos') @UseInterceptors(FilesInterceptor('photos')) - uploadGalleryPhotos(@CurrentUser() user: User, @Param('id') id: string, @UploadedFiles() files: Express.Multer.File[] | undefined) { + async uploadGalleryPhotos(@CurrentUser() user: User, @Param('id') id: string, @UploadedFiles() files: Express.Multer.File[] | undefined) { if (!files?.length) { throw new HttpException({ error: 'No files uploaded' }, 400); } + await this.commitJourneyUploads(files); const filePaths = files.map((f) => ({ path: `journey/${f.filename}` })); const photos = this.journey.uploadGalleryPhotos(Number(id), user.id, filePaths); if (!photos.length) { @@ -304,8 +357,13 @@ export class JourneyController { } @Post(':id/gallery/video') - @UseInterceptors(FileFieldsInterceptor([{ name: 'video', maxCount: 1 }, { name: 'poster', maxCount: 1 }], VIDEO_UPLOAD)) - uploadGalleryVideo( + @UseInterceptors(FileFieldsInterceptor( + [{ name: 'video', maxCount: 1 }, { name: 'poster', maxCount: 1 }], + // Inherits the module's storage engine (spool + journeyUploadFilename) via + // the interceptor's shallow options merge; overrides only cap and filter. + { limits: { fileSize: MAX_VIDEO_SIZE }, fileFilter: VIDEO_FILE_FILTER }, + )) + async uploadGalleryVideo( @CurrentUser() user: User, @Param('id') id: string, @UploadedFiles() files: { video?: Express.Multer.File[]; poster?: Express.Multer.File[] } | undefined, @@ -313,8 +371,8 @@ export class JourneyController { ) { const video = files?.video?.[0]; const poster = files?.poster?.[0]; - // multer already wrote both parts; clean them up on any rejection so a POST to - // a journey the user can't edit doesn't orphan a 500 MB clip on disk (#823). + // multer already spooled both parts; clean them up on a pre-commit rejection + // so a bad POST doesn't orphan a 500 MB clip (#823). const cleanup = () => { for (const f of [video, poster]) { if (f?.path) { try { fs.unlinkSync(f.path); } catch { /* best-effort */ } } @@ -324,6 +382,7 @@ export class JourneyController { cleanup(); throw new HttpException({ error: 'No video uploaded' }, 400); } + await this.commitJourneyUploads(poster ? [video, poster] : [video]); const durationMs = body?.duration_ms != null ? Number(body.duration_ms) : null; const photos = this.journey.uploadGalleryPhotos(Number(id), user.id, [{ path: `journey/${video.filename}`, @@ -332,7 +391,11 @@ export class JourneyController { durationMs: durationMs != null && Number.isFinite(durationMs) ? durationMs : null, }]); if (!photos.length) { - cleanup(); + // The parts are already committed (file.path is consumed), so the + // pre-commit unlink cleanup would silently orphan them — delete the + // final objects instead. Same observable outcome as before: bytes gone. + await this.storage.delete('journey', video.filename).catch(() => {}); + if (poster) await this.storage.delete('journey', poster.filename).catch(() => {}); throw new HttpException({ error: 'Not allowed' }, 403); } return { photos }; @@ -364,14 +427,13 @@ export class JourneyController { @Delete(':id/gallery/:journeyPhotoId') @HttpCode(204) - deleteGalleryPhoto(@CurrentUser() user: User, @Param('journeyPhotoId') journeyPhotoId: string): void { + async deleteGalleryPhoto(@CurrentUser() user: User, @Param('journeyPhotoId') journeyPhotoId: string): Promise { const photo = this.journey.deleteGalleryPhoto(Number(journeyPhotoId), user.id); if (!photo) { throw new HttpException({ error: 'Photo not found or not allowed' }, 404); } - if (photo.file_path) { - try { fs.unlinkSync(path.join(__dirname, '../../../uploads', photo.file_path)); } catch { /* file already gone */ } - } + await this.deletePhotoObject(photo.file_path); + await this.reclaimDerived(photo.file_path, photo.thumbnail_path); } // ── Journeys /:id ─────────────────────────────────────────────────────── @@ -396,10 +458,11 @@ export class JourneyController { @Post(':id/cover') @HttpCode(200) // Express answers cover with res.json (200). @UseInterceptors(FileInterceptor('cover')) - cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) { + async cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) { if (!file) { throw new HttpException({ error: 'No file uploaded' }, 400); } + await this.commitJourneyUploads([file]); const result = this.journey.updateJourney(Number(id), user.id, { cover_image: `journey/${file.filename}` }); if (!result) { throw new HttpException({ error: 'Journey not found' }, 404); diff --git a/server/src/nest/journey/journey.module.ts b/server/src/nest/journey/journey.module.ts index b0f4f63b9..706182d3b 100644 --- a/server/src/nest/journey/journey.module.ts +++ b/server/src/nest/journey/journey.module.ts @@ -11,16 +11,28 @@ import { AuthModule } from '../auth/auth.module'; import { MulterModule } from '@nestjs/platform-express'; import { AllowedFileTypesModule } from '../files/allowed-file-types.module'; import { AllowedFileTypesService } from '../files/allowed-file-types.service'; -import { buildJourneyImageUploadOptions } from './journey.controller'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { journeyImageFileFilter, journeyUploadFilename } from './journey.controller'; @Module({ // MemoriesModule: the journey gallery streams provider assets and uploads to Immich. imports: [ MulterModule.registerAsync({ - imports: [AllowedFileTypesModule], - inject: [AllowedFileTypesService], - useFactory: (allowedTypes: AllowedFileTypesService) => buildJourneyImageUploadOptions(allowedTypes), + imports: [StorageModule, AllowedFileTypesModule], + inject: [StorageService, AllowedFileTypesService], + // NO defParamCharset here — deliberate, documented asymmetry with the + // trip-file options (see journey.controller.ts). + useFactory: (storage: StorageService, allowedTypes: AllowedFileTypesService) => + buildStorageUploadOptions(storage, { + category: 'journey', + maxSize: 20 * 1024 * 1024, + fileFilter: journeyImageFileFilter(allowedTypes), + filename: journeyUploadFilename, + }), }), + StorageModule, AuthModule, AddonsModule, MemoriesModule, JourneyDomainModule], controllers: [JourneyController, JourneyPublicController], providers: [JourneyService, JourneyBookService, JourneyMcp], diff --git a/server/src/nest/maps/maps.controller.ts b/server/src/nest/maps/maps.controller.ts index 88d7096a8..07b75d872 100644 --- a/server/src/nest/maps/maps.controller.ts +++ b/server/src/nest/maps/maps.controller.ts @@ -11,7 +11,8 @@ import { UseGuards, } from '@nestjs/common'; import type { Response } from 'express'; -import { createReadStream } from 'node:fs'; +import type { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import type { MapsAutocompleteResult, MapsPlaceDetailsResult, @@ -22,6 +23,8 @@ import type { } from '@trek/shared'; import type { User } from '../../types'; import { MapsService } from './maps.service'; +import { StorageService } from '../storage/storage.service'; +import { isClientAbortError } from '../storage/storage.types'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { CurrentUser } from '../auth/current-user.decorator'; import { MapsSearchDto, MapsAutocompleteDto, MapsResolveUrlDto } from './maps.dto'; @@ -60,7 +63,10 @@ function toHttpException(err: unknown, fallbackMessage: string, defaultStatus: n @Controller('api/maps') @UseGuards(JwtAuthGuard) export class MapsController { - constructor(private readonly maps: MapsService) {} + constructor( + private readonly maps: MapsService, + private readonly storage: StorageService, + ) {} @Post('search') @HttpCode(200) // Express answers with res.json (200); Nest would otherwise default POST to 201. @@ -166,9 +172,9 @@ export class MapsController { } @Get('place-photo/:placeId/bytes') - placePhotoBytes(@Param('placeId') placeId: string, @Res() res: Response): void { - const fp = this.maps.photoBytesPath(placeId); - if (!fp) { + async placePhotoBytes(@Param('placeId') placeId: string, @Res() res: Response): Promise { + const key = await this.maps.photoBytesKey(placeId); + if (!key) { // Same reasoning as the JSON endpoint above, and the bigger half of #1727: // places keep this URL in image_url, so a trip render asks for one photo per // place and every entry the cache sweep dropped answered 404 — a whole map @@ -177,19 +183,36 @@ export class MapsController { this.emptyPhoto(res); return; } - // Stream the cached file directly instead of res.sendFile(): the send library - // bundled under @nestjs/platform-express rejects absolute Windows paths (drive - // letter, no `root`) with a NotFoundError that surfaced as an unhandled 500, - // even though the file exists. A plain read stream serves the bytes - // cross-platform. Cached photos are always JPEG (placePhotoCache writes - // `.jpg`). + // Bytes are stream-piped (spec §Serving pins this route's stream contract); + // headers go on before the stream attempt, matching the old createReadStream + // ordering — an early failure overrides them via emptyPhoto exactly as the + // old error event did. Cached photos are always JPEG (placePhotoCache + // writes `.jpg`). res.set('Cache-Control', 'public, max-age=2592000, immutable'); res.type('image/jpeg'); - const stream = createReadStream(fp); - stream.on('error', () => { + let stream: Readable; + try { + ({ stream } = await this.storage.getStream('photos-google', key)); + } catch { + // Cache-delete race — same terminal state as the old stream error path. if (!res.headersSent) this.emptyPhoto(res); - }); - stream.pipe(res); + return; + } + try { + // pipeline destroys the source on any failure (no leaked read handle), + // matching storage.service.ts's sendToResponse contract. + await pipeline(stream, res); + } catch (err) { + if (!res.headersSent) { + // Same terminal state as the old pre-flush stream error path. + this.emptyPhoto(res); + return; + } + // Bytes are already on the wire — a client abort mid-download is the + // client's problem now, not ours; a real source error still surfaces + // (to the exception filter, which now no-ops safely on headersSent). + if (!isClientAbortError(err)) throw err; + } } // 204 for "no bytes to serve". Overrides the immutable Cache-Control the hit diff --git a/server/src/nest/maps/maps.module.ts b/server/src/nest/maps/maps.module.ts index 8a26d0e3b..d0056199b 100644 --- a/server/src/nest/maps/maps.module.ts +++ b/server/src/nest/maps/maps.module.ts @@ -3,6 +3,7 @@ import { MapsController } from './maps.controller'; import { MapsService } from './maps.service'; import { MapsMcp } from './maps.mcp'; import { PlacePhotosModule } from '../place-photos/place-photos.module'; +import { StorageModule } from '../storage/storage.module'; /** * Maps / geo domain (L3 leaf module). Registered in AppModule. Exports @@ -11,7 +12,7 @@ import { PlacePhotosModule } from '../place-photos/place-photos.module'; * Nothing outside the container consumes this domain, so there is no bridge. */ @Module({ - imports: [PlacePhotosModule], + imports: [PlacePhotosModule, StorageModule], controllers: [MapsController], providers: [MapsService, MapsMcp], exports: [MapsService], diff --git a/server/src/nest/maps/maps.service.ts b/server/src/nest/maps/maps.service.ts index 97a495576..10362a3a9 100644 --- a/server/src/nest/maps/maps.service.ts +++ b/server/src/nest/maps/maps.service.ts @@ -542,8 +542,8 @@ export class MapsService { return this.getPlacePhoto(userId, placeId, lat, lng, name) as Promise; } - photoBytesPath(placeId: string): string | null { - return this.photoCache.serveFilePath(placeId); + photoBytesKey(placeId: string): Promise { + return this.photoCache.serveKey(placeId); } reverse(lat: string, lng: string, lang?: string): Promise { @@ -1816,7 +1816,7 @@ export class MapsService { name?: string, ): Promise<{ photoUrl: string | null; attribution: string | null }> { // Disk cache hit — serve immediately, no Google call - const diskHit = this.photoCache.get(placeId); + const diskHit = await this.photoCache.get(placeId); if (diskHit) return { photoUrl: diskHit.photoUrl, attribution: diskHit.attribution }; // "No photo for this place" is an empty result, not a missing resource: a trip @@ -1842,7 +1842,7 @@ export class MapsService { // or timed out only for a few minutes. let providerFailed = false; - const fetchPromise = (async (): Promise<{ filePath: string; attribution: string | null } | null> => { + const fetchPromise = (async (): Promise<{ attribution: string | null } | null> => { await acquirePhotoFetchSlot(); try { const apiKey = this.getMapsKey(userId); @@ -1851,7 +1851,7 @@ export class MapsService { // (right-click) places and as a fallback when a Google place yields no photo, // so a place added via search still gets a marker image when Google returns // nothing. Returns null (without marking an error) so the caller decides. - const fetchWikimediaFallback = async (): Promise<{ filePath: string; attribution: string | null } | null> => { + const fetchWikimediaFallback = async (): Promise<{ attribution: string | null } | null> => { if (isNaN(lat) || isNaN(lng)) return null; try { const wiki = await this.fetchWikimediaPhoto(lat, lng, name); @@ -1865,7 +1865,7 @@ export class MapsService { } const bytes = Buffer.from(await imgRes.arrayBuffer()); const cached = await this.photoCache.put(placeId, bytes, wiki.attribution); - return { filePath: cached.filePath, attribution: cached.attribution }; + return { attribution: cached.attribution }; } catch { providerFailed = true; return null; @@ -1876,7 +1876,7 @@ export class MapsService { // key, request rejected, no photos, or a failed media download — so the // caller can fall back to Wikimedia; the misses that were Google's fault // flag providerFailed on the way out. - const fetchGooglePhoto = async (): Promise<{ filePath: string; attribution: string | null } | null> => { + const fetchGooglePhoto = async (): Promise<{ attribution: string | null } | null> => { if (!apiKey) return null; // Fetch details to get the photo name @@ -1940,7 +1940,7 @@ export class MapsService { console.error('Failed to persist photo URL to database:', dbErr); } - return { filePath: cached.filePath, attribution }; + return { attribution }; }; // Prefer the Google photo (higher quality); if Google yields nothing, fall diff --git a/server/src/nest/memories/immich.service.ts b/server/src/nest/memories/immich.service.ts index db6f5392a..fbe96529f 100644 --- a/server/src/nest/memories/immich.service.ts +++ b/server/src/nest/memories/immich.service.ts @@ -3,10 +3,12 @@ import type { Response } from 'express'; import { maybe_encrypt_api_key, decrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { checkSsrf, safeFetch } from '../../utils/ssrfGuard'; import { AuditService } from '../audit/audit.service'; +import { StorageService } from '../storage/storage.service'; +import fsPromises from 'node:fs/promises'; +import path from 'node:path'; import { DatabaseService } from '../database/database.service'; import { MemoriesAccessService } from './memories-access.service'; import { fail, handleServiceResult, pipeAsset, type Selection } from './memories.helpers'; -import { UPLOADS_ROOT } from './uploads-root'; const ALBUM_PAGE_SIZE = 1000; const ALBUM_MAX_PAGES = 20; @@ -24,6 +26,7 @@ export class ImmichService { private readonly db: DatabaseService, private readonly audit: AuditService, private readonly access: MemoriesAccessService, + private readonly storage: StorageService, ) {} getImmichCredentials(userId: number) { @@ -575,14 +578,15 @@ export class ImmichService { const creds = this.getImmichCredentials(userId); if (!creds) return null; - const fs = await import('node:fs'); - const path = await import('node:path'); - - const fullPath = path.join(UPLOADS_ROOT, filePath); - if (!fs.existsSync(fullPath)) return null; + // Journey uploads store the uploads-relative 'journey/' path; only + // those are local storage objects this can push. + const name = filePath.startsWith('journey/') ? filePath.slice('journey/'.length) : null; + if (!name) return null; try { - const fileBuffer = fs.readFileSync(fullPath); + // A missing object throws StorageNotFoundError into the catch below — + // the same null the old existsSync guard answered. + const fileBuffer = await this.storage.withLocalFile('journey', name, (p) => fsPromises.readFile(p)); const boundary = '----ImmichUpload' + Date.now(); const ext = path.extname(fileName).toLowerCase(); const mimeTypes: Record = { diff --git a/server/src/nest/memories/journey-thumbs.job.ts b/server/src/nest/memories/journey-thumbs.job.ts new file mode 100644 index 000000000..c354facda --- /dev/null +++ b/server/src/nest/memories/journey-thumbs.job.ts @@ -0,0 +1,37 @@ +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; +import { logInfo, logError } from '../audit/audit-log.logger'; +import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { ThumbnailService } from './thumbnail.service'; + +/** + * Journey thumbnail cleanup: daily — reclaim journey/thumbs/ objects whose + * photo row is gone (spec In-scope fix #2: delete sites only started removing + * the derived thumb with this slice, so historical orphans need a sweeper). + * The Journey addon gate lives inside sweepOrphanThumbs, so toggling the + * addon takes effect without a restart. + */ +@Injectable() +export class JourneyThumbsJob implements OnApplicationBootstrap { + constructor( + private readonly thumbnails: ThumbnailService, + private readonly registrar: CronRegistrarService, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + // Run once on startup to reclaim orphans left over from before this sweeper existed. + void this.sweep(); + this.registrar.register('journey-thumbs', '0 4 * * *', () => { + void this.sweep(); + }); + } + + async sweep(): Promise { + try { + const removed = await this.thumbnails.sweepOrphanThumbs(); + if (removed > 0) logInfo(`Journey thumbnail cleanup: removed ${removed} orphaned thumbnail(s)`); + } catch (err: unknown) { + logError(`Journey thumbnail cleanup: ${err instanceof Error ? err.message : err}`); + } + } +} diff --git a/server/src/nest/memories/memories.module.ts b/server/src/nest/memories/memories.module.ts index cc0be4c77..83d2c4c91 100644 --- a/server/src/nest/memories/memories.module.ts +++ b/server/src/nest/memories/memories.module.ts @@ -9,6 +9,7 @@ import { PhotoCaptureBackfillService } from './photo-capture-backfill.service'; import { ThumbnailService } from './thumbnail.service'; import { TrekPhotoCacheService } from './trek-photo-cache.service'; import { TrekPhotoCacheJob } from './trek-photo-cache.job'; +import { JourneyThumbsJob } from './journey-thumbs.job'; import { SchedulingModule } from '../scheduling/scheduling.module'; import { UnifiedMemoriesController } from './unified.controller'; import { ImmichMemoriesController } from './immich.controller'; @@ -22,6 +23,7 @@ import { PhotoProviderRegistry } from './photo-provider.registry'; import { ImmichPhotoProvider } from './providers/immich.provider'; import { SynologyPhotoProvider } from './providers/synology.provider'; import { NotificationsModule } from '../notifications/notifications.module'; +import { StorageModule } from '../storage/storage.module'; /** * Memories (photo-providers) domain — mounted at /api/integrations/memories. @@ -44,7 +46,7 @@ import { NotificationsModule } from '../notifications/notifications.module'; * adapters themselves so the set is readable in one place. */ @Module({ - imports: [NotificationsModule, AddonsModule, AuditModule, TrekPhotosModule, RealtimeModule, SchedulingModule], + imports: [NotificationsModule, AddonsModule, AuditModule, TrekPhotosModule, RealtimeModule, SchedulingModule, StorageModule], controllers: [UnifiedMemoriesController, ImmichMemoriesController, SynologyMemoriesController], providers: [ MemoriesService, @@ -57,6 +59,7 @@ import { NotificationsModule } from '../notifications/notifications.module'; ThumbnailService, TrekPhotoCacheService, TrekPhotoCacheJob, + JourneyThumbsJob, ImmichPhotoProvider, SynologyPhotoProvider, PhotoProviderRegistry, diff --git a/server/src/nest/memories/photo-capture-backfill.service.ts b/server/src/nest/memories/photo-capture-backfill.service.ts index e548fb6bb..a41908878 100644 --- a/server/src/nest/memories/photo-capture-backfill.service.ts +++ b/server/src/nest/memories/photo-capture-backfill.service.ts @@ -1,8 +1,7 @@ import { Injectable } from '@nestjs/common'; -import path from 'node:path'; import exifr from 'exifr'; -import { UPLOADS_ROOT } from './uploads-root'; import { PhotoResolverService } from './photo-resolver.service'; +import { StorageService } from '../storage/storage.service'; import { TrekPhotosRepository } from '../photos/trek-photos.repository'; /** @@ -23,6 +22,7 @@ export class PhotoCaptureBackfillService { constructor( private readonly resolver: PhotoResolverService, private readonly photos: TrekPhotosRepository, + private readonly storage: StorageService, ) {} /** Fire-and-forget for a batch that was just added. */ @@ -74,18 +74,24 @@ export class PhotoCaptureBackfillService { filePath?: string | null, ): Promise<{ takenAt: string | null; lat: number | null; lng: number | null } | null> { if (!filePath) return null; - // Never let a stored path climb out of the uploads tree. - const abs = path.resolve(UPLOADS_ROOT, filePath); - if (abs !== UPLOADS_ROOT && !abs.startsWith(UPLOADS_ROOT + path.sep)) return null; + // photos.file_path is uploads-relative 'journey/' by every writer; + // anything else reads as a miss (same rule as photo-resolver). Central key + // validation (storage-keys.ts) rejects a name that still carries a path, + // so a stored path can never climb out of the journey category. + if (!filePath.startsWith('journey/')) return null; + const name = filePath.slice('journey/'.length); type Exif = { DateTimeOriginal?: Date; CreateDate?: Date; latitude?: number; longitude?: number }; let parsed: Exif | null; try { - parsed = (await exifr.parse(abs, { - pick: ['DateTimeOriginal', 'CreateDate', 'latitude', 'longitude'], - })) as Exif | null; + parsed = await this.storage.withLocalFile('journey', name, async abs => + (await exifr.parse(abs, { + pick: ['DateTimeOriginal', 'CreateDate', 'latitude', 'longitude'], + })) as Exif | null, + ); } catch { - // Not an image, a truncated upload, a video — none of it is an error here. + // A vanished object, an invalid key, not an image, a truncated upload, + // a video — none of it is an error here. return null; } if (!parsed) return null; diff --git a/server/src/nest/memories/photo-resolver.service.ts b/server/src/nest/memories/photo-resolver.service.ts index fbc9b432c..142435754 100644 --- a/server/src/nest/memories/photo-resolver.service.ts +++ b/server/src/nest/memories/photo-resolver.service.ts @@ -1,16 +1,14 @@ import { Injectable } from '@nestjs/common'; import { Response } from 'express'; -import path from 'path'; -import fs from 'fs'; import type { TrekPhoto } from '../../types'; import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { TrekPhotosRepository } from '../photos/trek-photos.repository'; -import { UPLOADS_ROOT } from './uploads-root'; import { ThumbnailService } from './thumbnail.service'; import { TrekPhotoCacheService } from './trek-photo-cache.service'; import { fail, success, type AssetInfo, type ServiceResult } from './memories.helpers'; import { PhotoProviderRegistry } from './photo-provider.registry'; import type { PhotoAssetRef } from './photo-provider'; +import { StorageService } from '../storage/storage.service'; /** * Resolves a stored trek_photo to bytes or metadata by asking whichever provider @@ -29,8 +27,16 @@ export class PhotoResolverService { private readonly thumbnails: ThumbnailService, private readonly cache: TrekPhotoCacheService, private readonly providers: PhotoProviderRegistry, + private readonly storage: StorageService, ) {} + /** photos.file_path / thumbnail_path are uploads-relative 'journey/…' by + * every writer; anything else reads as a local miss, matching the old + * existsSync(join(uploadsRoot, file_path)) === false path. */ + private storageName(relPath: string): string | null { + return relPath.startsWith('journey/') ? relPath.slice('journey/'.length) : null; + } + // ── Streaming ──────────────────────────────────────────────────────────── private async streamCachedThumbnail( @@ -41,12 +47,12 @@ export class PhotoResolverService { ): Promise { const key = this.cache.cacheKey(photo.provider!, photo.asset_id!, 'thumbnail', photo.owner_id!); - if (this.cache.serveFresh(res, key)) return; + if (await this.cache.serveFresh(res, key)) return; const existing = this.cache.getInFlight(key); if (existing) { const bytes = await existing; - if (bytes && this.cache.serveFresh(res, key)) return; + if (bytes && (await this.cache.serveFresh(res, key))) return; await fallback(); return; } @@ -59,7 +65,7 @@ export class PhotoResolverService { this.cache.setInFlight(key, promise); const bytes = await promise; - if (bytes && this.cache.serveFresh(res, key)) return; + if (bytes && (await this.cache.serveFresh(res, key))) return; await fallback(); } @@ -77,31 +83,24 @@ export class PhotoResolverService { } if (photo.file_path) { - const uploadsRoot = UPLOADS_ROOT; - if (kind === 'thumbnail') { const isVideo = photo.media_type === 'video'; let thumbRel = photo.thumbnail_path ?? null; // Only raster images get a lazily-generated Jimp thumbnail; Jimp can't decode // video, so a video relies on the poster captured at upload (#823). if (!thumbRel && !isVideo) { - const result = await this.thumbnails.ensureLocalThumbnail(uploadsRoot, photo.file_path); + const result = await this.thumbnails.ensureLocalThumbnail(photo.file_path); if (result) { thumbRel = result.thumbnailRelPath; this.photos.recordLocalThumbnail(photo.id, thumbRel, result.width, result.height); } } if (thumbRel) { - const thumbAbs = path.join(uploadsRoot, thumbRel); - if (fs.existsSync(thumbAbs)) { + const thumbName = this.storageName(thumbRel); + if (thumbName && (await this.storage.exists('journey', thumbName).catch(() => false))) { res.set('Cache-Control', 'public, max-age=86400, immutable'); res.set('X-Content-Type-Options', 'nosniff'); - // Explicit { root } + basename, not the absolute path: under the Nest - // ExpressAdapter, res.sendFile(absolutePath) resolves against the - // rewritten req.url and 404s spuriously even though the file is right - // there (same trap as files-download.controller.ts). The existsSync - // above already pins this inside the uploads tree. - res.sendFile(path.basename(thumbAbs), { root: path.dirname(thumbAbs) }); + await this.storage.sendToResponse('journey', thumbName, res); return; } } @@ -114,11 +113,11 @@ export class PhotoResolverService { // Images fall through to original if the thumbnail is unavailable. } - const localPath = path.join(uploadsRoot, photo.file_path); - if (fs.existsSync(localPath)) { + const name = this.storageName(photo.file_path); + if (name && (await this.storage.exists('journey', name).catch(() => false))) { res.set('Cache-Control', 'public, max-age=86400'); res.set('X-Content-Type-Options', 'nosniff'); - res.sendFile(path.basename(localPath), { root: path.dirname(localPath) }); + await this.storage.sendToResponse('journey', name, res); return; } } diff --git a/server/src/nest/memories/thumbnail.service.ts b/server/src/nest/memories/thumbnail.service.ts index d37197cee..e70443ba6 100644 --- a/server/src/nest/memories/thumbnail.service.ts +++ b/server/src/nest/memories/thumbnail.service.ts @@ -1,55 +1,75 @@ import { Jimp } from 'jimp' -import path from 'path' -import fs from 'fs/promises' -import crypto from 'crypto' +import path from 'node:path' +import crypto from 'node:crypto' import { Injectable } from '@nestjs/common' import { ADDON_IDS } from '../../addons' import { AddonsService } from '../addons/addons.service' +import { DatabaseService } from '../database/database.service' +import { StorageService } from '../storage/storage.service' const THUMB_MAX = 800 const THUMB_QUALITY = 80 +/** + * Storage name (category 'journey') of the derived thumbnail for an + * uploads-relative journey photo path ('journey/'). The hash input is + * deliberately the uploads-relative path — the scheme predates the storage + * layer, and changing it would orphan every existing thumbnail. + */ +export function journeyThumbName(originalRelPath: string): string { + const hash = crypto.createHash('sha1').update(originalRelPath).digest('hex').slice(0, 16) + return `thumbs/${hash}.jpg` +} + /** * Downscaled JPEGs for locally uploaded journey photos. Gated on the journey - * addon, because that is the only surface that uploads them. + * addon, because that is the only surface that uploads them. Category-addressed + * since storage slice 4: originals and thumbs are ('journey', ) objects. */ @Injectable() export class ThumbnailService { - constructor(private readonly addons: AddonsService) {} + constructor( + private readonly addons: AddonsService, + private readonly storage: StorageService, + private readonly db: DatabaseService, + ) {} async ensureLocalThumbnail( - uploadsRoot: string, originalRelPath: string, ): Promise<{ thumbnailRelPath: string; width: number; height: number } | null> { if (!this.addons.isAddonEnabled(ADDON_IDS.JOURNEY)) return null - const originalAbs = path.join(uploadsRoot, originalRelPath) - try { await fs.access(originalAbs) } catch { return null } + // The DB stores uploads-relative paths ('journey/'); anything else + // is not a local journey photo and has no thumbnail to derive. + const origName = originalRelPath.startsWith('journey/') ? originalRelPath.slice('journey/'.length) : null + if (!origName) return null // Deterministic name so concurrent requests don't race on the same photo. - const hash = crypto.createHash('sha1').update(originalRelPath).digest('hex').slice(0, 16) - const thumbRel = `journey/thumbs/${hash}.jpg` - const thumbAbs = path.join(uploadsRoot, thumbRel) + const thumbName = journeyThumbName(originalRelPath) + const thumbRel = `journey/${thumbName}` try { const [srcStat, dstStat] = await Promise.all([ - fs.stat(originalAbs), - fs.stat(thumbAbs).catch(() => null), + this.storage.stat('journey', origName), + this.storage.stat('journey', thumbName), ]) + if (!srcStat) return null if (dstStat && dstStat.mtimeMs >= srcStat.mtimeMs) { - const img = await Jimp.read(thumbAbs) + const img = await this.storage.withLocalFile('journey', thumbName, (p) => Jimp.read(p)) return { thumbnailRelPath: thumbRel, width: img.bitmap.width, height: img.bitmap.height } } - await fs.mkdir(path.dirname(thumbAbs), { recursive: true }) - // Jimp auto-applies EXIF orientation on read, matching sharp's .rotate() behavior. - const img = await Jimp.read(originalAbs) + const img = await this.storage.withLocalFile('journey', origName, (p) => Jimp.read(p)) const { width: w, height: h } = img.bitmap if (w > THUMB_MAX || h > THUMB_MAX) { img.scaleToFit({ w: THUMB_MAX, h: THUMB_MAX }) } - await img.write(thumbAbs as `${string}.jpg`, { quality: THUMB_QUALITY }) + // Jimp writes straight to a path, so spool on the category's volume and + // commit through storage for the atomic rename. + const spool = path.join(this.storage.spoolDirFor('journey'), `thumb-${crypto.randomUUID()}.jpg`) + await img.write(spool as `${string}.jpg`, { quality: THUMB_QUALITY }) + await this.storage.put('journey', thumbName, { tmpPath: spool }) return { thumbnailRelPath: thumbRel, width: img.bitmap.width, height: img.bitmap.height } } catch { @@ -57,4 +77,39 @@ export class ThumbnailService { return null } } + + /** + * Reclaim journey/thumbs/ objects no live trek_photos row can account for + * (spec In-scope fix #2). The live set is every hash derivable from a + * journey file_path plus every recorded thumbnail_path under thumbs/ — + * video posters live in journey/ proper and are reclaimed at the delete + * sites, not here. Gated on the Journey addon like the generator, inside + * the method so an admin toggle takes effect without a restart + * (airtrail-sync.job.ts precedent). + */ + async sweepOrphanThumbs(): Promise { + if (!this.addons.isAddonEnabled(ADDON_IDS.JOURNEY)) return 0 + + const live = new Set() + for (const row of this.db.all<{ file_path: string }>( + "SELECT file_path FROM trek_photos WHERE file_path LIKE 'journey/%'", + )) { + live.add(journeyThumbName(row.file_path)) + } + for (const row of this.db.all<{ thumbnail_path: string }>( + "SELECT thumbnail_path FROM trek_photos WHERE thumbnail_path LIKE 'journey/thumbs/%'", + )) { + live.add(row.thumbnail_path.slice('journey/'.length)) + } + + let removed = 0 + for await (const stat of this.storage.list('journey', 'thumbs/')) { + if (!stat.key.endsWith('.jpg') || live.has(stat.key)) continue + try { + await this.storage.delete('journey', stat.key) + removed++ + } catch { /* race */ } + } + return removed + } } diff --git a/server/src/nest/memories/trek-photo-cache.job.ts b/server/src/nest/memories/trek-photo-cache.job.ts index d452fed24..512ee7144 100644 --- a/server/src/nest/memories/trek-photo-cache.job.ts +++ b/server/src/nest/memories/trek-photo-cache.job.ts @@ -19,15 +19,19 @@ export class TrekPhotoCacheJob implements OnApplicationBootstrap { onApplicationBootstrap(): void { if (!this.registrar.isEnabled()) return; // Run once immediately on startup to evict any entries left over from a previous run - try { - this.cache.sweepExpired(); - } catch { /* cache dir may not exist yet — harmless */ } - this.registrar.register('trek-photo-cache', '0 */2 * * *', () => this.tick(), { timezone: 'none' }); + void (async () => { + try { + await this.cache.sweepExpired(); + } catch { /* cache dir may not exist yet — harmless */ } + })(); + this.registrar.register('trek-photo-cache', '0 */2 * * *', () => { + void this.tick(); + }, { timezone: 'none' }); } - tick(): void { + async tick(): Promise { try { - this.cache.sweepExpired(); + await this.cache.sweepExpired(); } catch (err: unknown) { logError(`Trek photo cache cleanup: ${err instanceof Error ? err.message : err}`); } diff --git a/server/src/nest/memories/trek-photo-cache.service.ts b/server/src/nest/memories/trek-photo-cache.service.ts index ddf86302e..291d403e3 100644 --- a/server/src/nest/memories/trek-photo-cache.service.ts +++ b/server/src/nest/memories/trek-photo-cache.service.ts @@ -1,13 +1,11 @@ -import path from 'node:path'; -import fs from 'node:fs'; -import fsPromises from 'node:fs/promises'; import crypto from 'node:crypto'; +import { Readable } from 'node:stream'; import { Injectable } from '@nestjs/common'; import { Response } from 'express'; import { DatabaseService } from '../database/database.service'; -import { UPLOADS_ROOT } from './uploads-root'; +import { StorageService } from '../storage/storage.service'; +import { StorageNotFoundError } from '../storage/storage.types'; -const TREK_PHOTO_DIR = path.join(UPLOADS_ROOT, 'photos/trek'); export const CACHE_TTL = 60 * 60 * 1000; // 1 hour /** @@ -23,28 +21,24 @@ export const CACHE_TTL = 60 * 60 * 1000; // 1 hour */ const inFlight = new Map>(); -function ensureDir(): void { - if (!fs.existsSync(TREK_PHOTO_DIR)) { - fs.mkdirSync(TREK_PHOTO_DIR, { recursive: true }); - } +/** Storage name (category 'photos-trek') for a cache key. */ +function objectName(key: string): string { + return `${key}.bin`; } -function cachedFilePath(key: string): string { - return path.join(TREK_PHOTO_DIR, `${key}.bin`); -} - -/** Disk + metadata cache for provider thumbnails and originals. */ +/** Storage + metadata cache for provider thumbnails and originals. */ @Injectable() export class TrekPhotoCacheService { - constructor(private readonly db: DatabaseService) {} + constructor( + private readonly db: DatabaseService, + private readonly storage: StorageService, + ) {} cacheKey(provider: string, assetId: string, kind: string, ownerId: number): string { return crypto.createHash('sha1').update(`${provider}:${assetId}:${kind}:${ownerId}`).digest('hex'); } - - - getFresh(key: string): { filePath: string; contentType: string } | null { + async getFresh(key: string): Promise<{ contentType: string } | null> { const row = this.db.get<{ content_type: string; fetched_at: number }>( 'SELECT content_type, fetched_at FROM trek_photo_cache_meta WHERE cache_key = ?', key, ); @@ -56,22 +50,16 @@ export class TrekPhotoCacheService { return null; } - const fp = cachedFilePath(key); - if (!fs.existsSync(fp)) { + if (!(await this.storage.exists('photos-trek', objectName(key)))) { this.db.run('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?', key); return null; } - return { filePath: fp, contentType: row.content_type }; + return { contentType: row.content_type }; } async put(key: string, bytes: Buffer, contentType: string): Promise { - ensureDir(); - const fp = cachedFilePath(key); - const tmp = fp + '.tmp'; - - await fsPromises.writeFile(tmp, bytes); - await fsPromises.rename(tmp, fp); + await this.storage.put('photos-trek', objectName(key), Readable.from(bytes)); this.db.run( 'INSERT OR REPLACE INTO trek_photo_cache_meta (cache_key, content_type, fetched_at) VALUES (?, ?, ?)', @@ -79,15 +67,21 @@ export class TrekPhotoCacheService { ); } - serveFresh(res: Response, key: string): boolean { - const entry = this.getFresh(key); + async serveFresh(res: Response, key: string): Promise { + const entry = await this.getFresh(key); if (!entry) return false; res.set('Content-Type', entry.contentType); res.set('Cache-Control', 'public, max-age=3600'); - // Root-relative, see photo-resolver.service.ts — an absolute path 404s under - // the Nest ExpressAdapter. - res.sendFile(path.basename(entry.filePath), { root: path.dirname(entry.filePath) }); + try { + // send() keeps a pre-set Content-Type, so entry.contentType survives the + // .bin extension. + await this.storage.sendToResponse('photos-trek', objectName(key), res); + } catch (err) { + // getFresh→send delete race: fall back like a cache miss. + if (err instanceof StorageNotFoundError && !res.headersSent) return false; + throw err; + } return true; } @@ -100,7 +94,7 @@ export class TrekPhotoCacheService { promise.finally(() => inFlight.delete(key)); } - sweepExpired(): void { + async sweepExpired(): Promise { const cutoff = Date.now() - CACHE_TTL * 2; const stale = this.db.all<{ cache_key: string }>( 'SELECT cache_key FROM trek_photo_cache_meta WHERE fetched_at < ?', cutoff, @@ -108,8 +102,21 @@ export class TrekPhotoCacheService { for (const row of stale) { this.db.run('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?', row.cache_key); - const fp = cachedFilePath(row.cache_key); - if (fs.existsSync(fp)) fs.unlinkSync(fp); + await this.storage.delete('photos-trek', objectName(row.cache_key)); + } + + // Pass 2 (fix #4, spec rev 3.2): getFresh's expiry path deletes the meta + // row but never the object — reclaim row-less .bin objects past the same + // cutoff. The mtime guard spares an in-flight put (object lands before its + // row); nested keys are skipped because the old readdir-free sweep could + // never touch them either. + for await (const stat of this.storage.list('photos-trek')) { + if (stat.key.includes('/') || !stat.key.endsWith('.bin') || stat.mtimeMs >= cutoff) continue; + const key = stat.key.slice(0, -'.bin'.length); + const row = this.db.get('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?', key); + if (!row) { + await this.storage.delete('photos-trek', stat.key).catch(() => { /* race */ }); + } } } } diff --git a/server/src/nest/memories/uploads-root.ts b/server/src/nest/memories/uploads-root.ts deleted file mode 100644 index b0b5d0492..000000000 --- a/server/src/nest/memories/uploads-root.ts +++ /dev/null @@ -1,14 +0,0 @@ -import path from 'node:path'; - -/** - * `/uploads`, the one anchor for everything the app stores on disk. - * - * `__dirname` moves with the file, and three separate copies of this join used - * to live in services/memories/ — every one of them a silent breakage waiting - * for the next move (multer would have created the wrong directory without a - * word). One constant, one test (UPLOADS-ROOT-001), and the depth is stated - * rather than counted by hand: `server/{src,dist}/nest/memories` sits three - * levels under `server/`, the same depth `services/memories/` sat at — so the - * resolved path is unchanged by the move. - */ -export const UPLOADS_ROOT = path.resolve(__dirname, '..', '..', '..', 'uploads'); diff --git a/server/src/nest/notifications/notification-events.ts b/server/src/nest/notifications/notification-events.ts index edd9784ba..5b542efc4 100644 --- a/server/src/nest/notifications/notification-events.ts +++ b/server/src/nest/notifications/notification-events.ts @@ -39,6 +39,7 @@ export type NotifEventType = | 'collab_message' | 'packing_tagged' | 'version_available' + | 'replica_failure' | 'synology_session_cleared' | 'plugin_notification'; @@ -55,12 +56,13 @@ export const ALL_EVENT_TYPES: NotifEventType[] = [ 'collab_message', 'packing_tagged', 'version_available', + 'replica_failure', 'synology_session_cleared', 'plugin_notification', ]; /** Events that target admins only (shown in admin panel, not in user settings). */ -export const ADMIN_SCOPED_EVENTS = new Set(['version_available']); +export const ADMIN_SCOPED_EVENTS = new Set(['version_available', 'replica_failure']); // Compile-time guard: shared NotificationEventKey and server NotifEventType must // stay in sync. It sits next to the union it guards — it used to live in the diff --git a/server/src/nest/notifications/notifications.module.ts b/server/src/nest/notifications/notifications.module.ts index 275132121..a5aa951f6 100644 --- a/server/src/nest/notifications/notifications.module.ts +++ b/server/src/nest/notifications/notifications.module.ts @@ -4,23 +4,36 @@ import { NotificationsMcp } from './notifications.mcp'; import { NotificationsService } from './notifications.service'; import { NotificationPreferencesService } from './notification-preferences.service'; import { ReminderJobsService } from './reminder-jobs.service'; +import { StorageHealthNotifierService } from './storage-health-notifier.service'; import { NtfyService } from './transports/ntfy.service'; import { WebhookService } from './transports/webhook.service'; import { MailerModule } from './mailer/mailer.module'; import { AuthModule } from '../auth/auth.module'; import { SchedulingModule } from '../scheduling/scheduling.module'; +import { StorageModule } from '../storage/storage.module'; /** Notifications domain (L6 leaf module). Registered in AppModule. * AuthModule feeds NotificationsMcp's demo gate; MailerModule carries SMTP, * which lives outside this module so AuthService can send the password-reset * mail without AuthModule and NotificationsModule importing each other. + * StorageModule feeds StorageHealthNotifierService, which bridges replica + * failures into admin notifications — this direction has no cycle (Storage + * imports only AppConfig+Audit+Scheduling). * NotificationsService and NotificationPreferencesService are exported for * in-container consumers (AdminController's dev test send and preferences tab, * the plugin RPC surface, HostSurfaceRpc). */ @Module({ - imports: [AuthModule, MailerModule, SchedulingModule], + imports: [AuthModule, MailerModule, SchedulingModule, StorageModule], controllers: [NotificationsController, AdminNotificationPreferencesController], - providers: [NotificationsService, NotificationPreferencesService, WebhookService, NtfyService, NotificationsMcp, ReminderJobsService], + providers: [ + NotificationsService, + NotificationPreferencesService, + WebhookService, + NtfyService, + NotificationsMcp, + ReminderJobsService, + StorageHealthNotifierService, + ], exports: [NotificationsService, NotificationPreferencesService], }) export class NotificationsModule {} diff --git a/server/src/nest/notifications/notifications.service.ts b/server/src/nest/notifications/notifications.service.ts index 9f3eec369..f5923d080 100644 --- a/server/src/nest/notifications/notifications.service.ts +++ b/server/src/nest/notifications/notifications.service.ts @@ -96,7 +96,7 @@ type RespondResult = { success: boolean; error?: string; notification?: Notifica interface EventNotifConfig { inAppType: 'simple' | 'navigate'; titleKey: string; - textKey: string; + textKey: string | ((params: Record) => string); navigateTextKey?: string; navigateTarget: (params: Record) => string | null; } @@ -210,6 +210,13 @@ const EVENT_NOTIFICATION_CONFIG: Record = { navigateTextKey: 'notif.action.view_admin', navigateTarget: () => '/admin', }, + replica_failure: { + inAppType: 'navigate', + titleKey: 'notif.replica_failure.title', + textKey: (p) => (p.suppressed && p.suppressed !== '0' ? 'notif.replica_failure.textSuppressed' : 'notif.replica_failure.text'), + navigateTextKey: 'notif.action.view', + navigateTarget: () => '/admin?tab=storage', + }, synology_session_cleared: { inAppType: 'simple', titleKey: 'notifications.synologySessionCleared.title', @@ -703,6 +710,10 @@ export class NotificationsService { const navigateTarget = inApp?.navigateTarget ?? config.navigateTarget(params); const fullLink = navigateTarget ? `${appUrl}${navigateTarget}` : undefined; + // Resolve the in-app text key once, so all three in-app branches below agree + // (e.g. replica_failure picks the "N more failures suppressed" phrasing off params.suppressed). + const textKey = typeof config.textKey === 'function' ? config.textKey(params) : config.textKey; + // Fetch sender info once for in-app WS payloads const sender = actorId ? this.db.get<{ username: string; avatar: string | null }>('SELECT username, avatar FROM users WHERE id = ?', actorId) ?? null @@ -728,7 +739,7 @@ export class NotificationsService { event_type: event, title_key: config.titleKey, title_params: params, - text_key: config.textKey, + text_key: textKey, text_params: params, positive_text_key: inApp.positiveTextKey ?? 'notif.action.accept', negative_text_key: inApp.negativeTextKey ?? 'notif.action.decline', @@ -744,7 +755,7 @@ export class NotificationsService { event_type: event, title_key: config.titleKey, title_params: params, - text_key: config.textKey, + text_key: textKey, text_params: params, navigate_text_key: config.navigateTextKey ?? 'notif.action.view', navigate_target: navigateTarget, @@ -758,7 +769,7 @@ export class NotificationsService { event_type: event, title_key: config.titleKey, title_params: params, - text_key: config.textKey, + text_key: textKey, text_params: params, }; } diff --git a/server/src/nest/notifications/replica-failure-debouncer.ts b/server/src/nest/notifications/replica-failure-debouncer.ts new file mode 100644 index 000000000..5419b86c3 --- /dev/null +++ b/server/src/nest/notifications/replica-failure-debouncer.ts @@ -0,0 +1,28 @@ +/** + * Lazy per-backend debounce (backfill/stats/notifications spec): the first + * failure notifies immediately; later failures accumulate and the next one + * past the window rides out as a summary carrying the suppressed count. No + * timers — the summary is sent on the next failure, which is the only moment + * anyone would care. Process-local, like the health ring itself. + */ +export class ReplicaFailureDebouncer { + private readonly windows = new Map(); + + constructor( + private readonly windowMs: number, + private readonly now: () => number = Date.now, + ) {} + + /** null = suppress; otherwise the suppressed-count to report with this send. */ + admit(backend: string): number | null { + const now = this.now(); + const window = this.windows.get(backend); + if (!window || now - window.windowStart > this.windowMs) { + const suppressed = window?.suppressed ?? 0; + this.windows.set(backend, { windowStart: now, suppressed: 0 }); + return suppressed; + } + window.suppressed += 1; + return null; + } +} diff --git a/server/src/nest/notifications/storage-health-notifier.service.ts b/server/src/nest/notifications/storage-health-notifier.service.ts new file mode 100644 index 000000000..edc8435a6 --- /dev/null +++ b/server/src/nest/notifications/storage-health-notifier.service.ts @@ -0,0 +1,49 @@ +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { StorageEventsService } from '../storage/storage-events.service'; +import { NotificationsService } from './notifications.service'; +import { ReplicaFailureDebouncer } from './replica-failure-debouncer'; + +const DEBOUNCE_WINDOW_MS = 60 * 60 * 1000; + +/** + * Bridges storage replica failures into the admin notification channels. + * Lives HERE (not in StorageModule) because NotificationsModule → AuthModule + * → StorageModule — the reverse import would cycle. Subscribes at bootstrap; + * the debouncer keeps an outage from becoming a notification storm. + */ +@Injectable() +export class StorageHealthNotifierService implements OnApplicationBootstrap { + private readonly logger = new Logger(StorageHealthNotifierService.name); + private readonly debouncer = new ReplicaFailureDebouncer(DEBOUNCE_WINDOW_MS); + + constructor( + private readonly events: StorageEventsService, + private readonly notifications: NotificationsService, + ) {} + + onApplicationBootstrap(): void { + this.events.onReplicaFailure((failure) => { + const suppressed = this.debouncer.admit(failure.backend); + if (suppressed === null) return; + void this.notifications + .send({ + event: 'replica_failure', + actorId: null, + scope: 'admin', + targetId: 0, + params: { + backend: failure.backend, + key: failure.key, + op: failure.op, + error: failure.error, + suppressed: String(suppressed), + }, + }) + .catch((err: unknown) => { + this.logger.error( + `replica_failure notification failed: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + }); + } +} diff --git a/server/src/nest/notifications/transports/ntfy.service.ts b/server/src/nest/notifications/transports/ntfy.service.ts index 23288f485..95255259b 100644 --- a/server/src/nest/notifications/transports/ntfy.service.ts +++ b/server/src/nest/notifications/transports/ntfy.service.ts @@ -24,6 +24,7 @@ const NTFY_EVENT_META: Partial { return this.enrichment.credit(key); } } diff --git a/server/src/nest/place-enrichment/place-enrichment.service.ts b/server/src/nest/place-enrichment/place-enrichment.service.ts index 4f916679e..7633064f2 100644 --- a/server/src/nest/place-enrichment/place-enrichment.service.ts +++ b/server/src/nest/place-enrichment/place-enrichment.service.ts @@ -269,7 +269,7 @@ export class PlaceEnrichmentService { const placeId = req.placeId?.trim() || `coords:${req.lat}:${req.lng}`; const lang = req.lang; - const cached = this.readCache(placeId, lang); + const cached = await this.readCache(placeId, lang); if (cached) return cached; // One details lookup feeds all three halves: the pictures need its Commons @@ -318,8 +318,8 @@ export class PlaceEnrichmentService { * obligation does not end there — the place inspector reads this so a picture * that was chosen weeks ago still names whoever made it. */ - credit(key: string): { credit: string | null } { - return { credit: this.photoCache.get(key)?.attribution ?? null }; + async credit(key: string): Promise<{ credit: string | null }> { + return { credit: (await this.photoCache.get(key))?.attribution ?? null }; } /** @@ -492,7 +492,7 @@ export class PlaceEnrichmentService { credit: string | null, fetchBytes: () => Promise, ): Promise { - const hit = this.photoCache.get(key); + const hit = await this.photoCache.get(key); if (hit) return hit.photoUrl; const bytes = await withPhotoFetchSlot(fetchBytes); @@ -662,7 +662,7 @@ export class PlaceEnrichmentService { // ── Result cache ─────────────────────────────────────────────────────────── - private readCache(placeId: string, lang: string | undefined): CachedEnrichment | null { + private async readCache(placeId: string, lang: string | undefined): Promise { try { const row = this.database.get<{ payload_json: string; fetched_at: number }>( 'SELECT payload_json, fetched_at FROM place_details_cache WHERE place_id = ? AND lang = ? AND expanded = ?', @@ -679,7 +679,8 @@ export class PlaceEnrichmentService { // them. Treat any loss as a stale entry and rebuild, rather than serving // the survivors: filtering alone would leave the column empty for the rest // of the week for a place that has pictures perfectly available. - const photos = parsed.photos.filter((photo) => this.photoCache.get(photo.key)); + const checks = await Promise.all(parsed.photos.map((photo) => this.photoCache.get(photo.key))); + const photos = parsed.photos.filter((_, i) => checks[i]); if (photos.length !== parsed.photos.length) return null; const value: CachedEnrichment = { diff --git a/server/src/nest/place-photos/place-photo-cache.job.ts b/server/src/nest/place-photos/place-photo-cache.job.ts index 483691068..b15d5123d 100644 --- a/server/src/nest/place-photos/place-photo-cache.job.ts +++ b/server/src/nest/place-photos/place-photo-cache.job.ts @@ -20,13 +20,15 @@ export class PlacePhotoCacheJob implements OnApplicationBootstrap { onApplicationBootstrap(): void { if (!this.registrar.isEnabled()) return; // Run once on startup to reclaim orphans left over from before this sweeper existed. - this.sweep(); - this.registrar.register('place-photo-cache', '30 3 * * *', () => this.sweep()); + void this.sweep(); + this.registrar.register('place-photo-cache', '30 3 * * *', () => { + void this.sweep(); + }); } - sweep(): void { + async sweep(): Promise { try { - const removed = this.cache.sweepOrphans(); + const removed = await this.cache.sweepOrphans(); if (removed > 0) logInfo(`Place-photo cache cleanup: removed ${removed} orphaned file(s)/row(s)`); } catch (err: unknown) { logError(`Place-photo cache cleanup: ${err instanceof Error ? err.message : err}`); diff --git a/server/src/nest/place-photos/place-photo-cache.service.ts b/server/src/nest/place-photos/place-photo-cache.service.ts index 790154aac..c228b8f7a 100644 --- a/server/src/nest/place-photos/place-photo-cache.service.ts +++ b/server/src/nest/place-photos/place-photo-cache.service.ts @@ -1,11 +1,9 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { Jimp, JimpMime } from 'jimp'; import crypto from 'node:crypto'; -import fs from 'node:fs'; -import fsPromises from 'node:fs/promises'; -import path from 'node:path'; +import { Readable } from 'node:stream'; import { DatabaseService } from '../database/database.service'; -import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { StorageService } from '../storage/storage.service'; // How long a "no photo for this place" answer stays remembered. Nothing about it // changes until a photo appears upstream, so it is worth keeping: without it every @@ -31,7 +29,6 @@ const JPEG_QUALITY = 80; interface CachedPhoto { photoUrl: string; - filePath: string; attribution: string | null; } @@ -44,51 +41,38 @@ interface CachedPhoto { * simultaneous requests for the same uncached place would both call the * provider. That is the reason the nightly sweep (PlacePhotoCacheJob) injects * this container singleton rather than constructing its own. + * + * Bytes live in the 'photos-google' storage category as '.jpg' names. + * The registry's mode-aware prefix reproduces both TREK_PLACE_PHOTO_DIR + * layouts (unset: uploads/photos/google; set: that dir, bare keys), so the + * cache itself is mode-agnostic. */ @Injectable() -export class PlacePhotoCacheService implements OnModuleInit { +export class PlacePhotoCacheService { /** Provider calls that failed recently — in memory only, so a restart retries. */ private readonly recentFailures = new Map(); /** In-flight dedup: stops a stampede when several requests miss the same placeId at once. */ - private readonly inFlight = new Map>(); - /** placeIds whose file was confirmed on disk this session — saves an existsSync per hit. */ + private readonly inFlight = new Map>(); + /** placeIds whose object was confirmed in storage this session — saves a stat per hit. */ private readonly knownOnDisk = new Set(); constructor( private readonly db: DatabaseService, - private readonly env: RuntimeEnvService, + private readonly storage: StorageService, ) {} - /** - * Overridable for tests (mirrors the TREK_DB_FILE seam) so the suite never - * touches the real uploads tree. Read per call rather than frozen at import: - * the tests that set TREK_PLACE_PHOTO_DIR do so after this module loads. - */ - private get photoDir(): string { - return this.env.env().paths.placePhotoDir || path.join(__dirname, '../../../uploads/photos/google'); - } - - /** Ensure the upload dir exists once at startup, so put() never needs a sync FS call. */ - onModuleInit(): void { - try { - fs.mkdirSync(this.photoDir, { recursive: true }); - } catch { - /* already exists */ - } - } - - private filePath(placeId: string): string { + private fileName(placeId: string): string { // Hash to avoid filename collisions — coords:lat:lng pseudo-IDs contain characters that // collapse identically under sanitization (e.g. ':' and '.' both → '_') const hash = crypto.createHash('sha1').update(placeId).digest('hex'); - return path.join(this.photoDir, `${hash}.jpg`); + return `${hash}.jpg`; } private proxyUrl(placeId: string): string { return `/api/maps/place-photo/${encodeURIComponent(placeId)}/bytes`; } - get(placeId: string): CachedPhoto | null { + async get(placeId: string): Promise { const row = this.db.get<{ attribution: string | null }>( 'SELECT attribution FROM google_place_photo_meta WHERE place_id = ? AND error_at IS NULL', placeId, @@ -96,19 +80,17 @@ export class PlacePhotoCacheService implements OnModuleInit { if (!row) return null; - const fp = this.filePath(placeId); - if (!this.knownOnDisk.has(placeId)) { - // First time this placeId is checked this session — verify the file exists on disk. + // First time this placeId is checked this session — verify the object exists. // (Guards against volume wipes or manual deletion between server restarts.) - if (!fs.existsSync(fp)) { + if (!(await this.storage.exists('photos-google', this.fileName(placeId)))) { this.db.run('DELETE FROM google_place_photo_meta WHERE place_id = ?', placeId); return null; } this.knownOnDisk.add(placeId); } - return { photoUrl: this.proxyUrl(placeId), filePath: fp, attribution: row.attribution }; + return { photoUrl: this.proxyUrl(placeId), attribution: row.attribution }; } getErrored(placeId: string): boolean { @@ -170,12 +152,8 @@ export class PlacePhotoCacheService implements OnModuleInit { } async put(placeId: string, bytes: Buffer, attribution: string | null): Promise { - const fp = this.filePath(placeId); - const tmp = fp + '.tmp'; - const resized = await this.downscale(bytes); - await fsPromises.writeFile(tmp, resized); - await fsPromises.rename(tmp, fp); + await this.storage.put('photos-google', this.fileName(placeId), Readable.from(resized)); this.knownOnDisk.add(placeId); this.recentFailures.delete(placeId); @@ -185,19 +163,14 @@ export class PlacePhotoCacheService implements OnModuleInit { placeId, attribution, Date.now(), ); - return { photoUrl: this.proxyUrl(placeId), filePath: fp, attribution }; + return { photoUrl: this.proxyUrl(placeId), attribution }; } - getInFlight( - placeId: string, - ): Promise<{ filePath: string; attribution: string | null } | null> | undefined { + getInFlight(placeId: string): Promise<{ attribution: string | null } | null> | undefined { return this.inFlight.get(placeId); } - setInFlight( - placeId: string, - promise: Promise<{ filePath: string; attribution: string | null } | null>, - ): void { + setInFlight(placeId: string, promise: Promise<{ attribution: string | null } | null>): void { this.inFlight.set(placeId, promise); promise .finally(() => this.inFlight.delete(placeId)) @@ -206,12 +179,16 @@ export class PlacePhotoCacheService implements OnModuleInit { }); } - serveFilePath(placeId: string): string | null { - if (this.knownOnDisk.has(placeId)) return this.filePath(placeId); - const fp = this.filePath(placeId); - if (!fs.existsSync(fp)) return null; + /** + * Storage name (category 'photos-google') for a cached photo, or null when + * the object is missing. + */ + async serveKey(placeId: string): Promise { + const name = this.fileName(placeId); + if (this.knownOnDisk.has(placeId)) return name; + if (!(await this.storage.exists('photos-google', name))) return null; this.knownOnDisk.add(placeId); - return fp; + return name; } /** @@ -235,12 +212,10 @@ export class PlacePhotoCacheService implements OnModuleInit { return !!row; } - private deleteEntry(placeId: string): void { - try { - fs.unlinkSync(this.filePath(placeId)); - } catch { + private async deleteEntry(placeId: string): Promise { + await this.storage.delete('photos-google', this.fileName(placeId)).catch(() => { /* already gone */ - } + }); this.db.run('DELETE FROM google_place_photo_meta WHERE place_id = ?', placeId); this.knownOnDisk.delete(placeId); } @@ -249,41 +224,37 @@ export class PlacePhotoCacheService implements OnModuleInit { * Drop a cache entry if no place references it anymore. Called after a place delete * for prompt reclamation; the nightly sweep is the catch-all for every other path. */ - removeIfUnreferenced(placeId: string): void { + async removeIfUnreferenced(placeId: string): Promise { if (this.isReferenced(placeId)) return; - this.deleteEntry(placeId); + await this.deleteEntry(placeId); } /** * Reclaim orphaned cache files + meta rows. Runs on startup and nightly (scheduler). - * Two passes: (1) meta rows no place references; (2) stray .jpg files with no meta row. + * Two passes: (1) meta rows no place references; (2) stray .jpg objects with no meta row. */ - sweepOrphans(): number { + async sweepOrphans(): Promise { let removed = 0; const rows = this.db.all<{ place_id: string }>('SELECT place_id FROM google_place_photo_meta'); const keepFiles = new Set(); for (const { place_id } of rows) { if (this.isReferenced(place_id)) { - keepFiles.add(`${crypto.createHash('sha1').update(place_id).digest('hex')}.jpg`); + keepFiles.add(this.fileName(place_id)); } else { - this.deleteEntry(place_id); + await this.deleteEntry(place_id); removed++; } } - // Pass 2: files on disk that no surviving meta row maps to (e.g. left over from a - // crash between writeFile and the DB upsert, or a meta row deleted out-of-band). - let entries: string[]; - try { - entries = fs.readdirSync(this.photoDir); - } catch { - entries = []; - } - for (const entry of entries) { - if (!entry.endsWith('.jpg') || keepFiles.has(entry)) continue; + // Pass 2: objects that no surviving meta row maps to (e.g. left over from a + // crash between the object write and the DB upsert, or a meta row deleted + // out-of-band). list() recurses where the old readdirSync didn't — skip + // nested keys so a subdirectory of a relocated cache dir is never swept. + for await (const stat of this.storage.list('photos-google')) { + if (stat.key.includes('/') || !stat.key.endsWith('.jpg') || keepFiles.has(stat.key)) continue; try { - fs.unlinkSync(path.join(this.photoDir, entry)); + await this.storage.delete('photos-google', stat.key); removed++; } catch { /* race */ diff --git a/server/src/nest/place-photos/place-photos.module.ts b/server/src/nest/place-photos/place-photos.module.ts index 90c7385b3..b5a289621 100644 --- a/server/src/nest/place-photos/place-photos.module.ts +++ b/server/src/nest/place-photos/place-photos.module.ts @@ -3,6 +3,7 @@ import { PlacePhotoCacheService } from './place-photo-cache.service'; import { PlacePhotoCacheJob } from './place-photo-cache.job'; import { AppConfigModule } from '../app-config/app-config.module'; import { SchedulingModule } from '../scheduling/scheduling.module'; +import { StorageModule } from '../storage/storage.module'; /** The marker-photo cache. No controller of its own — maps serves the bytes, * places and share read through it, and PlacePhotoCacheJob sweeps it nightly. @@ -11,7 +12,7 @@ import { SchedulingModule } from '../scheduling/scheduling.module'; * imported explicitly because @Global only reaches modules that are in the * graph — which a single-domain e2e TestingModule is not. */ @Module({ - imports: [AppConfigModule, SchedulingModule], + imports: [AppConfigModule, SchedulingModule, StorageModule], providers: [PlacePhotoCacheService, PlacePhotoCacheJob], exports: [PlacePhotoCacheService], }) diff --git a/server/src/nest/places/place-image.ts b/server/src/nest/places/place-image.ts index b35d5fa29..d5b58bd90 100644 --- a/server/src/nest/places/place-image.ts +++ b/server/src/nest/places/place-image.ts @@ -1,11 +1,7 @@ -import fs from 'fs'; -import path from 'path'; +import path from 'node:path'; import { db } from '../../db/database'; +import type { StorageService } from '../storage/storage.service'; -// Custom place images (the user-uploaded thumbnails from #1136) live in their own -// uploads subdir, served statically like covers. Storing the full /uploads/places/… -// path in image_url lets every existing thumbnail renderer show it unchanged. -export const PLACE_IMAGES_DIR = path.resolve(__dirname, '../../../uploads/places'); const URL_PREFIX = '/uploads/places/'; export function placeImageUrl(filename: string): string { @@ -17,20 +13,21 @@ export function isUploadedPlaceImage(url: string | null | undefined): url is str } /** - * Delete a custom place-image file once nothing references it any more. A trip + * Delete a custom place-image object once nothing references it any more. A trip * place and a collection saved-place can share the same uploaded file — save-to- * collection and copy-to-trip copy image_url by reference — so we ref-count across - * both tables before unlinking. The path is confined to PLACE_IMAGES_DIR via - * basename, mirroring tripService.deleteOldCover. Best-effort: never throws. + * both tables before deleting. basename() keeps the storage name confined to the + * 'places' category, mirroring tripService.deleteOldCover. Best-effort: never + * throws (central key validation rejects a hostile stored value; the catch + * swallows it exactly like the old unlink guard). */ -export function reclaimPlaceImage(url: string | null | undefined): void { +export async function reclaimPlaceImage(storage: StorageService, url: string | null | undefined): Promise { if (!isUploadedPlaceImage(url)) return; const referenced = db.prepare('SELECT 1 FROM places WHERE image_url = ? LIMIT 1').get(url) || db.prepare('SELECT 1 FROM collection_places WHERE image_url = ? LIMIT 1').get(url); if (referenced) return; - try { - const resolved = path.resolve(path.join(PLACE_IMAGES_DIR, path.basename(url))); - if (resolved.startsWith(PLACE_IMAGES_DIR + path.sep) && fs.existsSync(resolved)) fs.unlinkSync(resolved); - } catch { /* best-effort */ } + await storage.delete('places', path.basename(url)).catch(() => { + /* best-effort */ + }); } diff --git a/server/src/nest/places/places.controller.ts b/server/src/nest/places/places.controller.ts index 75f875fa2..eb5775ac6 100644 --- a/server/src/nest/places/places.controller.ts +++ b/server/src/nest/places/places.controller.ts @@ -27,7 +27,8 @@ import { isUpdateConflict } from '../common/conflictResult'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { CurrentUser } from '../auth/current-user.decorator'; import { RequirePermission, TripAccessGuard } from '../permissions/trip-access.guard'; -import { PLACE_IMAGE_UPLOAD } from '../common/place-image-upload'; +import { PLACE_IMAGE_FILE_FILTER } from '../common/place-image-upload'; +import { StorageService } from '../storage/storage.service'; import { placeImageUrl } from './place-image'; import { PlaceBulkDeleteDto, @@ -112,6 +113,7 @@ export class PlacesController { constructor( private readonly places: PlacesService, private readonly env: RuntimeEnvService, + private readonly storage: StorageService, ) {} private requireTrip(tripId: string, user: User) { @@ -290,7 +292,7 @@ export class PlacesController { @Post('bulk-delete') @HttpCode(200) // Express answers bulk-delete with res.json (200), unlike the 201 imports. - bulkDelete( + async bulkDelete( @CurrentUser() user: User, @Param('tripId') tripId: string, @Body() body: PlaceBulkDeleteDto, @@ -311,7 +313,7 @@ export class PlacesController { for (const id of scoped) this.places.onDeleted(id); // Read the linked expenses before the delete — afterwards the link is gone (#1298). const expenseIds = this.places.linkedExpenseIds(tripId, scoped); - const deleted = this.places.removeMany(tripId, ids); + const deleted = await this.places.removeMany(tripId, ids); for (const id of deleted) { this.places.broadcast(tripId, 'place:deleted', { placeId: id }, socketId); } @@ -323,7 +325,7 @@ export class PlacesController { @Post('bulk-update') @HttpCode(200) - bulkUpdate( + async bulkUpdate( @CurrentUser() user: User, @Param('tripId') tripId: string, @Body() body: PlaceBulkUpdateDto, @@ -340,7 +342,7 @@ export class PlacesController { if (!('category_id' in body)) { throw new HttpException({ error: 'Provide at least one field to update' }, 400); } - const updated = this.places.updateMany(tripId, ids, { category_id: body.category_id as number | null }); + const updated = await this.places.updateMany(tripId, ids, { category_id: body.category_id as number | null }); for (const place of updated) { this.places.broadcast(tripId, 'place:updated', { place }, socketId); this.places.onUpdated(place.id); @@ -360,8 +362,8 @@ export class PlacesController { @Post(':id/image') @HttpCode(200) - @UseInterceptors(FileInterceptor('image', PLACE_IMAGE_UPLOAD)) - uploadImage( + @UseInterceptors(FileInterceptor('image', { fileFilter: PLACE_IMAGE_FILE_FILTER })) + async uploadImage( @CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @@ -378,9 +380,12 @@ export class PlacesController { if (!file) { throw new HttpException({ error: 'No image uploaded' }, 400); } + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before the DB row references the final path. + await this.storage.put('places', file.filename, { tmpPath: file.path }); // Reuse the existing image_url slot (the top-precedence thumbnail source); the // update path reclaims any previously uploaded file it replaces. - const result = this.places.update(tripId, id, { image_url: placeImageUrl(file.filename) } as never); + const result = await this.places.update(tripId, id, { image_url: placeImageUrl(file.filename) } as never); if (!result || isUpdateConflict(result)) { throw new HttpException({ error: 'Place not found' }, 404); } @@ -437,7 +442,7 @@ export class PlacesController { } @Put(':id') - update( + async update( @CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @@ -450,7 +455,7 @@ export class PlacesController { validateRouteColor(body); validateUrlFields(body); this.requireEdit(trip, user); - const result = this.places.update(tripId, id, body as never, ifMatch); + const result = await this.places.update(tripId, id, body as never, ifMatch); if (!result) { throw new HttpException({ error: 'Place not found' }, 404); } @@ -468,7 +473,7 @@ export class PlacesController { @Delete(':id') @UseGuards(TripAccessGuard) @RequirePermission('place_edit') - remove(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) { + async remove(@CurrentUser() user: User, @Param('tripId') tripId: string, @Param('id') id: string, @Headers('x-socket-id') socketId?: string) { // Scope the id to the trip before the hook (see bulkDelete), then sync the // journey ahead of the actual delete. if (!this.places.get(tripId, id)) { @@ -476,7 +481,7 @@ export class PlacesController { } this.places.onDeleted(Number(id)); const expenseIds = this.places.linkedExpenseIds(tripId, [id]); - if (!this.places.remove(tripId, id)) { + if (!(await this.places.remove(tripId, id))) { throw new HttpException({ error: 'Place not found' }, 404); } this.places.broadcast(tripId, 'place:deleted', { placeId: Number(id) }, socketId); diff --git a/server/src/nest/places/places.helpers.ts b/server/src/nest/places/places.helpers.ts index 9168d233d..532a4616d 100644 --- a/server/src/nest/places/places.helpers.ts +++ b/server/src/nest/places/places.helpers.ts @@ -101,13 +101,13 @@ export interface KmlImportOptions { // Reclaim a deleted place's cached marker photo if nothing else references it. // The cache key is the Google place_id, or — for coordinate-only places — the // pseudo-id embedded in the stored proxy URL (/api/maps/place-photo/{id}/bytes). -export function reclaimPhotoCache(cache: PlacePhotoCacheService, googlePlaceId: string | null, imageUrl: string | null): void { +export async function reclaimPhotoCache(cache: PlacePhotoCacheService, googlePlaceId: string | null, imageUrl: string | null): Promise { const candidates = new Set(); if (googlePlaceId) candidates.add(googlePlaceId); const m = imageUrl?.match(/^\/api\/maps\/place-photo\/(.+)\/bytes$/); if (m) { try { candidates.add(decodeURIComponent(m[1])); } catch { /* malformed url */ } } for (const id of candidates) { - try { cache.removeIfUnreferenced(id); } catch { /* best-effort */ } + try { await cache.removeIfUnreferenced(id); } catch { /* best-effort */ } } } diff --git a/server/src/nest/places/places.mcp.ts b/server/src/nest/places/places.mcp.ts index ac95a64cc..7399b4294 100644 --- a/server/src/nest/places/places.mcp.ts +++ b/server/src/nest/places/places.mcp.ts @@ -182,7 +182,7 @@ export class PlacesMcp { if (this.auth.isDemoUser(ctx.userId)) return demoDenied(); if (!this.db.canAccessTrip(tripId, ctx.userId)) return noAccess(); if (!this.guards.hasTripPermission('place_edit', tripId, ctx.userId)) return permissionDenied(); - const place = this.places.update(String(tripId), String(placeId), { name, description, lat, lng, address, category_id, price, currency, place_time, end_time, duration_minutes, notes, website, phone, transport_mode, osm_id, google_place_id, google_ftid }); + const place = await this.places.update(String(tripId), String(placeId), { name, description, lat, lng, address, category_id, price, currency, place_time, end_time, duration_minutes, notes, website, phone, transport_mode, osm_id, google_place_id, google_ftid }); if (!place) return { content: [{ type: 'text' as const, text: 'Place not found.' }], isError: true }; this.guards.safeBroadcast(tripId, 'place:updated', { place }); return ok({ place }); @@ -235,7 +235,7 @@ export class PlacesMcp { try { this.journey.onPlaceDeleted(placeId); } catch { /* non-fatal */ } // sync journeys before the row is gone // The link is gone once the place is, so read it first (#1298). const expenseIds = this.places.linkedExpenseIds(tripId, [placeId]); - const deleted = this.places.remove(String(tripId), String(placeId)); + const deleted = await this.places.remove(String(tripId), String(placeId)); if (!deleted) return { content: [{ type: 'text' as const, text: 'Place not found.' }], isError: true }; this.guards.safeBroadcast(tripId, 'place:deleted', { placeId }); for (const itemId of expenseIds) this.guards.safeBroadcast(tripId, 'budget:deleted', { itemId }); @@ -347,7 +347,7 @@ export class PlacesMcp { } // The link is gone once the places are, so read it first (#1298). const expenseIds = this.places.linkedExpenseIds(tripId, scoped); - const deleted = this.places.removeMany(String(tripId), placeIds); + const deleted = await this.places.removeMany(String(tripId), placeIds); for (const id of deleted) this.guards.safeBroadcast(tripId, 'place:deleted', { placeId: id }); for (const itemId of expenseIds) this.guards.safeBroadcast(tripId, 'budget:deleted', { itemId }); return ok({ deleted, count: deleted.length }); @@ -392,7 +392,7 @@ export class PlacesMcp { return { content: [{ type: 'text' as const, text: 'Provide at least one field to update.' }], isError: true }; } - const updated = this.places.updateMany(String(tripId), placeIds, fields); + const updated = await this.places.updateMany(String(tripId), placeIds, fields); for (const place of updated) this.guards.safeBroadcast(tripId, 'place:updated', { place }); return ok({ count: updated.length, updatedIds: updated.map(p => p.id), skipped: placeIds.length - updated.length }); } diff --git a/server/src/nest/places/places.module.ts b/server/src/nest/places/places.module.ts index 07725ff76..5a45a9586 100644 --- a/server/src/nest/places/places.module.ts +++ b/server/src/nest/places/places.module.ts @@ -15,6 +15,11 @@ import { QueryHelpersModule } from '../query-helpers/query-helpers.module'; import { MapsModule } from '../maps/maps.module'; import { AuthModule } from '../auth/auth.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; +import { MulterModule } from '@nestjs/platform-express'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { MAX_PLACE_IMAGE_SIZE } from '../common/place-image-upload'; /** * Places domain (S8 — Phase 2 trip sub-domain). Depends on L4 Categories + L5 @@ -25,7 +30,19 @@ import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; * is no places.bridge.ts: nothing outside the container consumes this domain. */ @Module({ - imports: [McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AuthModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule], + imports: [ + // Module-level options deliberately carry NO fileFilter: the GPX/KML import + // route keeps its inline memoryStorage config and would inherit a + // module-level filter via the interceptor's shallow merge — the image + // route passes PLACE_IMAGE_FILE_FILTER inline instead. + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: (storage: StorageService) => + buildStorageUploadOptions(storage, { category: 'places', maxSize: MAX_PLACE_IMAGE_SIZE }), + }), + StorageModule, + McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AuthModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule], controllers: [PlacesController], providers: [PlacesService, PlacesMcp, PlacesRpc], exports: [PlacesService], diff --git a/server/src/nest/places/places.rpc.ts b/server/src/nest/places/places.rpc.ts index 91f20afa6..3eb831a77 100644 --- a/server/src/nest/places/places.rpc.ts +++ b/server/src/nest/places/places.rpc.ts @@ -67,7 +67,7 @@ export class PlacesRpc { } @PluginMethod('places.update', { permission: 'db:write:places' }) - update(params: Record, ctx: PluginRpcContext): unknown { + async update(params: Record, ctx: PluginRpcContext): Promise { const tripId = num(params.tripId, 'tripId'); const placeId = num(params.placeId, 'placeId'); const actor = this.guards.requireActor(ctx, 'place'); @@ -76,7 +76,10 @@ export class PlacesRpc { this.guards.capStrings(parsed.data as Record, PLACE_STR_LIMITS); capUrls(parsed.data as Record); this.guards.requireTripEdit(tripId, actor, PLACE_EDIT_ACTION); - const place = this.places.update(String(tripId), String(placeId), parsed.data as PlaceUpdateInput); + // update is async (it may delete a superseded storage object): await it so the + // null-refusal check compares against the resolved place, not an always-truthy + // Promise, and the broadcast below carries the actual place. + const place = await this.places.update(String(tripId), String(placeId), parsed.data as PlaceUpdateInput); if (place === null) throw new ForbiddenResource(`no place ${placeId} on trip ${tripId}`); this.realtime.broadcast(tripId, 'place:updated', { place }); this.mirrorJourneys(() => this.journey.onPlaceUpdated(placeId)); @@ -84,7 +87,7 @@ export class PlacesRpc { } @PluginMethod('places.delete', { permission: 'db:write:places' }) - delete(params: Record, ctx: PluginRpcContext): unknown { + async delete(params: Record, ctx: PluginRpcContext): Promise { const tripId = num(params.tripId, 'tripId'); const placeId = num(params.placeId, 'placeId'); const actor = this.guards.requireActor(ctx, 'place'); @@ -101,7 +104,9 @@ export class PlacesRpc { this.mirrorJourneys(() => this.journey.onPlaceDeleted(placeId)); // The link is gone once the place is, so read it first (#1298). const expenseIds = this.places.linkedExpenseIds(tripId, [placeId]); - if (!this.places.remove(String(tripId), String(placeId))) { + // remove is async (it deletes the place's storage object): await it so the + // refusal check sees the resolved boolean, not an always-truthy Promise. + if (!(await this.places.remove(String(tripId), String(placeId)))) { throw new ForbiddenResource(`no place ${placeId} on trip ${tripId}`); } this.realtime.broadcast(tripId, 'place:deleted', { placeId }); diff --git a/server/src/nest/places/places.service.ts b/server/src/nest/places/places.service.ts index f73b1ee2e..fea5d1e94 100644 --- a/server/src/nest/places/places.service.ts +++ b/server/src/nest/places/places.service.ts @@ -26,6 +26,7 @@ import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.servic import { type UpdateConflict, isUpdateConflict } from '../common/conflictResult'; import { reclaimPlaceImage } from './place-image'; import { JourneyDomainService } from '../journey/journey-domain.service'; +import { StorageService } from '../storage/storage.service'; import { COORD_DEDUP_TOLERANCE, ENRICH_CONCURRENCY, @@ -113,6 +114,7 @@ export class PlacesService { private readonly unsplash: UnsplashService, private readonly photoCache: PlacePhotoCacheService, private readonly journey: JourneyDomainService, + private readonly storage: StorageService, ) {} verifyTripAccess(tripId: string, userId: number) { @@ -269,20 +271,36 @@ export class PlacesService { // Update place // ------------------------------------------------------------------------- - update( + async update( tripId: string, placeId: string, body: PlaceUpdateInput, ifMatch?: string, - ): PlaceWithTags | UpdateConflict | null { + ): Promise { + const { result, reclaim } = this.applyUpdate(tripId, placeId, body, ifMatch); + if (reclaim !== undefined) await reclaimPlaceImage(this.storage, reclaim); + return result; + } + + /** + * The synchronous DB half of update(). Split out so updateMany() can run it + * inside a better-sqlite3 transaction (which cannot await) and settle the + * storage reclaims after the transaction commits. + */ + private applyUpdate( + tripId: string, + placeId: string, + body: PlaceUpdateInput, + ifMatch?: string, + ): { result: PlaceWithTags | UpdateConflict | null; reclaim?: string | null } { const existingPlace = this.dbs.get('SELECT * FROM places WHERE id = ? AND trip_id = ?', placeId, tripId); - if (!existingPlace) return null; + if (!existingPlace) return { result: null }; // Optimistic concurrency (#1135): when the caller sent the version it based its // edit on and the row has moved on since, reject instead of clobbering. Absent // token => unconditional update (back-compat — old clients keep last-write-wins). if (ifMatch !== undefined && existingPlace.updated_at != null && String(existingPlace.updated_at) !== ifMatch) { - return { conflict: true, server: this.dbs.getPlaceWithTags(placeId) }; + return { result: { conflict: true, server: this.dbs.getPlaceWithTags(placeId) } }; } const { @@ -355,12 +373,13 @@ export class PlacesService { } // A custom uploaded thumbnail (#1136) that was just replaced or cleared leaves - // an orphan file behind — reclaim it once nothing references it any more. - if (image_url !== undefined && image_url !== existingPlace.image_url) { - reclaimPlaceImage(existingPlace.image_url); - } + // an orphan file behind — reclaim it (in the caller, once any enclosing + // transaction committed) if nothing references it any more. + const reclaim = image_url !== undefined && image_url !== existingPlace.image_url + ? existingPlace.image_url + : undefined; - return this.dbs.getPlaceWithTags(placeId); + return { result: this.dbs.getPlaceWithTags(placeId), reclaim }; } // ------------------------------------------------------------------------- @@ -381,7 +400,7 @@ export class PlacesService { return rows.map(r => r.id); } - remove(tripId: string, placeId: string): boolean { + async remove(tripId: string, placeId: string): Promise { const place = this.dbs.get<{ google_place_id: string | null; image_url: string | null }>( 'SELECT google_place_id, image_url FROM places WHERE id = ? AND trip_id = ?', placeId, tripId, ); @@ -393,12 +412,12 @@ export class PlacesService { this.dbs.run('DELETE FROM budget_items WHERE trip_id = ? AND place_id = ?', tripId, placeId); this.dbs.run('DELETE FROM places WHERE id = ?', placeId); }); - reclaimPhotoCache(this.photoCache, place.google_place_id, place.image_url); - reclaimPlaceImage(place.image_url); + await reclaimPhotoCache(this.photoCache, place.google_place_id, place.image_url); + await reclaimPlaceImage(this.storage, place.image_url); return true; } - removeMany(tripId: string, ids: number[]): number[] { + async removeMany(tripId: string, ids: number[]): Promise { if (ids.length === 0) return []; const selectStmt = this.dbs.prepare('SELECT google_place_id, image_url FROM places WHERE id = ? AND trip_id = ?'); const deleteStmt = this.dbs.prepare('DELETE FROM places WHERE id = ?'); @@ -417,8 +436,8 @@ export class PlacesService { }); // Reclaim after the transaction commits so isReferenced() sees the final place set. for (const row of reclaimable) { - reclaimPhotoCache(this.photoCache, row.google_place_id, row.image_url); - reclaimPlaceImage(row.image_url); + await reclaimPhotoCache(this.photoCache, row.google_place_id, row.image_url); + await reclaimPlaceImage(this.storage, row.image_url); } return deleted; } @@ -449,17 +468,22 @@ export class PlacesService { * fields change and everything else is preserved. IDs that don't belong to the * trip are skipped. Returns the updated places. */ - updateMany(tripId: string, ids: number[], body: PlaceUpdateInput): PlaceWithTags[] { + async updateMany(tripId: string, ids: number[], body: PlaceUpdateInput): Promise { if (ids.length === 0) return []; const updated: PlaceWithTags[] = []; + const reclaims: (string | null)[] = []; this.dbs.transaction(() => { for (const id of ids) { - // Bulk update sends no If-Match, so update() never returns a conflict - // here; the guard keeps the types honest. - const place = this.update(tripId, String(id), body); + // Bulk update sends no If-Match, so applyUpdate() never returns a + // conflict here; the guard keeps the types honest. + const { result: place, reclaim } = this.applyUpdate(tripId, String(id), body); if (place && !isUpdateConflict(place)) updated.push(place); + if (reclaim !== undefined) reclaims.push(reclaim); } }); + // Settle reclaims after the transaction commits, so the refcount sees the + // final image_url state. + for (const reclaim of reclaims) await reclaimPlaceImage(this.storage, reclaim); return updated; } diff --git a/server/src/nest/platform/platform.routes.ts b/server/src/nest/platform/platform.routes.ts index da6963e3c..1d30bf779 100644 --- a/server/src/nest/platform/platform.routes.ts +++ b/server/src/nest/platform/platform.routes.ts @@ -1,31 +1,132 @@ import express, { Request, Response, NextFunction } from 'express'; import path from 'node:path'; -import fs from 'node:fs'; import { readEnv } from '../../app-config'; import { verifyJwtAndLoadUser } from '../auth/jwt-verify'; import { db } from '../../db/database'; +import { StorageService } from '../storage/storage.service'; +import { StorageInvalidKeyError, StorageNotFoundError, type StorageCategory } from '../storage/storage.types'; // Platform / transport routes extracted verbatim from createApp() (app.ts) so they can be // mounted on either the legacy Express app or the NestJS Express instance (strangler A6/A8). // -// IMPORTANT — path resolution: the original blocks lived in src/app.ts, where __dirname -// resolves to the directory of app.js (one level above the uploads/public anchor), so they -// used '../uploads/...' and '../public'. This file lives three levels deeper -// (src/nest/platform/), so __dirname is three levels deeper too. The relative prefixes are -// therefore '../../../uploads/...' and '../../../public' — which resolve to the EXACT same -// absolute paths as before. This is the only intentional change; everything else is byte-for-byte -// identical. (rootDir/outDir preserve the tree, so the offset holds in both source/test and -// compiled/dist execution — matching the other nest controllers that use '../../../uploads/...'.) +// IMPORTANT — path resolution: the original block lived in src/app.ts, where __dirname +// resolves to the directory of app.js (one level above the public anchor), so it used +// '../public'. This file lives three levels deeper (src/nest/platform/), so __dirname is +// three levels deeper too — hence '../../../public', which resolves to the EXACT same +// absolute path as before. (rootDir/outDir preserve the tree, so the offset holds in both +// source/test and compiled/dist execution.) The /uploads/* routes no longer anchor paths +// here at all: they address files as (category, name) through StorageService (slice 3). -const UPLOADS_DIR = path.join(__dirname, '../../../uploads'); export const PUBLIC_DIR = path.join(__dirname, '../../../public'); +/** + * express.static replacement for the four public /uploads mounts (storage + * slice 3). Parity contract (spec §Serving): identical ETag / Last-Modified / + * conditional-GET / Range / HEAD behavior — res.sendFile inside + * storage.sendToResponse reads res.req, so the same send() machinery runs — + * and every MISS (not-found, invalid key, dot segment, directory) calls + * next() so the request falls through to the Nest router's standard 404 + * envelope, exactly like express.static's fallthrough. 412/416/5xx go to + * next(err) with send's own http-errors — the same finalhandler path + * express.static uses. + */ +export function storageStaticHandler(storage: StorageService, category: StorageCategory): express.RequestHandler { + return (req: Request, res: Response, next: NextFunction) => { + // serve-static fallthrough: non-GET/HEAD pass straight through (no 405). + if (req.method !== 'GET' && req.method !== 'HEAD') { + next(); + return; + } + // Inside an app.use mount req.path is the mount-stripped, query-free + // pathname. Malformed percent-encoding falls through, matching + // serve-static's silent fallthrough on send's 400 decode error. + let decoded: string; + try { + decoded = decodeURIComponent(req.path); + } catch { + next(); + return; + } + const name = decoded.replace(/^\/+/, ''); + return storage.sendToResponse(category, name, res).catch((err: unknown) => { + // MISS contract: unknown object, invalid key ('' at the mount root, + // '..', dot segments, control chars) → Nest 404 envelope. + if (err instanceof StorageNotFoundError || err instanceof StorageInvalidKeyError) { + next(); + return; + } + const e = err as { status?: number; code?: string; syscall?: string }; + // stat→sendFile race (send 404) and directories: static falls through too. + if (e.status === 404 || e.code === 'EISDIR') { + next(); + return; + } + // Client gone / bytes already on the wire: nothing useful to send — + // mirrors res.sendFile's own default callback (express response.js:454). + if (e.code === 'ECONNABORTED' || e.syscall === 'write') return; + next(err as Error); + }); + }; +} + +async function servePhoto(storage: StorageService, req: Request, res: Response): Promise { + const safeName = path.basename(req.params.filename); + // Parity: after basename(), the old resolve()+startsWith guard could only + // fire when the remaining segment was '..' — keep that exact 403. + if (safeName === '..') { + res.status(403).send('Forbidden'); + return; + } + // Existence is (and was) checked BEFORE auth — the 404-vs-401 order is + // observable and pinned. Invalid keys ('.', dotfiles) read as a miss. + if (!(await storage.exists('photos', safeName).catch(() => false))) { + res.status(404).send('Not found'); + return; + } + const sendPhoto = () => + storage.sendToResponse('photos', safeName, res).catch((err: unknown) => { + // exists→send delete race: same 404 text as the miss branch. + if ((err instanceof StorageNotFoundError || err instanceof StorageInvalidKeyError) && !res.headersSent) { + res.status(404).send('Not found'); + return; + } + throw err; + }); + + const authHeader = req.headers.authorization; + const rawToken = (req.query.token as string) || (authHeader?.startsWith('Bearer ') ? authHeader.slice(7) : null); + if (!rawToken) { + res.status(401).send('Authentication required'); + return; + } + + // JWT session path (with pv check). + const user = verifyJwtAndLoadUser(rawToken); + if (user) return sendPhoto(); + + // Share-token path: require the token to cover the exact trip the + // photo belongs to. Expired tokens fall through to 401. + const photo = db.prepare('SELECT trip_id FROM photos WHERE filename = ?').get(safeName) as { trip_id: number } | undefined; + if (!photo) { + res.status(401).send('Authentication required'); + return; + } + const share = db + .prepare("SELECT trip_id FROM share_tokens WHERE token = ? AND (expires_at IS NULL OR expires_at > datetime('now'))") + .get(rawToken) as { trip_id: number } | undefined; + if (!share || share.trip_id !== photo.trip_id) { + res.status(401).send('Authentication required'); + return; + } + return sendPhoto(); +} + /** * Static + guarded /uploads/* routes. Must be applied BEFORE the API route mounts * (identical to its original position near the top of createApp). */ -export function applyPlatformUploads(app: express.Application): void { +export function applyPlatformUploads(app: express.Application, storage: StorageService): void { // Static: avatars, covers, and journey photos. // // Security model (audit SEC-M9): these paths are unauthenticated by @@ -44,48 +145,19 @@ export function applyPlatformUploads(app: express.Application): void { // not embedded in unauthenticated UI contexts, so that endpoint IS // gated (session JWT with pv, or a share token scoped to the photo's // trip). - app.use('/uploads/avatars', express.static(path.join(UPLOADS_DIR, 'avatars'))); - app.use('/uploads/covers', express.static(path.join(UPLOADS_DIR, 'covers'))); - app.use('/uploads/journey', express.static(path.join(UPLOADS_DIR, 'journey'))); - app.use('/uploads/places', express.static(path.join(UPLOADS_DIR, 'places'))); + app.use('/uploads/avatars', storageStaticHandler(storage, 'avatars')); + app.use('/uploads/covers', storageStaticHandler(storage, 'covers')); + app.use('/uploads/journey', storageStaticHandler(storage, 'journey')); + app.use('/uploads/places', storageStaticHandler(storage, 'places')); // Photos require either a valid logged-in session (via JWT with the // password_version gate) OR a share token that covers the SPECIFIC // photo's trip. Previously any share token for any trip could request // any photo filename by UUID — fine in practice because UUIDs are // unguessable, but the auth model was wrong. - app.get('/uploads/photos/:filename', (req: Request, res: Response) => { - const safeName = path.basename(req.params.filename); - const filePath = path.join(UPLOADS_DIR, 'photos', safeName); - const resolved = path.resolve(filePath); - if (!resolved.startsWith(path.resolve(UPLOADS_DIR, 'photos'))) { - return res.status(403).send('Forbidden'); - } - // existsSync here is cheap and avoids a sendFile error frame; kept - // sync because the handler is already short-lived. - if (!fs.existsSync(resolved)) return res.status(404).send('Not found'); - - const authHeader = req.headers.authorization; - const rawToken = (req.query.token as string) || (authHeader?.startsWith('Bearer ') ? authHeader.slice(7) : null); - if (!rawToken) return res.status(401).send('Authentication required'); - - // JWT session path (with pv check). - const user = verifyJwtAndLoadUser(rawToken); - if (user) return res.sendFile(resolved); - - // Share-token path: require the token to cover the exact trip the - // photo belongs to. Expired tokens fall through to 401. - const photo = db.prepare('SELECT trip_id FROM photos WHERE filename = ?').get(safeName) as { trip_id: number } | undefined; - if (!photo) return res.status(401).send('Authentication required'); - - const share = db.prepare( - "SELECT trip_id FROM share_tokens WHERE token = ? AND (expires_at IS NULL OR expires_at > datetime('now'))" - ).get(rawToken) as { trip_id: number } | undefined; - if (!share || share.trip_id !== photo.trip_id) { - return res.status(401).send('Authentication required'); - } - res.sendFile(resolved); - }); + app.get('/uploads/photos/:filename', (req: Request, res: Response, next: NextFunction) => + servePhoto(storage, req, res).catch(next), + ); // Block direct access to /uploads/files app.use('/uploads/files', (_req: Request, res: Response) => { diff --git a/server/src/nest/share/share.controller.ts b/server/src/nest/share/share.controller.ts index ee0429877..2a789d629 100644 --- a/server/src/nest/share/share.controller.ts +++ b/server/src/nest/share/share.controller.ts @@ -1,8 +1,11 @@ import { Body, Controller, Delete, Get, HttpException, Param, Post, Res, UseGuards } from '@nestjs/common'; import type { Response } from 'express'; -import { createReadStream } from 'node:fs'; +import type { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; import type { User } from '../../types'; import { ShareService } from './share.service'; +import { StorageService } from '../storage/storage.service'; +import { isClientAbortError } from '../storage/storage.types'; import { ShareLinkDto } from './share.dto'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; import { CurrentUser } from '../auth/current-user.decorator'; @@ -78,30 +81,65 @@ export class TripShareController { @Public('share-token validated: a shared trip link has to work for somebody without an account') @Controller('api/shared') export class SharedController { - constructor(private readonly share: ShareService) {} + constructor( + private readonly share: ShareService, + private readonly storage: StorageService, + ) {} /** * Public, token-scoped place-photo proxy. The shared payload rewrites place * image URLs to this route so thumbnails load without a session cookie (the * /api/maps bytes endpoint is JwtAuthGuard'd). The service validates the token * and that the place belongs to its trip; a miss streams nothing and answers - * 404. Declared before the bare ':token' read route. Streaming mirrors - * MapsController.placePhotoBytes (cached photos are always JPEG). + * an empty 204, mirroring MapsController.placePhotoBytes (#1727's rationale + * extended to shared pages: shared payloads keep this URL in place + * image_urls, so evicted cache entries would otherwise 404 once per place + * per render). Declared before the bare ':token' read route. Cached photos + * are always JPEG. */ @Get(':token/place-photo/:placeId/bytes') - placePhotoBytes(@Param('token') token: string, @Param('placeId') placeId: string, @Res() res: Response): void { - const fp = this.share.getSharedPlacePhotoPath(token, placeId); - if (!fp) { - res.status(404).json({ error: 'Photo not cached' }); + async placePhotoBytes(@Param('token') token: string, @Param('placeId') placeId: string, @Res() res: Response): Promise { + const key = await this.share.getSharedPlacePhotoKey(token, placeId); + if (!key) { + this.emptyPhoto(res); return; } + // Bytes are stream-piped like the maps handler; headers go on before the + // stream attempt so an early failure overrides them exactly as the old + // createReadStream error event did. res.set('Cache-Control', 'public, max-age=2592000, immutable'); res.type('image/jpeg'); - const stream = createReadStream(fp); - stream.on('error', () => { - if (!res.headersSent) res.status(404).json({ error: 'Photo not cached' }); - }); - stream.pipe(res); + let stream: Readable; + try { + ({ stream } = await this.storage.getStream('photos-google', key)); + } catch { + // Cache-delete race — same terminal state as the stream error path. + if (!res.headersSent) this.emptyPhoto(res); + return; + } + try { + // pipeline destroys the source on any failure (no leaked read handle), + // matching storage.service.ts's sendToResponse contract. + await pipeline(stream, res); + } catch (err) { + if (!res.headersSent) { + // Same terminal state as the old pre-flush stream error path. + this.emptyPhoto(res); + return; + } + // Bytes are already on the wire — a client abort mid-download is the + // client's problem now, not ours; a real source error still surfaces + // (to the exception filter, which now no-ops safely on headersSent). + if (!isClientAbortError(err)) throw err; + } + } + + // 204 for "no bytes to serve". Overrides the immutable Cache-Control the hit + // path already set — a photo that reappears in the cache must not stay hidden + // behind a month-old empty response. + private emptyPhoto(res: Response): void { + res.set('Cache-Control', 'no-store'); + res.status(204).end(); } @Get(':token') diff --git a/server/src/nest/share/share.module.ts b/server/src/nest/share/share.module.ts index 12e5afc6c..dbd991d10 100644 --- a/server/src/nest/share/share.module.ts +++ b/server/src/nest/share/share.module.ts @@ -6,11 +6,12 @@ import { SettingsModule } from '../settings/settings.module'; import { PermissionsModule } from '../permissions/permissions.module'; import { QueryHelpersModule } from '../query-helpers/query-helpers.module'; import { PlacePhotosModule } from '../place-photos/place-photos.module'; +import { StorageModule } from '../storage/storage.module'; import { AuthModule } from '../auth/auth.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; @Module({ - imports: [McpSharedModule, SettingsModule, PermissionsModule, QueryHelpersModule, AuthModule, PlacePhotosModule], + imports: [McpSharedModule, SettingsModule, PermissionsModule, QueryHelpersModule, AuthModule, PlacePhotosModule, StorageModule], controllers: [TripShareController, SharedController], providers: [ShareService, ShareMcp], }) diff --git a/server/src/nest/share/share.service.ts b/server/src/nest/share/share.service.ts index 11aec174b..e975164a7 100644 --- a/server/src/nest/share/share.service.ts +++ b/server/src/nest/share/share.service.ts @@ -297,14 +297,15 @@ export class ShareService { } /** - * Resolves the on-disk path for a cached place photo requested through a public - * share link. Validates that the token is valid + unexpired and that the place - * actually belongs to that token's trip (matched via the stored proxy URL, which - * covers both Google `placeId` and Wikimedia `coords:` pseudo-IDs without - * depending on google_place_id). Returns null — never throws — so the caller - * answers a plain 404, mirroring the authenticated bytes endpoint. + * Resolves the storage name (category 'photos-google') for a cached place + * photo requested through a public share link. Validates that the token is + * valid + unexpired and that the place actually belongs to that token's trip + * (matched via the stored proxy URL, which covers both Google `placeId` and + * Wikimedia `coords:` pseudo-IDs without depending on google_place_id). + * Returns null — never throws — so the caller answers a plain miss, + * mirroring the authenticated bytes endpoint. */ - getSharedPlacePhotoPath(token: string, placeId: string): string | null { + async getSharedPlacePhotoKey(token: string, placeId: string): Promise { const shareRow = this.dbs.get<{ trip_id: string; share_map: number }>( "SELECT trip_id, share_map FROM share_tokens WHERE token = ? AND (expires_at IS NULL OR expires_at > datetime('now'))", token, @@ -319,6 +320,6 @@ export class ShareService { const place = this.dbs.get('SELECT 1 FROM places WHERE trip_id = ? AND image_url = ?', shareRow.trip_id, expectedUrl); if (!place) return null; - return this.photoCache.serveFilePath(placeId); + return this.photoCache.serveKey(placeId); } } diff --git a/server/src/nest/storage/content-type.ts b/server/src/nest/storage/content-type.ts new file mode 100644 index 000000000..0aa587e97 --- /dev/null +++ b/server/src/nest/storage/content-type.ts @@ -0,0 +1,22 @@ +import path from 'node:path'; + +/** + * Extension → mime for the types TREK stores (backfill/migration copies need + * a contentType for S3 puts; the upload path gets it from the request, copy + * paths must derive it). Deliberately tiny — no dependency. + */ +const MIME_BY_EXT: Record = { + '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.gif': 'image/gif', + '.webp': 'image/webp', '.svg': 'image/svg+xml', '.heic': 'image/heic', '.avif': 'image/avif', + '.mp4': 'video/mp4', '.mov': 'video/quicktime', '.webm': 'video/webm', '.mkv': 'video/x-matroska', + '.mp3': 'audio/mpeg', '.m4a': 'audio/mp4', '.wav': 'audio/wav', + '.pdf': 'application/pdf', '.zip': 'application/zip', '.gz': 'application/gzip', + '.txt': 'text/plain', '.md': 'text/markdown', '.csv': 'text/csv', '.json': 'application/json', + '.ics': 'text/calendar', '.gpx': 'application/gpx+xml', '.kml': 'application/vnd.google-earth.kml+xml', + '.doc': 'application/msword', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xls': 'application/vnd.ms-excel', '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', +}; + +export function contentTypeFor(key: string): string { + return MIME_BY_EXT[path.extname(key).toLowerCase()] ?? 'application/octet-stream'; +} diff --git a/server/src/nest/storage/drivers/local.driver.ts b/server/src/nest/storage/drivers/local.driver.ts new file mode 100644 index 000000000..4c474927d --- /dev/null +++ b/server/src/nest/storage/drivers/local.driver.ts @@ -0,0 +1,211 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import type { Readable } from 'node:stream'; +import { assertValidKey, assertValidPrefix } from '../storage-keys'; +import { + isLocalTempFile, + StorageBackendError, + StorageInvalidKeyError, + StorageNotFoundError, + type ByteRange, + type LocalTempFile, + type ObjectStat, + type StorageDriver, +} from '../storage.types'; + +const SPOOL_DIR_NAME = '.tmp'; +/** Boot spool-reap age gate: entries younger than this survive the sweep. */ +const SPOOL_REAP_AGE_MS = 60 * 60 * 1000; + +function errnoCode(err: unknown): string | undefined { + return (err as NodeJS.ErrnoException).code; +} + +/** ENOENT on the path or any parent segment being a file — both are a miss. */ +function isMissing(err: unknown): boolean { + const code = errnoCode(err); + return code === 'ENOENT' || code === 'ENOTDIR'; +} + +/** + * The only real driver type in v1: plain files under a root directory. + * + * `put` is atomic — stream sources spool into the hidden `/.tmp` and + * commit with a single same-volume `rename()`; caller-owned temp files rename + * directly with an EXDEV copy-fallback for the cross-volume edge case. The + * root is realpath'd at `init()` because in Docker both storage anchors are + * symlinks (Dockerfile: `/app/server/uploads → /app/uploads`, + * `/app/server/data → /app/data`) and a naive `startsWith` containment check + * misbehaves on symlinked roots — the trap backup.impl.ts:423 and + * nest/plugins/paths.ts already work around with `realpathSync`. + */ +export class LocalDriver implements StorageDriver { + readonly id: string; + private readonly configuredRoot: string; + private realRoot: string | null = null; + + constructor(opts: { id: string; root: string }) { + this.id = opts.id; + this.configuredRoot = opts.root; + } + + /** + * Ensure root + spool + category prefix dirs exist and resolve the real + * root. Runs on every registry load (boot and reload()) so a newly + * configured backend is usable immediately; `cleanSpool` is passed at boot + * only — on a reload it could delete an in-flight upload's spool file. + */ + init(opts: { ensurePrefixes?: string[]; cleanSpool?: boolean } = {}): void { + fs.mkdirSync(this.configuredRoot, { recursive: true }); + this.realRoot = fs.realpathSync(this.configuredRoot); + fs.mkdirSync(this.spoolDir(), { recursive: true }); + for (const prefix of opts.ensurePrefixes ?? []) { + assertValidPrefix(prefix); + if (prefix) fs.mkdirSync(path.join(this.realRoot, prefix), { recursive: true }); + } + if (opts.cleanSpool) { + for (const entry of fs.readdirSync(this.spoolDir())) { + const entryPath = path.join(this.spoolDir(), entry); + // Age gate: only reap entries older than the threshold. Crash leftovers + // are always old by the next boot; a fresh entry belongs to another + // process spooling into the same tree right now (the vitest integration + // workers share uploads/, and a second worker booting mid-upload must + // not delete the first one's in-flight spool file). + try { + if (Date.now() - fs.statSync(entryPath).mtimeMs < SPOOL_REAP_AGE_MS) continue; + } catch { + continue; // raced away already — nothing to reap + } + fs.rmSync(entryPath, { recursive: true, force: true }); + } + } + } + + private root(): string { + if (!this.realRoot) { + throw new StorageBackendError(`LocalDriver '${this.id}' used before init()`); + } + return this.realRoot; + } + + private spoolDir(): string { + return path.join(this.root(), SPOOL_DIR_NAME); + } + + getSpoolDir(): string { + return this.spoolDir(); + } + + /** + * Central key validation plus defense-in-depth containment: even a key that + * somehow passed validation must resolve inside the real root. + */ + private resolvePath(key: string): string { + assertValidKey(key); + const resolved = path.resolve(this.root(), key); + if (!resolved.startsWith(this.root() + path.sep)) { + throw new StorageInvalidKeyError(key); + } + return resolved; + } + + getLocalPath(key: string): string { + return this.resolvePath(key); + } + + async put(key: string, source: Readable | LocalTempFile): Promise { + const dest = this.resolvePath(key); + await fs.promises.mkdir(path.dirname(dest), { recursive: true }); + + if (isLocalTempFile(source)) { + try { + await fs.promises.rename(source.tmpPath, dest); + } catch (err) { + if (errnoCode(err) !== 'EXDEV') { + throw new StorageBackendError(`put failed for '${key}' on '${this.id}'`, err); + } + await fs.promises.copyFile(source.tmpPath, dest); + await fs.promises.unlink(source.tmpPath); + } + return; + } + + const spool = path.join(this.spoolDir(), randomUUID()); + try { + await pipeline(source, fs.createWriteStream(spool)); + await fs.promises.rename(spool, dest); + } catch (err) { + await fs.promises.rm(spool, { force: true }); + throw err instanceof Error && !errnoCode(err) + ? err // source-stream failure: surface the caller's own error untouched + : new StorageBackendError(`put failed for '${key}' on '${this.id}'`, err); + } + } + + async getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }> { + const resolved = this.resolvePath(key); + const stat = await this.stat(key); + if (!stat) throw new StorageNotFoundError(key); + const stream = fs.createReadStream( + resolved, + range ? { start: range.start, end: range.end } : undefined, + ); + return { stream, stat }; + } + + async stat(key: string): Promise { + const resolved = this.resolvePath(key); + try { + const st = await fs.promises.stat(resolved); + if (!st.isFile()) return null; + return { key, size: st.size, mtimeMs: st.mtimeMs }; + } catch (err) { + if (isMissing(err)) return null; + throw new StorageBackendError(`stat failed for '${key}' on '${this.id}'`, err); + } + } + + async delete(key: string): Promise { + const resolved = this.resolvePath(key); + try { + await fs.promises.unlink(resolved); + } catch (err) { + if (isMissing(err)) return; // idempotent + throw new StorageBackendError(`delete failed for '${key}' on '${this.id}'`, err); + } + } + + async *list(prefix: string): AsyncIterable { + assertValidPrefix(prefix); + const root = this.root(); + yield* this.walk(prefix ? path.resolve(root, prefix) : root, root); + } + + private async *walk(dir: string, root: string): AsyncIterable { + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(dir, { withFileTypes: true }); + } catch (err) { + if (isMissing(err)) return; // unpopulated prefix — empty, not an error + throw new StorageBackendError(`list failed under '${dir}' on '${this.id}'`, err); + } + for (const entry of entries) { + // Skips the .tmp spool (and any dotfile) defensively — key validation + // already makes dot segments unreachable, but list walks the real disk. + if (entry.name.startsWith('.')) continue; + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + yield* this.walk(entryPath, root); + } else if (entry.isFile()) { + const st = await fs.promises.stat(entryPath); + yield { + key: path.relative(root, entryPath).split(path.sep).join('/'), + size: st.size, + mtimeMs: st.mtimeMs, + }; + } + } + } +} diff --git a/server/src/nest/storage/drivers/mirror.driver.ts b/server/src/nest/storage/drivers/mirror.driver.ts new file mode 100644 index 000000000..7eb809ebd --- /dev/null +++ b/server/src/nest/storage/drivers/mirror.driver.ts @@ -0,0 +1,395 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import type { Readable } from 'node:stream'; +import { assertValidKey, assertValidPrefix } from '../storage-keys'; +import { contentTypeFor } from '../content-type'; +import { + describeError, + isLocalTempFile, + StorageInvalidKeyError, + StorageNotFoundError, + type ByteRange, + type LocalTempFile, + type ObjectStat, + type PutOptions, + type StorageDriver, +} from '../storage.types'; + +export interface ReplicaFailure { + backend: string; + key: string; + op: 'put' | 'delete' | 'stat' | 'list'; + error: string; + at: number; +} + +export interface BackfillProgress { + done: number; + total: number; + copied: number; + skipped: number; + failed: number; + /** Replica objects removed by the sweep phase — absent from the primary and not raced back. */ + deleted: number; +} + +export interface BackfillHooks { + onProgress(progress: BackfillProgress): void; + isCancelled(): boolean; +} + +export interface BackfillResult extends BackfillProgress { + cancelled: boolean; +} + +/** + * Composite backend: one primary (source of truth) plus best-effort replicas. + * Runs entirely over the driver interface — the second in-tree consumer of + * the contract suite. Any category may be assigned a mirror (the v1 + * backups-only rule was lifted by the replicas-on-primary spec); replica + * writes are synchronous in the request path, which is why hot categories + * carry an advisory in the admin UI rather than a prohibition here. + * + * Writes hit the primary first and must succeed; replica failures are + * reported through `onReplicaFailure` (surfaced as health status), never + * thrown. Reads honor one invariant: a read succeeds if ANY member — primary + * or any replica — holds the object. `getStream` and `stat` both fall back + * to replicas on a primary miss (`StorageNotFoundError` / a `null` stat) as + * well as a primary infra error; only a deterministic `StorageInvalidKeyError` + * never triggers fallback (every member would reject the same key). Fallback + * is READ-ONLY and never resurrects anything: a hit found on a replica is + * NOT written back to the primary (or to any other replica that missed it). + * The object stays absent everywhere the read didn't find it — closing that + * staleness window (a primary that failed a delete, or is genuinely missing + * an object a replica still has) is `backfill`'s sweep phase, below: a + * "Sync now" run doesn't just copy forward, it also deletes replica objects + * the primary no longer has. + */ +export class MirrorDriver implements StorageDriver { + readonly id: string; + private readonly primary: StorageDriver; + private readonly replicas: readonly StorageDriver[]; + private readonly tempDir: () => string; + private readonly onReplicaFailure?: (failure: ReplicaFailure) => void; + + constructor(opts: { + id: string; + primary: StorageDriver; + replicas: StorageDriver[]; + /** Global scratch dir (data/tmp) — mirrors have no same-volume spool of their own. */ + tempDir: () => string; + onReplicaFailure?: (failure: ReplicaFailure) => void; + }) { + this.id = opts.id; + this.primary = opts.primary; + this.replicas = [...opts.replicas]; + this.tempDir = opts.tempDir; + this.onReplicaFailure = opts.onReplicaFailure; + } + + getSpoolDir(): null { + return null; // callers fall back to storage.tempDir() + } + + getLocalPath(key: string): string | null { + assertValidKey(key); + return this.primary.getLocalPath?.(key) ?? null; + } + + async put(key: string, source: Readable | LocalTempFile, opts?: PutOptions): Promise { + assertValidKey(key); + // A Readable can only be consumed once, so materialize the source as a + // local file (stream → spool under tempDir(); temp-file sources as-is), + // then feed each target its own stream: the bytes survive the primary + // write for the replicas, and every child put stays atomic. + let file: string; + if (isLocalTempFile(source)) { + file = source.tmpPath; + } else { + file = path.join(this.tempDir(), randomUUID()); + try { + await pipeline(source, fs.createWriteStream(file)); + } catch (err) { + await fs.promises.rm(file, { force: true }); + throw err; + } + } + try { + await this.primary.put(key, fs.createReadStream(file), opts); + for (const replica of this.replicas) { + try { + await replica.put(key, fs.createReadStream(file), opts); + } catch (err) { + this.reportReplicaFailure(replica.id, key, 'put', err); + } + } + } finally { + // Ownership of LocalTempFile sources transferred to us — consume it. + await fs.promises.rm(file, { force: true }); + } + } + + async getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }> { + assertValidKey(key); + try { + return await this.primary.getStream(key, range); + } catch (err) { + // Widened per the "ANY member holds it" invariant: a primary + // StorageNotFoundError now falls through to the replicas too, not just + // an infra error. Only StorageInvalidKeyError skips fallback — every + // member rejects the same key deterministically, so trying them is + // pointless. `fromReplicas`' catch-and-continue is safe to reuse here + // (unlike for `stat` below): a replica miss THROWS StorageNotFoundError, + // so the loop naturally moves on to the next replica. + if (err instanceof StorageInvalidKeyError) throw err; + return this.fromReplicas((replica) => replica.getStream(key, range), err); + } + } + + /** + * Stat-specific fallback loop — deliberately NOT `fromReplicas`. `stat` + * signals a miss by returning `null`, not by throwing, so a naive + * catch-and-continue would treat a replica's clean miss as a "success" and + * stop searching right there. Here `null` explicitly means "keep looking": + * primary hit returns immediately; a primary infra error is remembered but + * doesn't stop the search; replicas are tried in order and the first + * non-null answer wins; an unreachable/erroring replica is treated the + * same as a miss (keep looking); if nothing answers, the remembered + * primary error is rethrown, else the result is a genuine `null`. + */ + async stat(key: string): Promise { + assertValidKey(key); + let primaryErr: unknown; + try { + const primaryStat = await this.primary.stat(key); + if (primaryStat !== null) return primaryStat; + } catch (err) { + if (!shouldFallback(err)) throw err; + primaryErr = err; + } + for (const replica of this.replicas) { + let replicaStat: ObjectStat | null; + try { + replicaStat = await replica.stat(key); + } catch { + continue; // unreachable/erroring replica — keep looking + } + if (replicaStat !== null) return replicaStat; + } + if (primaryErr !== undefined) throw primaryErr; + return null; + } + + async delete(key: string): Promise { + assertValidKey(key); + await this.primary.delete(key); // hard primary failures propagate + for (const replica of this.replicas) { + try { + await replica.delete(key); + } catch (err) { + this.reportReplicaFailure(replica.id, key, 'delete', err); + } + } + } + + async *list(prefix: string): AsyncIterable { + assertValidPrefix(prefix); + let yieldedAny = false; + try { + for await (const stat of this.primary.list(prefix)) { + yieldedAny = true; + yield stat; + } + return; + } catch (err) { + // Once items have been handed out, switching sources mid-stream would + // duplicate or drop keys — only an up-front primary failure falls back. + if (yieldedAny || !shouldFallback(err)) throw err; + for (const replica of this.replicas) { + try { + for await (const stat of replica.list(prefix)) yield stat; + return; + } catch { + /* try next replica */ + } + } + throw err; + } + } + + /** + * Copy existing primary objects under the given prefixes to every replica, + * then sweep each replica for objects the primary no longer has (backfill/ + * stats spec, extended by the deletion-sweep spec — a "Sync now" run makes + * the target match the source, it isn't a one-way trash-can). NOT part of + * the StorageDriver contract — mirror specific and additive. + * + * Three phases: enumerate (honest total), copy, sweep. `done`/`total` only + * ever reflect the copy phase's primary-side enumeration; the sweep phase + * doesn't touch them. `copied`/`skipped`/`failed` count replica-level copy + * outcomes; `deleted` counts replica-level sweep deletions. Replica errors + * (stat/put/delete/list) flow through the same failure funnel the write + * path uses and never abort the run; PRIMARY errors from the copy phase + * (list/getStream) propagate — a getStream failure is deliberately kept + * outside the per-replica try/catch so it can never be misclassified as a + * replica failure. Objects are compared by size only — the same criterion + * a re-put would settle anyway. + * + * Sweep: the copy loop above already visits every primary key under the + * given prefixes, so it doubles as the key-set collection (no extra + * primary listing pass). Once the copy loop finishes, each replica is + * listed per prefix; any key it holds that isn't in that primary key-set + * is re-checked against `primary.stat` — the authoritative race gate for a + * primary that regained the key between the copy loop and here — and only + * deleted if that re-check still comes back empty. A replica whose `list` + * itself fails is reported and skipped; the sweep continues with the next + * replica. Cancellation is honored per key, same as the copy phase. + */ + async backfill(prefixes: readonly string[], hooks: BackfillHooks): Promise { + const progress: BackfillProgress = { done: 0, total: 0, copied: 0, skipped: 0, failed: 0, deleted: 0 }; + for (const prefix of prefixes) { + for await (const _stat of this.primary.list(prefix)) progress.total += 1; + } + hooks.onProgress({ ...progress }); + + const primaryKeys = new Set(); + for (const prefix of prefixes) { + for await (const stat of this.primary.list(prefix)) { + if (hooks.isCancelled()) return { ...progress, cancelled: true }; + primaryKeys.add(stat.key); + + // Phase A: decide, per replica, whether it needs this key. Replica + // stat failures are reported and counted here — they never touch the + // primary-only try/catch below. + const needy: StorageDriver[] = []; + for (const replica of this.replicas) { + try { + const existing = await replica.stat(stat.key); + if (existing && existing.size === stat.size) { + progress.skipped += 1; + } else { + needy.push(replica); + } + } catch (err) { + progress.failed += 1; + this.reportReplicaFailure(replica.id, stat.key, 'stat', err); + } + } + + // Phase B: only if some replica needs the bytes, materialize the key + // once (the put() spool pattern). A primary getStream/pipeline + // failure here RETHROWS — it is a primary error, not a replica one. + if (needy.length > 0) { + const file = path.join(this.tempDir(), randomUUID()); + try { + const { stream } = await this.primary.getStream(stat.key); + await pipeline(stream, fs.createWriteStream(file)); + } catch (err) { + await fs.promises.rm(file, { force: true }); + throw err; + } + try { + for (const replica of needy) { + try { + await replica.put(stat.key, fs.createReadStream(file), { contentType: contentTypeFor(stat.key) }); + progress.copied += 1; + } catch (err) { + progress.failed += 1; + this.reportReplicaFailure(replica.id, stat.key, 'put', err); + } + } + } finally { + await fs.promises.rm(file, { force: true }); + } + } + + progress.done += 1; + if (progress.done > progress.total) progress.total = progress.done; + hooks.onProgress({ ...progress }); + } + } + progress.total = progress.done; + + // Phase 3: deletion sweep. Bounds the staleness window a failed replica + // delete (or a genuinely-missing-on-primary object) can leave open for + // mirror read-fallback — see the class doc above. + for (const prefix of prefixes) { + for (const replica of this.replicas) { + try { + for await (const stat of replica.list(prefix)) { + if (hooks.isCancelled()) return { ...progress, cancelled: true }; + if (primaryKeys.has(stat.key)) continue; + + // Race gate: the primary may have regained this key between the + // copy loop above and now — authoritative, re-checked per key. + let primaryStat: ObjectStat | null; + try { + primaryStat = await this.primary.stat(stat.key); + } catch (err) { + progress.failed += 1; + this.reportReplicaFailure(replica.id, stat.key, 'stat', err); + hooks.onProgress({ ...progress }); + continue; + } + if (primaryStat) { + hooks.onProgress({ ...progress }); // spared — primary regained it + continue; + } + + try { + await replica.delete(stat.key); + progress.deleted += 1; + } catch (err) { + progress.failed += 1; + this.reportReplicaFailure(replica.id, stat.key, 'delete', err); + } + hooks.onProgress({ ...progress }); + } + } catch (err) { + // The replica's list() itself failed (up front or mid-stream): + // report against the prefix, skip this replica, sweep continues. + progress.failed += 1; + this.reportReplicaFailure(replica.id, prefix, 'list', err); + } + } + } + + return { ...progress, cancelled: false }; + } + + private async fromReplicas(fn: (replica: StorageDriver) => Promise, primaryErr: unknown): Promise { + for (const replica of this.replicas) { + try { + return await fn(replica); + } catch { + /* try next replica */ + } + } + throw primaryErr; + } + + private reportReplicaFailure(backend: string, key: string, op: ReplicaFailure['op'], err: unknown): void { + this.onReplicaFailure?.({ + backend, + key, + op, + error: describeError(err), + at: Date.now(), + }); + } +} + +/** + * Used by `list()`'s primary-error branch and by `stat()`'s primary-error + * branch: fall back only on a real (infra) error, never on a deterministic + * bad-key rejection. `StorageNotFoundError` is excluded too, though it's + * moot for `stat` (the contract signals a miss via `null`, never a throw) — + * `getStream`, which DOES throw `StorageNotFoundError` on a miss, uses its + * own inline check instead so a primary miss can fall through to replicas + * per the "ANY member holds it" invariant. + */ +function shouldFallback(err: unknown): boolean { + return !(err instanceof StorageNotFoundError || err instanceof StorageInvalidKeyError); +} diff --git a/server/src/nest/storage/drivers/s3.driver.ts b/server/src/nest/storage/drivers/s3.driver.ts new file mode 100644 index 000000000..6d17bede8 --- /dev/null +++ b/server/src/nest/storage/drivers/s3.driver.ts @@ -0,0 +1,556 @@ +/// +import awsLite from '@aws-lite/client'; +import { PassThrough, Transform, type Readable } from 'node:stream'; +import fs from 'node:fs'; +import { assertValidKey, assertValidPrefix, isValidKey } from '../storage-keys'; +import { + StorageBackendError, + StorageNotFoundError, + isLocalTempFile, + type ByteRange, + type LocalTempFile, + type ObjectStat, + type PutOptions, + type StorageDriver, +} from '../storage.types'; + +/** + * S3-compatible driver over @aws-lite/client + @aws-lite/s3 (spec: + * docs/superpowers/specs/2026-08-17-s3-storage-driver-design.md). The client + * is a driver-internal detail — nothing outside this file imports aws-lite. + * aws-lite exposes no request timeout or abort signal, so the driver owns the + * deadline: per client call for single-request operations, inactivity-based + * for multipart Upload (destroying the interposed monitoring Transform drives + * Upload's own error path — in-flight parts drain, then AbortMultipartUpload + * fires). + */ + +/** One multipart chunk (aws-lite Upload default) — also the peek threshold. */ +export const MULTIPART_THRESHOLD = 10 * 1024 * 1024; + +export interface S3ClientOptions { + endpoint: string; + region: string; + accessKeyId: string; + secretAccessKey: string; + retries: number; + keepAlive: boolean; +} + +/** The S3 surface the driver consumes — structural, so tests inject a plain object. */ +export interface S3Api { + PutObject(input: Record): Promise; + Upload(input: Record): Promise; + GetObject(input: Record): Promise<{ + Body?: Readable; + ContentLength?: number; + ContentRange?: string; + LastModified?: Date; + ETag?: string; + }>; + HeadObject(input: Record): Promise<{ ContentLength?: number; LastModified?: Date; ETag?: string }>; + DeleteObject(input: Record): Promise; + ListObjectsV2(input: Record): Promise< + AsyncIterable<{ Contents?: Array<{ Key?: string; Size?: number; LastModified?: Date }> }> + >; +} + +// `Omit` rather than a bare `S3ClientOptions &`: +// intersecting with the required `keepAlive: boolean` would make the +// `keepAlive?: boolean` below a no-op (TS keeps the property required when +// either side of an `&` requires it). +export type S3DriverOptions = Omit & { + id: string; + bucket: string; + keyPrefix: string; + timeoutMs: number; + keepAlive?: boolean; + clientFactory?: (opts: S3ClientOptions) => Promise; +}; + +/** Bare GetObject content-sniffs; streamResponsePayload is mandatory (spec, Client). */ +export async function defaultClientFactory(opts: S3ClientOptions): Promise { + const client = await awsLite({ + accessKeyId: opts.accessKeyId, + secretAccessKey: opts.secretAccessKey, + region: opts.region, + endpoint: opts.endpoint, + retries: opts.retries, + keepAlive: opts.keepAlive, + plugins: [await import('@aws-lite/s3')], + }); + // The `/// ` above pulls in the + // declaration-merged `AwsLiteClient.S3` (https://aws-lite.org/using-typescript), + // so `client.S3` itself needs no cast. This one narrow cast remains because + // of a genuine structural mismatch: the real PutObject/Upload signatures + // require `Bucket`/`Key` (plus other required fields), which S3Api's + // deliberately permissive `Record` input can't guarantee — + // method-syntax parameter bivariance still rejects that narrowing. This is + // the one sanctioned `as unknown as` boundary; no other cast in this file. + return client.S3 as unknown as S3Api; +} + +function statusCode(err: unknown): number | undefined { + return (err as { statusCode?: number }).statusCode; +} +function errorCode(err: unknown): string | undefined { + const code = (err as { code?: unknown }).code; + return typeof code === 'string' ? code : undefined; +} +/** aws-lite 404s: statusCode 404, code 'NoSuchKey' (GetObject) or 'NotFound' (HeadObject, bodiless). */ +function isNotFound(err: unknown): boolean { + const code = errorCode(err); + return statusCode(err) === 404 || code === 'NoSuchKey' || code === 'NotFound'; +} + +/** + * Workaround for a verified upstream signing defect, scoped ONLY to the + * buffered zero-byte `Body` path in `put()` below — a zero-byte PutObject + * `Body: Buffer` fails against real S3-compatible servers (confirmed against + * MinIO) with 403 SignatureDoesNotMatch, while any non-empty `Body` succeeds + * via the identical code path. Root cause, traced through the installed deps: + * + * - `@aws-lite/s3`'s PutObject (node_modules/@aws-lite/s3/src/put-object.mjs:99, + * the default `!ApplyChecksum` "unsigned payload" branch) always sets + * `headers['content-length'] = dataSize` — a *number*, including `0`. + * - `aws4`'s signer (node_modules/aws4/aws4.js:151-152, aws4@1.13.2) then runs + * `if (request.body && !headers['Content-Length'] && !headers['content-length']) + * headers['Content-Length'] = Buffer.byteLength(request.body)`. + * `!0 === true`, so aws4 treats the already-set `0` as "unset" and injects + * a SECOND, differently-cased `Content-Length` header onto the same object. + * JS object keys are case-sensitive, so both survive; aws4's + * `filterHeaders()` (aws4.js:302-308) lowercases each header entry for + * canonicalization WITHOUT deduping by that lowercased name — the two + * distinct keys stay two distinct entries — so `canonicalHeaders()`/ + * `signedHeaders()` (aws4.js:311-322, which just map/join whatever + * `filterHeaders()` produced) sign `content-length` *twice* + * (`SignedHeaders=content-length;content-length;...`, confirmed live + * against MinIO (see the manual procedure in docker-compose.minio-test.yml)). + * MinIO's own canonicalization of the + * headers it actually received doesn't reproduce that duplicate, so the + * signatures disagree — 403. Any non-empty body makes + * `headers['content-length']` a truthy number, so aws4's guard never fires + * and no duplicate is ever added — hence the bug is exactly the size-0 + * boundary, nothing else. + * + * The `File` route (`put-object.mjs:87`, `payload = createReadStream(File)`) + * is NOT affected and must NOT get this workaround: aws-lite's request + * builder (`@aws-lite/client/src/request/index.js:60,90`) sets + * `params.body = undefined` for any stream payload, so aws4.js:151's + * `request.body && ...` guard never even evaluates truthy there — a + * zero-byte temp file signs correctly today. Only the buffered `Body: Buffer` + * call site below needs this. + * + * `ApplyChecksum: true` routes PutObject through @aws-lite/s3's OTHER branch + * (put-object.mjs:106-118), which never pre-sets Content-Length itself — + * aws4 then adds it exactly once, and independently computes a correct + * `x-amz-content-sha256` for the (empty) body instead of `UNSIGNED-PAYLOAD`. + * Verified end-to-end against real MinIO: 200 OK, correct empty-object ETag. + * The extra checksum cost is nil for a 0-byte body. Delete this workaround + * once aws4 dedupes header keys case-insensitively before signing (or + * @aws-lite/s3 stops pre-setting a falsy-numeric Content-Length). + */ +function zeroByteChecksumWorkaround(size: number): { ApplyChecksum: true } | Record { + return size === 0 ? { ApplyChecksum: true } : {}; +} + +export class S3Driver implements StorageDriver { + readonly id: string; + private readonly bucket: string; + private readonly keyPrefix: string; + private readonly timeoutMs: number; + private readonly clientOptions: S3ClientOptions; + private readonly clientFactory: (opts: S3ClientOptions) => Promise; + private clientPromise: Promise | null = null; + + constructor(opts: S3DriverOptions) { + this.id = opts.id; + this.bucket = opts.bucket; + this.keyPrefix = normalizeKeyPrefix(opts.keyPrefix); + this.timeoutMs = opts.timeoutMs; + this.clientFactory = opts.clientFactory ?? defaultClientFactory; + this.clientOptions = { + endpoint: opts.endpoint, + region: opts.region, + accessKeyId: opts.accessKeyId, + secretAccessKey: opts.secretAccessKey, + retries: opts.retries, + keepAlive: opts.keepAlive ?? true, + }; + } + + /** No local filesystem — callers fall back to storage.tempDir(). */ + getSpoolDir(): null { + return null; + } + + async put(key: string, source: Readable | LocalTempFile, opts?: PutOptions): Promise { + assertValidKey(key); + const s3 = await this.client(); + const base: Record = { Bucket: this.bucket, Key: this.keyPrefix + key }; + if (opts?.contentType) base.ContentType = opts.contentType; + + if (isLocalTempFile(source)) { + // Ownership transfer: consume the tmp file on success AND failure. + try { + const { size } = await fs.promises.stat(source.tmpPath); + if (size < MULTIPART_THRESHOLD) { + await this.deadlined(s3.PutObject({ ...base, File: source.tmpPath }), `put '${key}'`); + } else { + await this.uploadMonitored(s3, base, fs.createReadStream(source.tmpPath), key); + } + } catch (err) { + throw this.wrap(err, `put failed for '${key}'`); + } finally { + await fs.promises.rm(source.tmpPath, { force: true }); + } + return; + } + + // Bounded peek (spec, Client): a stream ending within the threshold — + // including zero bytes — becomes one PutObject with the buffered Body (the + // same memory Upload would buffer for part 1, and one round trip instead + // of 3+; aws-lite's Upload also breaks on empty streams). A larger stream + // flows into Upload with the prefix re-attached. An unbounded stream never + // reaches PutObject, which would buffer it wholesale. + const peeked = await peekStream(source, MULTIPART_THRESHOLD); // source errors surface untouched + // `readableEnded` guards a race: a source whose 'end' fires between the + // peek resolving (size > threshold) and pipe() attaching would never end + // the PassThrough — but ended means the chunks already hold everything. + if (peeked.ended || source.readableEnded) { + try { + const body = Buffer.concat(peeked.chunks); + await this.deadlined( + s3.PutObject({ ...base, Body: body, ...zeroByteChecksumWorkaround(body.length) }), + `put '${key}'`, + ); + } catch (err) { + throw this.wrap(err, `put failed for '${key}'`); + } + return; + } + const combined = new PassThrough(); + for (const chunk of peeked.chunks) combined.write(chunk); + source.pipe(combined); + source.on('error', (err) => combined.destroy(err)); + try { + await this.uploadMonitored(s3, base, combined, key, source); + } catch (err) { + throw this.wrap(err, `put failed for '${key}'`); + } + } + + /** + * Multipart Upload under an INACTIVITY deadline: a fixed whole-operation + * deadline would break large backups, so the timer resets on data movement. + * + * The reset signal comes from an interposed Transform, NOT a `'data'` + * listener on `body` itself: `Readable.on('data', ...)` calls `resume()` + * on the next tick, so `body` would start flowing before aws-lite's Upload + * ever attaches its own consumer (it awaits a full CreateMultipartUpload + * round trip first) — anything emitted in that window has nowhere to land + * and is lost, silently truncating the object. A Transform's readable side + * only flows once something actually reads it, so bytes queue in its + * internal buffer (bounded by backpressure) until Upload's real consumer + * attaches; nothing is dropped, and "inactivity" tracks real data + * movement through the transform. + * + * Expiry destroys the Transform, the piped `body`, and (for the streaming + * put path) the original source — driving Upload's own error path + * (in-flight parts drain, then AbortMultipartUpload fires, verified + * upstream). Any OTHER Upload failure gets the same cleanup so a failed + * large upload never leaks an open fd/socket. + */ + private async uploadMonitored( + s3: S3Api, + base: Record, + body: Readable, + key: string, + sourceToDestroy?: Readable, + ): Promise { + let timer: NodeJS.Timeout | undefined; + const expire = () => { + const err = new StorageBackendError( + `put '${key}' stalled for ${this.timeoutMs}ms on '${this.id}' — multipart upload aborted`, + ); + monitored.destroy(err); + body.destroy(err); + sourceToDestroy?.destroy(err); + }; + const reset = () => { + clearTimeout(timer); + timer = setTimeout(expire, this.timeoutMs); + timer.unref?.(); + }; + const monitored = new Transform({ + transform(chunk, _encoding, callback) { + reset(); + callback(null, chunk); + }, + }); + // Workaround for a verified upstream defect: a ContentType param corrupts + // @aws-lite/s3 Upload's final CompleteMultipartUpload request. Upload + // builds that request as `headers: { ...xml, ...getHeadersFromParams( + // params) }` (node_modules/@aws-lite/s3/src/upload.mjs:76-81) — the + // param-derived `content-type` (e.g. application/zip) is spread AFTER and + // clobbers the `application/xml` default, which flips @aws-lite/client's + // payload serializer (client/src/request/index.js:69) off its + // `XMLContentType` branch: the parts list is sent as JSON, and the server + // rejects it with 400 MalformedXML ("XML syntax error on line 0: EOF" — + // confirmed live against MinIO AIStor; any S3 would refuse the same). + // Dropping ContentType here means multipart objects are stored without + // it, which TREK never reads back — sendToResponse derives Content-Type + // from the key/caller, never from object metadata. The single-request + // PutObject paths keep ContentType. Delete this (and pass `base` + // through) once Upload stops leaking params into its Complete headers. + const { ContentType: _dropped, ...uploadBase } = base; + // Attached BEFORE pipe()/destroy() can fire: an 'error' emitted on a + // Readable with no listener crashes the process, and both a genuine + // upstream read failure and our own expire()/catch cleanup emit one. + // `monitored` gets its own last-resort no-op: real aws-lite Upload + // attaches its own listener to `monitored` (the `Body` it receives) + // only after its CreateMultipartUpload round trip, so an expiry that + // fires in that window would otherwise destroy an unlistened stream — + // Upload's later read still observes the already-errored/destroyed + // state, so this doesn't swallow anything Upload would have seen. + // + // `onBodyError` is intentionally never `.off()`'d: `expire()` calls + // `monitored.destroy(err)` and `body.destroy(err)` back to back, each + // scheduling its own deferred 'error' emission. The FIRST (monitored's) + // rejects `s3.Upload(...)`, whose `catch`/`finally` runs as an + // interleaved microtask — before the SECOND (body's) emission gets its + // turn. Removing the listener there would leave that second emission + // unlistened and crash the process (verified with a standalone repro). + // Leaving it attached is harmless: `body` is single-use and discarded + // with `uploadMonitored`, so nothing keeps accumulating listeners. + const onBodyError = (err: Error) => monitored.destroy(err); + body.on('error', onBodyError); + monitored.on('error', () => {}); + reset(); + body.pipe(monitored); + try { + await s3.Upload({ ...uploadBase, Body: monitored, ChunkSize: MULTIPART_THRESHOLD }); + } catch (err) { + monitored.destroy(); + body.destroy(); + sourceToDestroy?.destroy(); + throw err; + } finally { + clearTimeout(timer); + } + } + + async getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }> { + assertValidKey(key); + const s3 = await this.client(); + const params: Record = { + Bucket: this.bucket, + Key: this.keyPrefix + key, + // Mandatory: a bare GetObject content-sniffs and parses JSON/XML bodies. + streamResponsePayload: true, + }; + if (range) params.Range = `bytes=${range.start}-${range.end ?? ''}`; + let res; + try { + // Deadline covers time-to-headers; body streaming is never deadlined. + res = await this.deadlined(s3.GetObject(params), `get '${key}'`); + } catch (err) { + if (isNotFound(err)) throw new StorageNotFoundError(key); + throw this.wrap(err, `get failed for '${key}'`); + } + if (!res.Body) throw new StorageBackendError(`get returned no body for '${key}' on '${this.id}'`); + // LocalDriver parity: on a ranged read the stat is the FULL object's — + // total from the 206 Content-Range, never the ranged ContentLength. + const size = range ? totalFromContentRange(res.ContentRange, key, this.id) : (res.ContentLength ?? 0); + // The ETag's opaque value passes through untouched — on a ranged GET it + // is still the FULL object's tag (same as `size` above), which is exactly + // what a conditional/If-Range client needs. + return { + stream: res.Body, + stat: { key, size, mtimeMs: res.LastModified?.getTime() ?? 0, etag: normalizeEtag(res.ETag) }, + }; + } + + async stat(key: string): Promise { + assertValidKey(key); + const s3 = await this.client(); + try { + const res = await this.deadlined(s3.HeadObject({ Bucket: this.bucket, Key: this.keyPrefix + key }), `stat '${key}'`); + return { + key, + size: res.ContentLength ?? 0, + mtimeMs: res.LastModified?.getTime() ?? 0, + etag: normalizeEtag(res.ETag), + }; + } catch (err) { + if (isNotFound(err)) return null; + throw this.wrap(err, `stat failed for '${key}'`); + } + } + + async delete(key: string): Promise { + assertValidKey(key); + const s3 = await this.client(); + try { + await this.deadlined(s3.DeleteObject({ Bucket: this.bucket, Key: this.keyPrefix + key }), `delete '${key}'`); + } catch (err) { + if (isNotFound(err)) return; // idempotent — S3 204s on missing keys; some compatibles 404 + throw this.wrap(err, `delete failed for '${key}'`); + } + } + + async *list(prefix: string): AsyncIterable { + assertValidPrefix(prefix); + const s3 = await this.client(); + let pages: AsyncIterable<{ Contents?: Array<{ Key?: string; Size?: number; LastModified?: Date }> }>; + try { + pages = await this.deadlined( + s3.ListObjectsV2({ Bucket: this.bucket, Prefix: this.keyPrefix + prefix, paginate: 'iterator' }), + `list '${prefix}'`, + ); + } catch (err) { + throw this.wrap(err, `list failed under '${prefix}'`); + } + const it = pages[Symbol.asyncIterator](); + for (;;) { + let next: IteratorResult<{ Contents?: Array<{ Key?: string; Size?: number; LastModified?: Date }> }>; + try { + // Per-page deadline (each iteration is one ListObjectsV2 request). + next = await this.deadlined(it.next(), `list '${prefix}'`); + } catch (err) { + throw this.wrap(err, `list failed under '${prefix}'`); + } + if (next.done) return; + for (const obj of next.value.Contents ?? []) { + const objectKey = obj.Key ?? ''; + if (!objectKey.startsWith(this.keyPrefix)) continue; // out-of-band write outside our namespace + const key = objectKey.slice(this.keyPrefix.length); + // Defensive parity with LocalDriver's dotfile skip: such keys can only + // exist from out-of-band writes; serving/sweeps must never see them. + if (!isValidKey(key)) continue; + yield { key, size: obj.Size ?? 0, mtimeMs: obj.LastModified?.getTime() ?? 0 }; + } + } + } + + private client(): Promise { + // Cached across calls; a failed init clears the cache so the next call retries. + this.clientPromise ??= this.clientFactory(this.clientOptions).catch((err) => { + this.clientPromise = null; + throw new StorageBackendError(`s3 client init failed on '${this.id}'`, err); + }); + return this.clientPromise; + } + + private deadlined(op: Promise, what: string): Promise { + let timer: NodeJS.Timeout | undefined; + const expiry = new Promise((_, reject) => { + timer = setTimeout( + // Cannot abort the socket (aws-lite has no signal) — the call is + // rejected and a hung request lingers until the socket dies. + () => reject(new StorageBackendError(`${what} timed out after ${this.timeoutMs}ms on '${this.id}'`)), + this.timeoutMs, + ); + timer.unref?.(); + }); + return Promise.race([op, expiry]).finally(() => clearTimeout(timer)) as Promise; + } + + private wrap(err: unknown, message: string): Error { + if (err instanceof StorageBackendError || err instanceof StorageNotFoundError) return err; + return new StorageBackendError(`${message} on '${this.id}'`, err); + } +} + +/** + * '' stays ''; anything else becomes one or more slash-joined segments with a + * trailing slash, or throws StorageInvalidKeyError. app-config/env.ts's + * s3Preconditions validates the same slash-stripped shape at boot — keep the + * two strip-regexes identical (`/^\/+|\/+$/g`). + */ +function normalizeKeyPrefix(raw: string): string { + const trimmed = raw.replace(/^\/+|\/+$/g, ''); + if (trimmed === '') return ''; + assertValidPrefix(`${trimmed}/`); + return `${trimmed}/`; +} + +/** + * Re-quotes the entity-tag aws-lite hands back. Verified live against AIStor + * RELEASE.2026-08-07T18-34-35Z: the SDK strips the ETag's surrounding quotes + * before it reaches us (`a925576942e9…`, not `"a925576942e9…"`), and an + * unquoted entity-tag is not a valid HTTP ETag (RFC 9110 §8.8.3, + * `etag = DQUOTE *etagc DQUOTE`) — emitting one straight into an ETag + * response header invites intermediaries to drop or mangle it. Only the + * quotes the SDK removed are restored; the opaque value itself is never + * touched, and an already-quoted or weak tag (from another S3-compatible + * backend, or a future SDK that stops stripping) passes through unchanged. + * An absent or empty ETag stays absent — "the backend didn't say" (see + * ObjectStat.etag), never a synthesized tag. + */ +function normalizeEtag(raw: string | undefined): string | undefined { + if (!raw) return undefined; + return raw.startsWith('"') || raw.startsWith('W/"') ? raw : `"${raw}"`; +} + +/** 'bytes 2-5/10' → 10. A 206 without a parseable total is a backend defect. */ +function totalFromContentRange(contentRange: string | undefined, key: string, id: string): number { + const total = contentRange?.match(/\/(\d+)$/)?.[1]; + if (total === undefined) { + throw new StorageBackendError(`ranged get for '${key}' on '${id}' returned no Content-Range total`); + } + return Number(total); +} + +/** + * Reads up to `limit` bytes in paused mode. Resolves `ended: true` when the + * stream finished within the limit (chunks = the whole payload), else + * `ended: false` with the peeked prefix still unconsumed downstream. Rejects + * with the source's own error untouched (LocalDriver parity). + */ +function peekStream(source: Readable, limit: number): Promise<{ chunks: Buffer[]; size: number; ended: boolean }> { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + const cleanup = () => { + source.off('readable', onReadable); + source.off('end', onEnd); + source.off('error', onError); + }; + const onReadable = () => { + for (;;) { + const raw = source.read() as Buffer | string | null; + if (raw === null) return; + // A source not explicitly in binary mode (e.g. an object-mode + // Readable.from(string), as used by callers/tests) can yield string + // chunks; PutObject's Body and Buffer.concat both need real Buffers. + const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw); + chunks.push(chunk); + size += chunk.length; + // Strictly `>`, not `>=`: a stream ending at exactly `limit` bytes is + // "ended within the limit" and buffers to PutObject, while a + // LocalTempFile of exactly `MULTIPART_THRESHOLD` (see put()'s `size < + // MULTIPART_THRESHOLD` check) goes to Upload instead. Both boundaries + // are correct on their own terms — deliberate asymmetry, not a bug; + // do not "align" them. + if (size > limit) { + cleanup(); + resolve({ chunks, size, ended: false }); + return; + } + } + }; + const onEnd = () => { + cleanup(); + resolve({ chunks, size, ended: true }); + }; + const onError = (err: Error) => { + cleanup(); + reject(err); + }; + source.on('readable', onReadable); + source.on('end', onEnd); + source.on('error', onError); + onReadable(); + }); +} diff --git a/server/src/nest/storage/http-serving.ts b/server/src/nest/storage/http-serving.ts new file mode 100644 index 000000000..0668b462c --- /dev/null +++ b/server/src/nest/storage/http-serving.ts @@ -0,0 +1,133 @@ +import type { ObjectStat } from './storage.types'; + +/** + * The pure decision half of the proxied byte-path (storage.service.ts's + * `sendStreamed`). Local backends get conditional-GET/Range/HEAD for free from + * express's `res.sendFile` (send); a remote backend has no such machinery, so + * this module reproduces the same contract from an `ObjectStat` plus the + * request headers — no fs, no driver, no Response. Everything here is a pure + * function so the state machine (304 before 416, weak-vs-strong comparison, + * suffix ranges, the degraded no-validator case) is provable in isolation. + * + * Deliberately out of scope: If-Match / If-Unmodified-Since (and therefore + * 412). No TREK caller sends a precondition on a GET, and answering one + * half-heartedly is worse than not advertising it. + */ + +/** Request headers as node/express hands them over (repeats arrive as arrays). */ +export type ServingHeaders = Record; + +/** Cache validators for one object; either half may be absent (see validatorsFor). */ +export interface Validators { + etag?: string; + /** HTTP-date. Present only when the backend knows a real mtime. */ + lastModified?: string; +} + +export type RangeDecision = + | { kind: 'full' } + | { kind: 'partial'; start: number; end: number } + | { kind: 'unsatisfiable' }; + +/** + * ETag: the driver's own (S3 returns one on every HEAD/GET), else the same + * weak `W/"-"` express's send derives, so a local→remote + * migration doesn't invalidate every cached object. + * + * `mtimeMs === 0` is the degraded case, not a 1970 timestamp: S3-compatible + * backends can answer without a LastModified, and emitting + * `Thu, 01 Jan 1970 00:00:00 GMT` would let a client "revalidate" against a + * date that is fresh forever. No mtime means no Last-Modified and no derived + * ETag — serving without validators is correct, just uncached. + */ +export function validatorsFor(stat: ObjectStat): Validators { + const validators: Validators = {}; + if (stat.etag) validators.etag = stat.etag; + if (stat.mtimeMs > 0) { + const mtime = Math.floor(stat.mtimeMs); + validators.etag ??= `W/"${stat.size.toString(16)}-${mtime.toString(16)}"`; + validators.lastModified = new Date(mtime).toUTCString(); + } + return validators; +} + +/** + * RFC 9110 §13.2.2 evaluation order: If-None-Match, when present, is the whole + * answer — a client that sends both headers has an ETag it trusts more than + * the clock, so a tag mismatch means 200 even if the date says fresh. + */ +export function isNotModified(headers: ServingHeaders, validators: Validators): boolean { + const ifNoneMatch = firstValue(headers['if-none-match']); + if (ifNoneMatch !== undefined) return etagMatches(ifNoneMatch, validators.etag); + + const ifModifiedSince = firstValue(headers['if-modified-since']); + if (ifModifiedSince === undefined || validators.lastModified === undefined) return false; + const since = Date.parse(ifModifiedSince); + if (Number.isNaN(since)) return false; + // Both sides are HTTP-dates (second granularity), so this is already the + // truncated comparison the spec asks for — validators.lastModified was + // itself produced by toUTCString(). + return Date.parse(validators.lastModified) <= since; +} + +/** + * Single-range only: a multi-range request would need a + * `multipart/byteranges` body, which no TREK client asks for and which is + * never wrong to answer with the full entity (RFC 9110 §14.2 — a server MAY + * ignore Range). Anything that isn't exactly one `bytes=-` spec — + * commas, another unit, trailing junk — takes the same 200 path. + */ +export function decideRange(headers: ServingHeaders, size: number, etag: string | undefined): RangeDecision { + const raw = firstValue(headers['range']); + if (raw === undefined) return { kind: 'full' }; + + // If-Range: only a strong-exact tag lets the client resume; anything else + // (a weak tag, an HTTP-date form, a tag we can't match) means the entity may + // have changed under them, so send all of it rather than a stitched-together + // corrupt file. + const ifRange = firstValue(headers['if-range']); + if (ifRange !== undefined && !(etag !== undefined && !isWeak(etag) && ifRange.trim() === etag)) { + return { kind: 'full' }; + } + + const match = /^bytes=(\d*)-(\d*)$/.exec(raw.trim()); + if (match === null) return { kind: 'full' }; + const [, rawStart, rawEnd] = match; + if (rawStart === '' && rawEnd === '') return { kind: 'full' }; // 'bytes=-' is malformed, not a range + + const last = size - 1; + if (rawStart === '') { + // Suffix form: the last N bytes, computed from the pre-stat size — the + // reason the serving path stats before it streams. + const suffix = Number(rawEnd); + if (suffix === 0 || size === 0) return { kind: 'unsatisfiable' }; + return { kind: 'partial', start: Math.max(0, size - suffix), end: last }; + } + + const start = Number(rawStart); + if (start > last) return { kind: 'unsatisfiable' }; // also the size-0 case (last === -1) + const end = rawEnd === '' ? last : Math.min(Number(rawEnd), last); + if (end < start) return { kind: 'unsatisfiable' }; + return { kind: 'partial', start, end }; +} + +/** Weak comparison (RFC 9110 §8.8.3.2) — the only one If-None-Match may use. */ +function etagMatches(ifNoneMatch: string, etag: string | undefined): boolean { + const raw = ifNoneMatch.trim(); + if (raw === '*') return true; // "any current representation" — and we have one + if (etag === undefined) return false; + const target = stripWeak(etag); + return raw.split(',').some((candidate) => stripWeak(candidate.trim()) === target); +} + +function isWeak(etag: string): boolean { + return etag.startsWith('W/'); +} + +function stripWeak(etag: string): string { + return isWeak(etag) ? etag.slice(2) : etag; +} + +function firstValue(header: string | string[] | undefined): string | undefined { + return Array.isArray(header) ? header[0] : header; +} diff --git a/server/src/nest/storage/storage-admin.controller.ts b/server/src/nest/storage/storage-admin.controller.ts new file mode 100644 index 000000000..6f065656d --- /dev/null +++ b/server/src/nest/storage/storage-admin.controller.ts @@ -0,0 +1,177 @@ +import { Body, Controller, Delete, Get, HttpCode, HttpException, Param, Post, Put, Req, UseGuards } from '@nestjs/common'; +import type { Request } from 'express'; +import type { StorageUsage } from '@trek/shared'; +import { redactStorageSecrets } from './storage-secrets'; +import type { User } from '../../types'; +import { JwtAuthGuard } from '../auth/jwt-auth.guard'; +import { AdminGuard } from '../auth/admin.guard'; +import { CurrentUser } from '../auth/current-user.decorator'; +import { AuditService } from '../audit/audit.service'; +import { getClientIp } from '../audit/client-ip'; +import { ManagedForbidden } from '../common/managed'; +import { StorageAdminService } from './storage-admin.service'; +import { StorageConfigDto, StorageMigrationRequestDto, StorageTestRequestDto } from './storage-admin.dto'; +import { + BackfillBusyError, + BackfillTargetError, + MigrationRequestError, + MigrationTargetError, +} from './storage-jobs.service'; +import { StatsBusyError } from './storage-stats.service'; +import { StorageConflictError } from './storage.types'; + +/** + * /api/admin/storage — the admin surface over the storage registry (spec: + * docs/superpowers/specs/2026-08-19-storage-admin-config-design.md). Backends, + * category assignment, and mirror composition are managed here; secrets are + * stored encrypted and only ever rendered as the mask. Pipeline refusals are + * operator-grade registry messages, surfaced verbatim in the 400 envelope. + */ +@Controller('api/admin/storage') +@UseGuards(JwtAuthGuard, AdminGuard) +@ManagedForbidden('storage backends and their credentials are hoster-level configuration') +export class StorageAdminController { + constructor( + private readonly service: StorageAdminService, + private readonly audit: AuditService, + ) {} + + @Get() + get() { + return this.service.state(); + } + + @Put() + update(@CurrentUser() user: User, @Body() body: StorageConfigDto, @Req() req: Request) { + try { + this.service.applyConfig(body); + } catch (err) { + // The conflict branch must come before the blanket 400: a + // StorageConflictError IS an Error, so the generic catch-all below + // would otherwise swallow it into a 400 (audit #7 — the client can't + // tell "stale version, reload and retry" from "bad config" without 409). + if (err instanceof StorageConflictError) throw new HttpException({ error: err.message }, 409); + throw new HttpException({ error: err instanceof Error ? err.message : String(err) }, 400); + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_update', + ip: getClientIp(req), + details: redactStorageSecrets(body) as unknown as Record, + }); + // The defaults-tab contract: never echo the request — answer the fresh effective world. + return this.service.state(); + } + + @Post('test') + @HttpCode(200) // a probe answers 200 with per-target results, not a 201 resource + async test(@CurrentUser() user: User, @Body() body: StorageTestRequestDto, @Req() req: Request) { + let result; + try { + result = await this.service.testBackend(body.backend); + } catch (err) { + throw new HttpException({ error: err instanceof Error ? err.message : String(err) }, 400); + } + // Audited because the probe writes and deletes an object; names only, never options. + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_test', + ip: getClientIp(req), + details: { + backend: body.backend.name, + type: body.backend.type, + ok: result.ok, + targets: result.targets.map(({ name, ok }) => ({ name, ok })), + }, + }); + return result; + } + + /** Start a replica catch-up for a routed mirror. One at a time, globally. */ + @Post('backends/:name/backfill') + @HttpCode(200) + backfillStart(@CurrentUser() user: User, @Param('name') name: string, @Req() req: Request): { started: true } { + try { + this.service.startBackfill(name); + } catch (err) { + if (err instanceof BackfillTargetError) throw new HttpException({ error: err.message }, 404); + if (err instanceof BackfillBusyError) throw new HttpException({ error: err.message }, 409); + throw err; + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_backfill', + ip: getClientIp(req), + details: { backend: name }, + }); + return { started: true }; + } + + @Delete('backends/:name/backfill') + backfillCancel(@CurrentUser() user: User, @Param('name') name: string, @Req() req: Request): { cancelled: true } { + if (!this.service.cancelBackfill(name)) { + throw new HttpException({ error: `no active sync for '${name}'` }, 404); + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_backfill_cancel', + ip: getClientIp(req), + details: { backend: name }, + }); + return { cancelled: true }; + } + + /** Start a category migration: copy → flip → delta sweep. One storage job at a time. */ + @Post('migrations') + @HttpCode(200) + migrationStart(@CurrentUser() user: User, @Body() body: StorageMigrationRequestDto, @Req() req: Request): { started: true } { + const { category, to } = body; + try { + this.service.startMigration(category, to); + } catch (err) { + if (err instanceof MigrationRequestError) throw new HttpException({ error: err.message }, 400); + if (err instanceof MigrationTargetError) throw new HttpException({ error: err.message }, 404); + if (err instanceof BackfillBusyError) throw new HttpException({ error: err.message }, 409); + throw err; + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_migration', + ip: getClientIp(req), + details: { category, to }, + }); + return { started: true }; + } + + @Delete('migrations/:category') + migrationCancel(@CurrentUser() user: User, @Param('category') category: string, @Req() req: Request): { cancelled: true } { + if (!this.service.cancelMigration(category)) { + throw new HttpException({ error: `no running migration for '${category}'` }, 404); + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_migration_cancel', + ip: getClientIp(req), + details: { category }, + }); + return { cancelled: true }; + } + + @Post('stats/refresh') + @HttpCode(200) + async statsRefresh(@CurrentUser() user: User, @Req() req: Request): Promise { + let usage: StorageUsage; + try { + usage = await this.service.refreshStats(); + } catch (err) { + if (err instanceof StatsBusyError) throw new HttpException({ error: err.message }, 409); + throw err; + } + this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_stats_refresh', + ip: getClientIp(req), + }); + return usage; + } +} diff --git a/server/src/nest/storage/storage-admin.dto.ts b/server/src/nest/storage/storage-admin.dto.ts new file mode 100644 index 000000000..e52e8b476 --- /dev/null +++ b/server/src/nest/storage/storage-admin.dto.ts @@ -0,0 +1,13 @@ +import { createZodDto } from 'nestjs-zod'; +import { storageConfigPutSchema, storageMigrationRequestSchema, storageTestRequestSchema } from '@trek/shared'; + +/** + * createZodDto wrappers over the @trek/shared storage contracts — the global + * ZodValidationPipe (APP_PIPE) validates @Body() params typed with these, and + * validate-body-contracts.ts refuses boot for any unwrapped mutation body. + * The PUT body is storageConfigPutSchema (config + optimistic-concurrency + * `version`, audit #7) — storageConfigSchema alone still backs the seed file. + */ +export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {} +export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {} +export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {} diff --git a/server/src/nest/storage/storage-admin.service.ts b/server/src/nest/storage/storage-admin.service.ts new file mode 100644 index 000000000..090fac9c6 --- /dev/null +++ b/server/src/nest/storage/storage-admin.service.ts @@ -0,0 +1,192 @@ +import fs from 'node:fs'; +import { Injectable } from '@nestjs/common'; +import type { StorageAdminState, StorageBackend, StorageConfigPut, StorageTestResponse, StorageUsage } from '@trek/shared'; +import { DatabaseService } from '../database/database.service'; +import { + BACKENDS_KEY, + CATEGORIES_KEY, + VERSION_KEY, + StorageRegistryService, +} from './storage-registry.service'; +import { StorageService } from './storage.service'; +import { SEED_CONFIG_PATH } from './storage-paths'; +import { + assertNoMaskSentinels, + decryptBackendSecrets, + encryptStorageSecrets, + maskBackendOptions, + unmaskStorageConfig, +} from './storage-secrets'; +import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe'; +import { StorageBackendError, StorageConflictError, type StorageCategory } from './storage.types'; +import { StorageJobsService } from './storage-jobs.service'; +import { StorageStatsService } from './storage-stats.service'; + +/** + * Owner of the api/admin/storage read/write pipelines (spec: + * docs/superpowers/specs/2026-08-19-storage-admin-config-design.md, Server). + * Reads render from the registry's live snapshot; writes run + * unmask → preview → encrypt → persist → reload, so an admin save either + * fully applies or changes nothing — never the boot-time silent fallback. + */ +@Injectable() +export class StorageAdminService { + constructor( + private readonly db: DatabaseService, + private readonly registry: StorageRegistryService, + private readonly storage: StorageService, + private readonly jobs: StorageJobsService, + private readonly stats: StorageStatsService, + ) {} + + /** The effective world — secrets masked, categories cross-referenced per backend. */ + state(): StorageAdminState { + const snapshot = this.registry.snapshot(); + const assignments = Object.entries(snapshot.categories) as Array< + [keyof typeof snapshot.categories, { backend: string; source: 'default' | 'settings' }] + >; + return { + backends: snapshot.backends.map((backend) => ({ + name: backend.name, + type: backend.type, + source: backend.source, + options: maskBackendOptions(backend.type, backend.options), + categories: assignments.filter(([, a]) => a.backend === backend.name).map(([category]) => category), + })), + categories: snapshot.categories, + health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) }, + seedFilePresent: fs.existsSync(SEED_CONFIG_PATH), + usage: this.stats.readUsage(), + backfills: this.jobs.statuses(), + migrations: this.jobs.migrationStatuses(), + version: this.registry.currentConfigVersion(), + configError: this.registry.lastLoadError(), + }; + } + + /** + * Start a replica catch-up for a routed mirror. Delegates to the one-at-a-time + * job registry; throws BackfillTargetError (404) / BackfillBusyError (409). + */ + startBackfill(mirrorName: string): void { + this.jobs.startBackfill(mirrorName); + } + + /** True when an active run was cancelled; false when there was nothing to cancel. */ + cancelBackfill(mirrorName: string): boolean { + return this.jobs.cancelBackfill(mirrorName); + } + + /** + * Start a category migration to a new backend. Delegates to the one-at-a-time + * job registry; throws MigrationRequestError (400) / MigrationTargetError (404) / + * BackfillBusyError (409). + */ + startMigration(category: StorageCategory, to: string): void { + this.jobs.startMigration(category, to); + } + + /** True when an active migration was cancelled; false when there was nothing to cancel. */ + cancelMigration(category: string): boolean { + return this.jobs.cancelMigration(category); + } + + /** Runs and persists a fresh usage scan. Throws StatsBusyError (409) if one is already running. */ + refreshStats(): Promise { + return this.stats.scan(); + } + + /** + * Full-document replace of the two settings rows. Throws StorageBackendError + * on any refusal, or StorageConflictError (audit #7) when `config.version` + * no longer matches the stored counter — checked FIRST, before unmask/preview, + * so a stale submit never even previews against (let alone overwrites) a + * config that moved on since the form was loaded (e.g. a category + * migration's flip, which bumps the same counter). + */ + applyConfig(config: StorageConfigPut): void { + const currentVersion = this.registry.currentConfigVersion(); + if (config.version !== currentVersion) { + throw new StorageConflictError(currentVersion, config.version); + } + const unmasked = unmaskStorageConfig(config, this.storedBackendsRow()); + // unmask only resolves the secret fields it knows about; a mask sentinel + // submitted in a non-secret field would otherwise pass through untouched + // and get persisted verbatim as garbage-in. + assertNoMaskSentinels(unmasked); + this.registry.preview({ backends: unmasked.backends, categories: unmasked.categories }); + const encrypted = encryptStorageSecrets(unmasked); + this.db.transaction(() => { + const upsert = this.db.prepare( + 'INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', + ); + upsert.run(BACKENDS_KEY, JSON.stringify(encrypted.backends)); + upsert.run(CATEGORIES_KEY, JSON.stringify(encrypted.categories)); + upsert.run(VERSION_KEY, String(currentVersion + 1)); + }); + this.registry.reload(); + // Any running job whose backend the reloaded config no longer has ends + // cancelled rather than running invisibly against a stale driver ref + // (polish item 3) — see StorageJobsService.cancelJobsForMissingBackends. + this.jobs.cancelJobsForMissingBackends(); + } + + /** + * Probe an (unsaved or stored) candidate with ephemeral drivers. Mirrors are + * expanded and probed per target — the probe bypasses MirrorDriver, which + * hides replica failures by design. Registry state is never touched. + */ + async testBackend(candidate: StorageBackend): Promise { + const { backends } = unmaskStorageConfig( + { backends: [candidate], categories: {} }, + this.storedBackendsRow(), + ); + const backend = backends[0]!; + const targets = this.probeTargetsFor(backend).map(decryptBackendSecrets) as Array< + Extract + >; + const results: ProbeTargetResult[] = []; + for (const target of targets) { + results.push(await this.probeTarget(target)); + } + return { ok: results.every((r) => r.ok), targets: results }; + } + + /** Expand a mirror candidate into its concrete targets via the live snapshot. */ + private probeTargetsFor(backend: StorageBackend): StorageBackend[] { + if (backend.type !== 'mirror') return [backend]; + const byName = new Map(this.registry.snapshot().backends.map((b) => [b.name, b])); + const names = [backend.options.primary, ...backend.options.replicas]; + return names.map((name) => { + const resolved = byName.get(name); + if (!resolved) { + throw new StorageBackendError(`mirror '${backend.name}' references unknown backend '${name}'`); + } + if (resolved.type === 'mirror') { + throw new StorageBackendError(`mirror '${backend.name}' nests mirror '${name}' — nesting is rejected`); + } + return { name: resolved.name, type: resolved.type, options: resolved.options } as StorageBackend; + }); + } + + private async probeTarget(target: Extract): Promise { + let driver; + try { + driver = ephemeralDriverFor(target); + } catch (err) { + return { name: target.name, ok: false, error: err instanceof Error ? err.message : String(err) }; + } + return probeDriver(target.name, driver); + } + + /** The raw stored backends row — the unmask source (tolerates absent/garbage rows). */ + private storedBackendsRow(): unknown { + const row = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', BACKENDS_KEY); + if (!row?.value) return []; + try { + return JSON.parse(row.value) as unknown; + } catch { + return []; + } + } +} diff --git a/server/src/nest/storage/storage-events.service.ts b/server/src/nest/storage/storage-events.service.ts new file mode 100644 index 000000000..551cf9aab --- /dev/null +++ b/server/src/nest/storage/storage-events.service.ts @@ -0,0 +1,27 @@ +import { Injectable } from '@nestjs/common'; +import type { ReplicaFailure } from './drivers/mirror.driver'; + +/** + * In-process pub/sub for storage health events. Exists so consumers OUTSIDE + * StorageModule (the notifications notifier) can observe replica failures + * without StorageModule importing NotificationsModule — that would cycle + * (Notifications → Auth → Storage). + */ +@Injectable() +export class StorageEventsService { + private readonly replicaFailureSubscribers: Array<(failure: ReplicaFailure) => void> = []; + + onReplicaFailure(subscriber: (failure: ReplicaFailure) => void): void { + this.replicaFailureSubscribers.push(subscriber); + } + + emitReplicaFailure(failure: ReplicaFailure): void { + for (const subscriber of this.replicaFailureSubscribers) { + try { + subscriber(failure); + } catch { + // A broken subscriber must never break the write path. + } + } + } +} diff --git a/server/src/nest/storage/storage-jobs.service.ts b/server/src/nest/storage/storage-jobs.service.ts new file mode 100644 index 000000000..6554b20ae --- /dev/null +++ b/server/src/nest/storage/storage-jobs.service.ts @@ -0,0 +1,345 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import { Injectable, Logger } from '@nestjs/common'; +import { + STORAGE_CATEGORIES, + type StorageBackfillStatus, + type StorageCategory, + type StorageMigrationStatus, +} from '@trek/shared'; +import { contentTypeFor } from './content-type'; +import { MirrorDriver } from './drivers/mirror.driver'; +import { GLOBAL_TEMP_DIR } from './storage-paths'; +import { StorageRegistryService } from './storage-registry.service'; +import type { StorageDriver } from './storage.types'; + +export class BackfillTargetError extends Error {} +export class BackfillBusyError extends Error {} +export class MigrationRequestError extends Error {} +export class MigrationTargetError extends Error {} + +interface ActiveJob { + status: StorageBackfillStatus; + cancelled: boolean; +} + +interface ActiveMigration { + status: StorageMigrationStatus; + cancelled: boolean; +} + +/** + * One-at-a-time backfill registry (the ImportJobsService idiom, minus WS). + * Finished statuses linger for ttlMs so a remounted panel can show the + * outcome; the client polls the admin state while anything is running. + */ +@Injectable() +export class StorageJobsService { + private readonly logger = new Logger(StorageJobsService.name); + private readonly jobs = new Map(); + private readonly migrations = new Map(); + private ttlMs = 10 * 60_000; + + /** One storage job at a time, across both backfills and category migrations. */ + private anyJobRunning(): boolean { + return ( + [...this.jobs.values()].some((j) => j.status.status === 'running') || + [...this.migrations.values()].some((m) => m.status.status === 'running') + ); + } + + constructor(private readonly registry: StorageRegistryService) {} + + /** Test-only factory for a short TTL, since a second constructor param would confuse Nest DI. */ + static withTtl(registry: StorageRegistryService, ttlMs: number): StorageJobsService { + const svc = new StorageJobsService(registry); + svc.ttlMs = ttlMs; + return svc; + } + + startBackfill(mirrorName: string): void { + // Resolve/validate the target first so an unknown/non-mirror name 404s + // even while a sync is running — the busy check only applies once we + // know there's a real mirror to be busy about. + const snapshot = this.registry.snapshot(); + const categories = STORAGE_CATEGORIES.filter( + (category) => snapshot.categories[category]?.backend === mirrorName, + ); + if (categories.length === 0) { + throw new BackfillTargetError(`'${mirrorName}' is not a mirror routed by any category`); + } + const resolved = categories.map((category) => this.registry.resolve(category)); + const driver = resolved[0]!.driver; + if (!(driver instanceof MirrorDriver)) { + throw new BackfillTargetError(`'${mirrorName}' is not a mirror backend`); + } + if (this.anyJobRunning()) { + throw new BackfillBusyError('a sync is already running — one backfill at a time'); + } + const prefixes = resolved.map((r) => r.keyPrefix); + + const job: ActiveJob = { + cancelled: false, + status: { + backend: mirrorName, + status: 'running', + done: 0, + total: 0, + copied: 0, + skipped: 0, + failed: 0, + deleted: 0, + startedAt: Date.now(), + }, + }; + this.jobs.set(mirrorName, job); + + // Detached: the driver instances are resolved above, so a registry + // reload() mid-run keeps this job on them (the in-flight guarantee). + void driver + .backfill(prefixes, { + onProgress: (progress) => { + job.status = { ...job.status, ...progress }; + }, + isCancelled: () => job.cancelled, + }) + .then((result) => { + job.status = { + ...job.status, + ...result, + status: result.cancelled ? 'cancelled' : 'done', + finishedAt: Date.now(), + }; + }) + .catch((err: unknown) => { + job.status = { + ...job.status, + status: 'error', + error: err instanceof Error ? err.message : String(err), + finishedAt: Date.now(), + }; + this.logger.error(`backfill '${mirrorName}' aborted: ${job.status.error}`); + }) + .finally(() => { + setTimeout(() => { + if (this.jobs.get(mirrorName) === job && job.status.status !== 'running') this.jobs.delete(mirrorName); + }, this.ttlMs).unref?.(); + }); + } + + cancelBackfill(mirrorName: string): boolean { + const job = this.jobs.get(mirrorName); + if (!job || job.status.status !== 'running') return false; + job.cancelled = true; + return true; + } + + statuses(): StorageBackfillStatus[] { + return [...this.jobs.values()].map((job) => ({ ...job.status })); + } + + /** + * Start a category migration: copy every object from the category's + * current backend to `to`, flip the category assignment once the copy + * phase is clean, then sweep any delta the copy phase raced against. + * Throws MigrationRequestError (400) / MigrationTargetError (404) / + * BackfillBusyError (409, shared with backfills — one storage job at a time). + */ + startMigration(category: StorageCategory, to: string): void { + if (!STORAGE_CATEGORIES.includes(category)) { + throw new MigrationRequestError(`'${category}' is not a configurable category`); + } + const current = this.registry.resolve(category); + if (current.backendName === to) { + throw new MigrationRequestError(`'${category}' is already on '${to}'`); + } + const target = this.registry.driverByName(to); + if (!target) throw new MigrationTargetError(`no backend named '${to}'`); + if (this.anyJobRunning()) { + throw new BackfillBusyError('a sync is already running — one storage job at a time'); + } + // The DESTINATION prefix, computed for `to` before the flip — may differ + // from `current.keyPrefix` (audit #8: a mode-A '' source migrating onto a + // prefixed backend must land its objects under that backend's real + // prefix, or the registry can never resolve them again post-flip). + const destPrefix = this.registry.keyPrefixFor(category, to); + + const job: ActiveMigration = { + cancelled: false, + status: { + category, + from: current.backendName, + to, + status: 'running', + done: 0, + total: 0, + copied: 0, + skipped: 0, + failed: 0, + startedAt: Date.now(), + }, + }; + this.migrations.set(category, job); + // Detached, driver instances resolved above (in-flight guarantee, same as backfill). + void this.runMigration(job, current.driver, target, current.keyPrefix, destPrefix) + .catch((err: unknown) => { + job.status = { + ...job.status, + status: 'failed', + error: err instanceof Error ? err.message : String(err), + finishedAt: Date.now(), + }; + this.logger.error(`migration '${category}' aborted: ${job.status.error}`); + }) + .finally(() => { + setTimeout(() => { + if (this.migrations.get(category) === job && job.status.status !== 'running') { + this.migrations.delete(category); + } + }, this.ttlMs).unref?.(); + }); + } + + cancelMigration(category: string): boolean { + const job = this.migrations.get(category); + if (!job || job.status.status !== 'running') return false; + job.cancelled = true; + return true; + } + + migrationStatuses(): StorageMigrationStatus[] { + return [...this.migrations.values()].map((m) => ({ ...m.status })); + } + + /** + * After a config apply: any running job whose backend no longer exists (or, + * for a migration, whose from OR to backend is gone) ends 'cancelled' + * instead of running invisibly against stale driver refs (spec, polish item + * 3) — the resolved driver instances stay valid (the in-flight guarantee), + * only their backend name has fallen out of the config. Also cancels a + * running migration whose category route no longer matches its `from` — + * a save can re-route the category to a THIRD, still-defined backend + * without either of the migration's own backends disappearing, and letting + * the migration run to completion would flip the category back onto a + * backend the operator just moved it away from. A cancelled migration + * pre-flip is harmless (MIG-003's guarantee); post-flip the flag is never + * read again because the sweep phase always runs to completion. + */ + cancelJobsForMissingBackends(): void { + const names = new Set(this.registry.snapshot().backends.map((b) => b.name)); + for (const job of this.jobs.values()) { + if (job.status.status !== 'running') continue; + if (!names.has(job.status.backend)) job.cancelled = true; + } + for (const migration of this.migrations.values()) { + if (migration.status.status !== 'running') continue; + if (!names.has(migration.status.from) || !names.has(migration.status.to)) { + migration.cancelled = true; + continue; + } + if (this.registry.resolve(migration.status.category).backendName !== migration.status.from) { + migration.cancelled = true; + } + } + } + + /** Copy one needy object source→target via a spool file; returns false on a counted failure. */ + private async copyObject( + source: StorageDriver, + target: StorageDriver, + sourceKey: string, + destKey: string, + ): Promise { + const file = path.join(GLOBAL_TEMP_DIR, randomUUID()); + try { + const { stream } = await source.getStream(sourceKey); + await pipeline(stream, fs.createWriteStream(file)); // source errors RETHROW (abort → failed) + } catch (err) { + await fs.promises.rm(file, { force: true }); + throw err; + } + try { + await target.put(destKey, fs.createReadStream(file), { contentType: contentTypeFor(destKey) }); + return true; + } catch { + return false; // per-object failure: counted, blocks the flip + } finally { + await fs.promises.rm(file, { force: true }); + } + } + + private async runMigration( + job: ActiveMigration, + source: StorageDriver, + target: StorageDriver, + sourcePrefix: string, + destPrefix: string, + ): Promise { + const s = () => job.status; + // Rewrite a source key onto the destination backend's own prefix (audit + // #8): equal prefixes make this the identity function — byte-identical + // to pre-fix behavior — but a mode-A ('' prefix) source landing on a + // prefixed destination (or vice versa) now lands at a key the registry + // will actually resolve post-flip, instead of silently keeping the + // source's prefix verbatim. + const destKeyFor = (sourceKey: string): string => destPrefix + sourceKey.slice(sourcePrefix.length); + // Phase 1a: honest total. + for await (const _stat of source.list(sourcePrefix)) job.status = { ...s(), total: s().total + 1 }; + // Phase 1b: copy. + for await (const stat of source.list(sourcePrefix)) { + if (job.cancelled) { + job.status = { ...s(), status: 'cancelled', finishedAt: Date.now() }; + return; + } + const done = s().done + 1; + job.status = { ...s(), done, total: Math.max(s().total, done) }; + const destKey = destKeyFor(stat.key); + const existing = await target.stat(destKey); + if (existing && existing.size === stat.size) { + job.status = { ...s(), skipped: s().skipped + 1 }; + continue; + } + if (await this.copyObject(source, target, stat.key, destKey)) job.status = { ...s(), copied: s().copied + 1 }; + else job.status = { ...s(), failed: s().failed + 1 }; + } + if (s().failed > 0) { + job.status = { + ...s(), + status: 'failed', + error: `${s().failed} object(s) failed to copy — category not flipped`, + finishedAt: Date.now(), + }; + return; + } + // Close the last-object cancel window: the per-iteration check above only + // fires at the TOP of each copy iteration, so a cancel landing after the + // last object's await (or an empty category, which never enters the loop + // at all) would otherwise reach the flip unchecked. Everything from here + // to assignCategory() below is one synchronous stretch, so this is the + // last point a cancel can still be honored before the flip. + if (job.cancelled) { + job.status = { ...s(), status: 'cancelled', finishedAt: Date.now() }; + return; + } + // Phase 2: flip — the job, not the save, owns this. + this.registry.assignCategory(s().category, s().to); + // Phase 3: delta sweep + reclaimable tally. Cancellation is ignored here + // (bounded). Reclaimable stays SOURCE-keyed — it counts what's left to + // reclaim on the old backend, not where it landed on the new one. + const reclaimable = { objects: 0, bytes: 0 }; + for await (const stat of source.list(sourcePrefix)) { + reclaimable.objects += 1; + reclaimable.bytes += stat.size; + const destKey = destKeyFor(stat.key); + const existing = await target.stat(destKey); + if (existing && existing.size === stat.size) continue; + const done = s().done + 1; + job.status = { ...s(), done, total: Math.max(s().total, done) }; + if (await this.copyObject(source, target, stat.key, destKey)) job.status = { ...s(), copied: s().copied + 1 }; + else job.status = { ...s(), failed: s().failed + 1 }; // sweep failure: reported, flip already happened + } + job.status = { ...s(), status: 'done', total: s().done, reclaimable, finishedAt: Date.now() }; + } +} diff --git a/server/src/nest/storage/storage-keys.ts b/server/src/nest/storage/storage-keys.ts new file mode 100644 index 000000000..32bb7a5ce --- /dev/null +++ b/server/src/nest/storage/storage-keys.ts @@ -0,0 +1,48 @@ +import { StorageInvalidKeyError } from './storage.types'; + +/** + * Central key validation — the ONE place storage keys are checked, replacing + * the scattered per-route guards (`path.basename` + `startsWith` shapes in + * files.service.ts / platform.routes.ts / journey-public.controller.ts) as + * call sites migrate. Drivers still confine resolved paths to their root as + * defense in depth, but no driver or route re-implements these rules. + * + * Rejected: `..` segments, absolute paths, backslashes, empty segments, + * leading-dot segments (matches `express.static`'s default + * `dotfiles: 'ignore'` and keeps the `.tmp` spool unreachable), and control + * characters. Spaces are valid — legacy `photos/` keys are + * user-named flat files. + */ + +const MAX_KEY_LENGTH = 1024; + +function hasControlChars(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code < 0x20 || code === 0x7f) return true; + } + return false; +} + +export function isValidKey(key: string): boolean { + if (!key || key.length > MAX_KEY_LENGTH) return false; + if (key.includes('\\') || hasControlChars(key)) return false; + for (const segment of key.split('/')) { + if (segment === '' || segment.startsWith('.')) return false; + } + return true; +} + +export function assertValidKey(key: string): void { + if (!isValidKey(key)) throw new StorageInvalidKeyError(key); +} + +/** Prefixes additionally allow `''` (list everything) and a trailing `/`. */ +export function isValidPrefix(prefix: string): boolean { + if (prefix === '') return true; + return isValidKey(prefix.endsWith('/') ? prefix.slice(0, -1) : prefix); +} + +export function assertValidPrefix(prefix: string): void { + if (!isValidPrefix(prefix)) throw new StorageInvalidKeyError(prefix); +} diff --git a/server/src/nest/storage/storage-paths.ts b/server/src/nest/storage/storage-paths.ts new file mode 100644 index 000000000..e5e06c596 --- /dev/null +++ b/server/src/nest/storage/storage-paths.ts @@ -0,0 +1,22 @@ +import path from 'node:path'; + +/** + * The storage domain's one `__dirname` anchor (the uploads-root.ts idiom, and + * eventually its replacement: the registry's defaults make this the single + * source of truth the other 16 copies collapse onto in later slices). + * + * `__dirname` moves with the file — `server/{src,dist}/nest/storage` sits + * three levels under `server/`, so three `..` hops resolve to `/` + * under both the src (vitest) and dist (runtime) layouts. The depth is stated + * rather than counted by hand and pinned by storage-keys.test.ts, exactly + * like uploads-root.test.ts pins UPLOADS_ROOT. In Docker both anchors are + * symlinks (`/app/server/uploads → /app/uploads`, `/app/server/data → + * /app/data`), which is why LocalDriver realpaths its root at init. + */ +export const DEFAULT_UPLOADS_ROOT = path.resolve(__dirname, '..', '..', '..', 'uploads'); +export const DATA_ROOT = path.resolve(__dirname, '..', '..', '..', 'data'); +export const DEFAULT_BACKUPS_ROOT = path.join(DATA_ROOT, 'backups'); +/** Driver-agnostic global scratch space (`data/tmp`) — see StorageService.tempDir(). */ +export const GLOBAL_TEMP_DIR = path.join(DATA_ROOT, 'tmp'); +/** Seed-once boot provisioning file — imported only when no storage.* row exists. */ +export const SEED_CONFIG_PATH = path.join(DATA_ROOT, 'storage-config.json'); diff --git a/server/src/nest/storage/storage-probe.ts b/server/src/nest/storage/storage-probe.ts new file mode 100644 index 000000000..d1927a6b9 --- /dev/null +++ b/server/src/nest/storage/storage-probe.ts @@ -0,0 +1,47 @@ +import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; +import type { StorageBackend } from '@trek/shared'; +import { LocalDriver } from './drivers/local.driver'; +import { S3Driver } from './drivers/s3.driver'; +import { StorageBackendError, describeError, type StorageDriver } from './storage.types'; + +/** + * Test-connection probes (spec: POST /api/admin/storage/test). Drivers are + * EPHEMERAL — constructed per probe, never registered, never touching + * registry state. A mirror is probed per target (primary and each replica + * individually) because MirrorDriver hides replica failures by design. + * + * Residue: a local probe may leave an empty `trek-probe/` directory in the + * target root. It is inert — not a category prefix, invisible to lists + * (objects only), and outside every retention pattern. + */ + +export interface ProbeTargetResult { + name: string; + ok: boolean; + error?: string; +} + +/** put → stat → delete of a unique probe key; any throw becomes the target's error. */ +export async function probeDriver(name: string, driver: StorageDriver): Promise { + const key = `trek-probe/${randomUUID()}`; + try { + await driver.put(key, Readable.from('trek storage probe')); + const stat = await driver.stat(key); + if (!stat) throw new StorageBackendError('probe object vanished between put and stat'); + await driver.delete(key); + return { name, ok: true }; + } catch (err) { + return { name, ok: false, error: describeError(err) }; + } +} + +/** A non-mirror backend definition (secrets already decrypted) → throwaway driver. */ +export function ephemeralDriverFor(backend: Extract): StorageDriver { + if (backend.type === 'local') { + const driver = new LocalDriver({ id: `probe-${backend.name}`, root: backend.options.root }); + driver.init({ ensurePrefixes: [], cleanSpool: false }); + return driver; + } + return new S3Driver({ id: `probe-${backend.name}`, ...backend.options }); +} diff --git a/server/src/nest/storage/storage-registry.service.ts b/server/src/nest/storage/storage-registry.service.ts new file mode 100644 index 000000000..11f8f5384 --- /dev/null +++ b/server/src/nest/storage/storage-registry.service.ts @@ -0,0 +1,673 @@ +import fs from 'node:fs'; +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { STORAGE_BACKEND_TYPES, storageConfigSchema } from '@trek/shared'; +import { DatabaseService } from '../database/database.service'; +import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; +import { LocalDriver } from './drivers/local.driver'; +import { MirrorDriver, type ReplicaFailure } from './drivers/mirror.driver'; +import { S3Driver } from './drivers/s3.driver'; +import { StorageEventsService } from './storage-events.service'; +import { DEFAULT_BACKUPS_ROOT, DEFAULT_UPLOADS_ROOT, GLOBAL_TEMP_DIR, SEED_CONFIG_PATH } from './storage-paths'; +import { assertNoMaskSentinels, encryptStorageSecrets } from './storage-secrets'; +import { + SERVED_CATEGORIES, + STORAGE_CATEGORIES, + StorageBackendError, + type ServedCategory, + type StorageCategory, + type StorageDriver, +} from './storage.types'; + +export interface ResolvedCategory { + driver: StorageDriver; + keyPrefix: string; + backendName: string; +} + +export type BackendSource = 'built-in' | 'env' | 'settings'; +export interface BackendSnapshot { + name: string; + type: 'local' | 's3' | 'mirror'; + source: BackendSource; + /** Stored options — secret fields still encrypted; masking is the admin layer's job. */ + options: Record; +} +export interface RegistrySnapshot { + backends: BackendSnapshot[]; + categories: Record; +} + +interface LocalBackendConfig { + name: string; + type: 'local'; + options: { root: string }; +} +interface MirrorBackendConfig { + name: string; + type: 'mirror'; + options: { primary: string; replicas: string[] }; +} +interface S3BackendConfig { + name: string; + type: 's3'; + options: { + endpoint: string; + region: string; + bucket: string; + keyPrefix: string; + accessKeyId: string; + secretAccessKey: string; + retries: number; + timeoutMs: number; + }; +} +type BackendConfig = LocalBackendConfig | MirrorBackendConfig | S3BackendConfig; + +interface RegistryState { + drivers: Map; + categories: Map; + snapshot: RegistrySnapshot; +} + +export const BACKENDS_KEY = 'storage.backends'; +export const CATEGORIES_KEY = 'storage.categories'; +/** + * Optimistic-concurrency counter (audit #7): bumped inside the SAME + * transaction by every writer of the two settings rows above — applyConfig + * and the migration flip (assignCategory) — so a stale admin PUT can never + * silently undo a flip that landed after the PUT's form was loaded. + */ +export const VERSION_KEY = 'storage.config_version'; +const REPLICA_FAILURE_RING_SIZE = 50; + +/** + * Category prefixes mirror the current uploads layout 1:1 so local keys map + * to existing paths and no data migration is required. `backups` is bare-key + * (the backend root IS the backups dir — spec rev 3.1), and `photos-google` + * flips to bare keys when it resolves to `place-photos-local` (the relocated + * TREK_PLACE_PHOTO_DIR layout, place-photo-cache.service.ts). + * + * Exported for tests/unit/uploads-dirs.test.ts, which pins the Dockerfile's + * `mkdir -p /app/uploads/...` list to these prefixes. + */ +export const CATEGORY_PREFIXES: Record = { + files: 'files/', + journey: 'journey/', + covers: 'covers/', + avatars: 'avatars/', + places: 'places/', + photos: 'photos/', + 'photos-google': 'photos/google/', + 'photos-trek': 'photos/trek/', + backups: '', +}; + +/** + * Named backend instances + the category→backend map, config-driven and + * swappable — v1 groundwork for the future admin UI (which becomes a settings + * editor plus a reload() call). + * + * Unlike the codebase default of uncached per-request reads + * (allowed-file-types.service.ts documents that convention), the registry + * holds its validated config in memory and swaps it on reload() — the + * permissions-cache precedent. Rationale: this sits on every byte-serving hot + * path and its config changes only via admin action. Nothing outside this + * class may cache a driver reference; resolution happens per call. + */ +@Injectable() +export class StorageRegistryService implements OnModuleInit { + private readonly logger = new Logger(StorageRegistryService.name); + private state: RegistryState | null = null; + private failures: ReplicaFailure[] = []; + /** + * Set by load() whenever the stored settings fail to parse/validate — the + * message a failing load logs, kept around so the admin state can surface + * it (audit minor: silently falling back masked a broken stored config, so + * a later save looked like a no-op edit but actually replaced it). Cleared + * on the next successful load/reload. + */ + private loadFailure: string | null = null; + + constructor( + private readonly db: DatabaseService, + private readonly env: RuntimeEnvService, + private readonly events: StorageEventsService, + ) {} + + onModuleInit(): void { + this.seedFromFileOnce(); + this.load(true); + } + + /** Re-read settings, validate, atomically swap. In-flight ops keep their resolved instances. */ + reload(): void { + this.load(false); + } + + /** + * Validate a candidate config by running the real build() — merge over + * built-ins → parse → validateConfig → driver construction (network-free) — + * and discard the result. Throws exactly what a failing load would log; + * never touches this.state (admin writes must not take the silent boot-time + * fallback). Shares one side effect with any successful save: local roots + * and prefix dirs are created — an uncreatable root is exactly the error to + * surface before persisting. cleanSpool stays boot-only (boot=false here). + */ + preview(candidate: { backends: unknown; categories: unknown }): void { + void this.build(candidate, false); + } + + resolve(category: ServedCategory): ResolvedCategory { + if (!this.state) throw new StorageBackendError('storage registry not initialized'); + const assignment = this.state.categories.get(category); + if (!assignment) throw new StorageBackendError(`unknown storage category: ${category}`); + const driver = this.state.drivers.get(assignment.backendName); + if (!driver) { + throw new StorageBackendError(`storage backend '${assignment.backendName}' missing for category '${category}'`); + } + return { driver, keyPrefix: assignment.keyPrefix, backendName: assignment.backendName }; + } + + /** The effective world with provenance — the admin GET renders from this. */ + snapshot(): RegistrySnapshot { + if (!this.state) throw new StorageBackendError('storage registry not initialized'); + return this.state.snapshot; + } + + /** Driver-agnostic global scratch space (data/tmp). */ + tempDir(): string { + return GLOBAL_TEMP_DIR; + } + + /** + * The key-prefix a category resolves to ON A GIVEN BACKEND — the same rule + * build() applies when assigning categories to the CURRENT config, exposed + * standalone so a caller can ask "what prefix would this category use on + * backend X" without X being the category's current assignment. The + * category migration job is exactly that caller: it must compute the + * DESTINATION prefix (on `to`, before the flip) while `resolve()` can only + * ever answer for the backend a category is assigned to right now. + */ + keyPrefixFor(category: ServedCategory, backendName: string): string { + return prefixFor(category, backendName); + } + + /** Driver instance for a defined backend, assigned or not; null when unknown. */ + driverByName(name: string): StorageDriver | null { + if (!this.state) return null; + return this.state.drivers.get(name) ?? null; + } + + /** Current optimistic-concurrency counter — 0 when never bumped (fresh install). */ + currentConfigVersion(): number { + return readConfigVersion(this.db); + } + + /** Non-null when the last load() fell back (last-good config or built-in defaults) — null once a load succeeds. */ + lastLoadError(): string | null { + return this.loadFailure; + } + + /** + * Persist one category assignment and reload — the migration flip's write + * path. Same transactional row write applyConfig uses, scoped to one key; + * the current stored map is read through the existing parseCategoryMap + * validator rather than re-parsing the raw row by hand. Bumps the shared + * version counter in the SAME transaction (audit #7) — a concurrent admin + * PUT built against the pre-flip version now conflicts instead of silently + * overwriting this assignment. + */ + assignCategory(category: StorageCategory, backend: string): void { + // Belt-and-braces alongside the migration job's own pre-flip cancel guard: + // never persist a category pointing at a backend that doesn't exist in + // the current snapshot (e.g. a config save removed it mid-migration). + const defined = this.state?.snapshot.backends ?? []; + const names = new Set(defined.map((b) => b.name)); + if (!names.has(backend)) { + throw new StorageBackendError(`cannot assign '${category}' to unknown backend '${backend}'`); + } + // The OTHER writer of the two settings rows, so it enforces the same + // shared-replica rule validateConfig does (see assertNoSharedReplicas): + // a mirror's sync sweep DELETES replica objects its primary doesn't hold, + // so routing a category onto a backend that is somebody's replica would + // hand this category's objects to that sweep. + const target = defined.find((b) => b.name === backend); + const owner = target?.type === 'mirror' ? String(target.options.primary) : backend; + const holder = defined.find((b) => b.type === 'mirror' && replicaNamesOf(b.options).includes(owner)); + if (holder) { + throw new StorageBackendError( + `cannot assign '${category}' to '${backend}' — '${owner}' is a mirror replica of '${holder.name}', and that mirror's sync sweep would delete the category's objects`, + ); + } + const stored = new Map(parseCategoryMap(this.readSettings().categories)); + stored.set(category, backend); + const next = Object.fromEntries(stored); + this.db.transaction(() => { + const upsert = this.db.prepare( + 'INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', + ); + upsert.run(CATEGORIES_KEY, JSON.stringify(next)); + upsert.run(VERSION_KEY, String(readConfigVersion(this.db) + 1)); + }); + this.reload(); + } + + recordReplicaFailure(failure: ReplicaFailure): void { + this.failures.push(failure); + if (this.failures.length > REPLICA_FAILURE_RING_SIZE) { + this.failures = this.failures.slice(-REPLICA_FAILURE_RING_SIZE); + } + this.events.emitReplicaFailure(failure); + } + + replicaFailures(): readonly ReplicaFailure[] { + return this.failures; + } + + /** + * Seed-once boot provisioning (spec: 2026-08-19-storage-admin-config-design.md). + * Imported only when NO storage.* row exists; runs the same validation the + * PUT pipeline runs (encryption gate, then preview); secrets are encrypted + * on import; the file is ignored (loudly) afterward. Every failure aborts + * boot with the exact error — an actively-provisioning operator must see + * it, so this deliberately runs OUTSIDE load()'s last-good safety net. + * Recovery: stop the server, DELETE FROM app_settings WHERE key LIKE + * 'storage.%', restart (documented in the README with slice 3). + */ + private seedFromFileOnce(): void { + const rowCount = this.db.get<{ n: number }>( + 'SELECT COUNT(*) AS n FROM app_settings WHERE key IN (?, ?)', + BACKENDS_KEY, + CATEGORIES_KEY, + ); + const filePresent = fs.existsSync(SEED_CONFIG_PATH); + if ((rowCount?.n ?? 0) > 0) { + if (filePresent) { + this.logger.log(`storage config rows exist — ignoring ${SEED_CONFIG_PATH}; manage storage in the admin UI`); + } + return; + } + if (!filePresent) return; + + const fail = (detail: string): never => { + throw new Error(`invalid storage seed file ${SEED_CONFIG_PATH}: ${detail}`); + }; + let json: unknown; + try { + json = JSON.parse(fs.readFileSync(SEED_CONFIG_PATH, 'utf8')); + } catch (err) { + return fail(err instanceof Error ? err.message : String(err)); + } + const parsed = storageConfigSchema.safeParse(json); + if (!parsed.success) { + return fail( + parsed.error.issues.map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`).join('; '), + ); + } + const config = parsed.data; + try { + assertNoMaskSentinels(config); + this.preview({ backends: config.backends, categories: config.categories }); + } catch (err) { + return fail(err instanceof Error ? err.message : String(err)); + } + const encrypted = encryptStorageSecrets(config); + this.db.transaction(() => { + const upsert = this.db.prepare( + 'INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', + ); + upsert.run(BACKENDS_KEY, JSON.stringify(encrypted.backends)); + upsert.run(CATEGORIES_KEY, JSON.stringify(encrypted.categories)); + }); + this.logger.log( + `storage config seeded from ${SEED_CONFIG_PATH} — the file is now ignored; manage storage in the admin UI`, + ); + } + + /** + * Invalid settings never take the server down: on any failure the previous + * state is kept (at boot, when there is no previous state, the built-in + * defaults — which cannot be misconfigured — are loaded instead). + */ + private load(boot: boolean): void { + try { + this.state = this.build(this.readSettings(), boot); + this.loadFailure = null; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const keeping = this.state ? 'last-good config' : 'built-in defaults'; + this.logger.error(`invalid storage settings — keeping ${keeping}: ${message}`); + this.loadFailure = message; + if (!this.state) { + this.state = this.build({ backends: [], categories: {} }, boot); + } + } + } + + private readSettings(): { backends: unknown; categories: unknown } { + const read = (key: string): unknown => { + const row = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', key); + if (!row?.value) return undefined; + try { + return JSON.parse(row.value) as unknown; + } catch (err) { + // JSON.parse's own SyntaxError can echo a snippet of the raw input + // around the failure position (Node 24 V8, e.g. `Unexpected token + // 'u', ..."cessKey": undefined_"... is not valid JSON`) — for a + // hand-edited/corrupted row that snippet can contain a fragment of a + // real secret, and load()'s catch stores err.message verbatim as + // configError, which the admin panel renders. Substitute a generic + // message; keep only the numeric position when the engine reports + // one the old ("at position N") way — never any quoted content. + const position = err instanceof Error ? /at position (\d+)/.exec(err.message)?.[1] : undefined; + throw new StorageBackendError( + `'${key}' contains malformed JSON` + (position ? ` (at position ${position})` : ''), + ); + } + }; + return { backends: read(BACKENDS_KEY), categories: read(CATEGORIES_KEY) }; + } + + private build(settings: { backends: unknown; categories: unknown }, boot: boolean): RegistryState { + // 1. Env is read fresh on every load (never snapshotted — RuntimeEnvService rule). + const placePhotoDir = this.env.env().paths.placePhotoDir; + + // 2. Built-in defaults; settings entries with the same name/category override. + // uploads-local's root is the computed default; relocation is a settings + // override row bearing the built-in's name. + const backends = new Map(); + backends.set('uploads-local', { name: 'uploads-local', type: 'local', options: { root: DEFAULT_UPLOADS_ROOT } }); + backends.set('backups-local', { name: 'backups-local', type: 'local', options: { root: DEFAULT_BACKUPS_ROOT } }); + const backendSources = new Map([ + ['uploads-local', 'built-in'], + ['backups-local', 'built-in'], + ]); + if (placePhotoDir) { + backends.set('place-photos-local', { name: 'place-photos-local', type: 'local', options: { root: placePhotoDir } }); + backendSources.set('place-photos-local', 'env'); + } + for (const config of parseBackendList(settings.backends)) { + backends.set(config.name, config); + backendSources.set(config.name, 'settings'); + } + + const categoryBackends = new Map(); + for (const category of SERVED_CATEGORIES) categoryBackends.set(category, 'uploads-local'); + categoryBackends.set('backups', 'backups-local'); + if (placePhotoDir) categoryBackends.set('photos-google', 'place-photos-local'); + const categorySources = new Map(); + for (const [category, backendName] of parseCategoryMap(settings.categories)) { + categoryBackends.set(category, backendName); + categorySources.set(category, 'settings'); + } + + // 3. Validate the merged config as a whole. + validateConfig(backends, categoryBackends); + + // 4. Category prefixes (photos-google mode decided from the final map). + const categories = new Map(); + for (const [category, backendName] of categoryBackends) { + categories.set(category, { backendName, keyPrefix: this.keyPrefixFor(category, backendName) }); + } + + // 5. Instantiate drivers: locals first (each ensures its own dirs; spool + // cleanup at boot only — a reload() could delete an in-flight upload's + // spool file), then mirrors over the local instances. + const drivers = new Map(); + for (const config of backends.values()) { + if (config.type !== 'local') continue; + const driver = new LocalDriver({ id: config.name, root: config.options.root }); + const ensurePrefixes = [...categories.entries()] + .filter(([, assignment]) => assignment.backendName === config.name) + .map(([, assignment]) => assignment.keyPrefix) + .filter((prefix) => prefix !== ''); + driver.init({ ensurePrefixes, cleanSpool: boot }); + drivers.set(config.name, driver); + } + for (const config of backends.values()) { + if (config.type !== 's3') continue; + drivers.set( + config.name, + new S3Driver({ id: config.name, ...config.options, secretAccessKey: decryptedSecret(config) }), + ); + } + for (const config of backends.values()) { + if (config.type !== 'mirror') continue; + drivers.set( + config.name, + new MirrorDriver({ + id: config.name, + primary: drivers.get(config.options.primary)!, + replicas: config.options.replicas.map((name) => drivers.get(name)!), + tempDir: () => this.tempDir(), + onReplicaFailure: (failure) => this.recordReplicaFailure(failure), + }), + ); + } + fs.mkdirSync(GLOBAL_TEMP_DIR, { recursive: true }); + + // 6. The effective world with provenance — assembled from the same merged + // maps, so it always matches what drivers/categories actually resolve to. + const snapshot: RegistrySnapshot = { + backends: [...backends.values()].map((config) => ({ + name: config.name, + type: config.type, + source: backendSources.get(config.name) ?? 'settings', + options: { ...config.options }, + })), + categories: Object.fromEntries( + [...categoryBackends.entries()] + // photos is served-legacy, never configurable — the admin world + // exposes only the 8 configurable categories. + .filter(([category]) => (STORAGE_CATEGORIES as readonly string[]).includes(category)) + .map(([category, backendName]) => [ + category, + { backend: backendName, source: categorySources.get(category) ?? 'default' }, + ]), + ) as RegistrySnapshot['categories'], + }; + + // 7. Single-assignment swap — callers mid-operation keep their instances. + return { drivers, categories, snapshot }; + } +} + +// ── settings parsing / validation (pure helpers) ────────────────────────────── + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Raw read of the version counter row — 0 for an absent/garbage row (fresh install, or a hand-edited DB). */ +function readConfigVersion(db: DatabaseService): number { + const row = db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', VERSION_KEY); + const parsed = row?.value ? Number.parseInt(row.value, 10) : 0; + return Number.isFinite(parsed) ? parsed : 0; +} + +function parseBackendList(raw: unknown): BackendConfig[] { + if (raw === undefined) return []; + if (!Array.isArray(raw)) throw new StorageBackendError(`'${BACKENDS_KEY}' must be a JSON array`); + return raw.map((entry): BackendConfig => { + if (!isRecord(entry) || typeof entry.name !== 'string' || !entry.name) { + throw new StorageBackendError(`'${BACKENDS_KEY}' entries need a non-empty string 'name'`); + } + const options = isRecord(entry.options) ? entry.options : {}; + if (entry.type === 'local') { + if (typeof options.root !== 'string' || !options.root) { + throw new StorageBackendError(`local backend '${entry.name}' needs a non-empty 'options.root'`); + } + return { name: entry.name, type: 'local', options: { root: options.root } }; + } + if (entry.type === 'mirror') { + const replicas = Array.isArray(options.replicas) ? options.replicas : null; + if (typeof options.primary !== 'string' || !replicas || replicas.some((r) => typeof r !== 'string')) { + throw new StorageBackendError(`mirror backend '${entry.name}' needs 'options.primary' and 'options.replicas'`); + } + return { name: entry.name, type: 'mirror', options: { primary: options.primary, replicas: replicas as string[] } }; + } + if (entry.type === 's3') { + const parsed = STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse(options); + if (!parsed.success) { + const first = parsed.error.issues[0]; + throw new StorageBackendError( + `s3 backend '${entry.name}' has invalid options` + + (first ? ` — ${first.path.join('.') || '(options)'}: ${first.message}` : ''), + ); + } + // secretAccessKey stays as stored (usually enc:v1:) — decrypted only at + // driver construction, so config maps and snapshots never hold plaintext. + return { name: entry.name, type: 's3', options: parsed.data }; + } + throw new StorageBackendError(`backend '${entry.name}' has unknown type '${String(entry.type)}'`); + }); +} + +function parseCategoryMap(raw: unknown): Array<[StorageCategory, string]> { + if (raw === undefined) return []; + if (!isRecord(raw)) throw new StorageBackendError(`'${CATEGORIES_KEY}' must be a JSON object`); + return Object.entries(raw).map(([category, backendName]) => { + if (!(STORAGE_CATEGORIES as readonly string[]).includes(category)) { + throw new StorageBackendError(`'${CATEGORIES_KEY}' names unknown category '${category}'`); + } + if (typeof backendName !== 'string' || !backendName) { + throw new StorageBackendError(`category '${category}' must map to a backend name`); + } + return [category as StorageCategory, backendName]; + }); +} + +/** + * Secrets live encrypted inside the storage.backends JSON (admin-config spec); + * plaintext passthrough is tolerated as belt-and-braces (decrypt_api_key's + * legacy-plaintext rule), though seed and PUT always store encrypted. + */ +function decryptedSecret(config: S3BackendConfig): string { + const plain = decrypt_api_key(config.options.secretAccessKey); + if (plain === null) { + throw new StorageBackendError( + `s3 backend '${config.name}': could not decrypt 'secretAccessKey' — was ENCRYPTION_KEY changed or the row edited by hand?`, + ); + } + return plain; +} + +/** The key-prefix a category uses on a given backend — see keyPrefixFor's doc comment. */ +function prefixFor(category: ServedCategory, backendName: string): string { + return category === 'photos-google' && backendName === 'place-photos-local' ? '' : CATEGORY_PREFIXES[category]; +} + +/** Replica names off a snapshot's loose options record (mirrors always carry a string[] there). */ +function replicaNamesOf(options: Record): string[] { + return Array.isArray(options.replicas) ? options.replicas : []; +} + +/** Human-readable prefix for an error message — '' is the backend's whole root. */ +function describePrefix(prefix: string): string { + return prefix === '' ? 'the backend root' : `'${prefix}'`; +} + +/** First overlapping pair across two swept-prefix sets; '' (the root) overlaps everything. */ +function overlappingPrefixes(a: ReadonlySet, b: ReadonlySet): [string, string] | null { + for (const left of a) { + for (const right of b) { + if (left.startsWith(right) || right.startsWith(left)) return [left, right]; + } + } + return null; +} + +/** + * The shared-replica rule (audit critical — the backfill deletion sweep). + * + * A mirror's "Sync now" sweep makes each replica MATCH the primary: it lists + * the replica under every prefix the mirror's categories occupy and deletes + * the keys the primary doesn't hold. `backups` occupies prefix '' — the + * replica's ENTIRE root — so a backend that is simultaneously a replica and a + * category target would have that category's objects deleted by the first + * sync (silent data loss from the flagship mirror flow). The sweep is + * correct; what must not exist is a config that expresses the overlap. Hence + * two refusals, both naming the backend and the conflict: + * + * 1. a backend that serves a category — directly, or as the PRIMARY of a + * mirror a category routes to — can never also be a mirror replica; + * 2. a backend replicating two mirrors whose swept prefixes overlap (equal, + * nested, or either one '') would have each mirror's sweep delete the + * other's objects. + * + * Disjoint prefixes are fine: one backend may replicate `files` for one + * mirror and `covers` for another. The admin UI keeps the replica picker in + * step by never offering a backend that already serves a category. + */ +function assertNoSharedReplicas( + backends: Map, + categories: Map, +): void { + const replicaOf = new Map(); // backend → mirrors listing it as a replica + for (const config of backends.values()) { + if (config.type !== 'mirror') continue; + for (const replica of config.options.replicas) { + replicaOf.set(replica, [...(replicaOf.get(replica) ?? []), config.name]); + } + } + if (replicaOf.size === 0) return; + + const servedBy = new Map(); // serving backend → its categories + const sweptBy = new Map>(); // mirror → the prefixes its sweep covers + for (const [category, backendName] of categories) { + // Existence was checked by the caller, so this lookup cannot miss. + const backend = backends.get(backendName)!; + const owner = backend.type === 'mirror' ? backend.options.primary : backendName; + servedBy.set(owner, [...(servedBy.get(owner) ?? []), category]); + if (backend.type === 'mirror') { + const prefixes = sweptBy.get(backendName) ?? new Set(); + prefixes.add(prefixFor(category, backendName)); + sweptBy.set(backendName, prefixes); + } + } + + const empty: ReadonlySet = new Set(); + for (const [replica, mirrors] of replicaOf) { + const served = servedBy.get(replica); + if (served) { + throw new StorageBackendError( + `backend '${replica}' is a mirror replica of '${mirrors[0]}' and also serves category '${served[0]}' — ` + + `a backend can be a replica or a category target, not both (a sync sweep deletes replica objects the mirror's primary doesn't hold)`, + ); + } + for (let i = 0; i < mirrors.length; i += 1) { + for (let j = i + 1; j < mirrors.length; j += 1) { + const overlap = overlappingPrefixes(sweptBy.get(mirrors[i]!) ?? empty, sweptBy.get(mirrors[j]!) ?? empty); + if (overlap) { + throw new StorageBackendError( + `backend '${replica}' replicates both '${mirrors[i]}' and '${mirrors[j]}', whose swept key prefixes overlap ` + + `(${describePrefix(overlap[0])} and ${describePrefix(overlap[1])}) — one mirror's sync sweep would delete the other's objects`, + ); + } + } + } + } +} + +function validateConfig(backends: Map, categories: Map): void { + for (const config of backends.values()) { + if (config.type !== 'mirror') continue; + for (const target of [config.options.primary, ...config.options.replicas]) { + const resolved = backends.get(target); + if (!resolved) { + throw new StorageBackendError(`mirror '${config.name}' references unknown backend '${target}'`); + } + if (resolved.type === 'mirror') { + throw new StorageBackendError(`mirror '${config.name}' nests mirror '${target}' — nesting is rejected`); + } + } + } + for (const [category, backendName] of categories) { + const backend = backends.get(backendName); + if (!backend) { + throw new StorageBackendError(`category '${category}' maps to unknown backend '${backendName}'`); + } + } + assertNoSharedReplicas(backends, categories); +} diff --git a/server/src/nest/storage/storage-secrets.ts b/server/src/nest/storage/storage-secrets.ts new file mode 100644 index 000000000..3a53d884c --- /dev/null +++ b/server/src/nest/storage/storage-secrets.ts @@ -0,0 +1,129 @@ +import { + MASKED_SETTING_VALUE, + storageSecretFields, + type StorageBackend, + type StorageBackendTypeId, + type StorageConfig, +} from '@trek/shared'; +import { decrypt_api_key, maybe_encrypt_api_key } from '../common/crypto/apiKeyCrypto'; +import { StorageBackendError } from './storage.types'; + +/** + * Secret-field plumbing for admin-managed storage config (spec: + * docs/superpowers/specs/2026-08-19-storage-admin-config-design.md). Secrets + * live encrypted INSIDE the storage.backends JSON (the LLM addon-config + * precedent), on exactly the fields the shared type registry marks `secret`. + * Pure functions — the registry's seed import and the admin service share + * them. + */ + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Map one backend's secret-field values; returns a new backend object. */ +function mapSecrets(backend: StorageBackend, fn: (value: unknown, field: string) => unknown): StorageBackend { + const fields = storageSecretFields(backend.type); + if (fields.length === 0) return backend; + const options: Record = { ...backend.options }; + for (const field of fields) options[field] = fn(options[field], field); + return { ...backend, options } as StorageBackend; +} + +/** + * No literal mask sentinel may ever become a stored value, in ANY option + * field — not just the fields the type registry marks `secret`. Secret + * fields are already covered structurally (`unmaskStorageConfig` throws + * when a mask has no stored counterpart); this closes the same hole for a + * mask sentinel submitted in a non-secret field, which would otherwise be + * persisted verbatim as garbage-in. Also the seed-file guard: the seed file + * has no mask round-trip, so a literal mask sentinel there is always a + * mistake. + */ +export function assertNoMaskSentinels(config: StorageConfig): void { + for (const backend of config.backends) { + for (const field of Object.keys(backend.options as Record)) { + if ((backend.options as Record)[field] === MASKED_SETTING_VALUE) { + throw new StorageBackendError( + `backend '${backend.name}' field '${field}' is the mask sentinel — a mask can never become a stored value`, + ); + } + } + } +} + +/** Encrypt every secret field (idempotent — enc:v1: values pass through unchanged). */ +export function encryptStorageSecrets(config: StorageConfig): StorageConfig { + return { + ...config, + backends: config.backends.map((backend) => + mapSecrets(backend, (value) => + typeof value === 'string' && value !== '' ? maybe_encrypt_api_key(value) : value, + ), + ), + }; +} + +/** + * Resolve mask echoes: a secret submitted as the mask is replaced by the + * stored (still-encrypted) value for the same backend name. A mask with no + * stored counterpart (rename, or a new backend) can never become a stored + * value — callers turn this throw into the 400. + */ +export function unmaskStorageConfig(config: StorageConfig, storedBackends: unknown): StorageConfig { + const stored = new Map>(); + if (Array.isArray(storedBackends)) { + for (const entry of storedBackends) { + if (isRecord(entry) && typeof entry.name === 'string' && isRecord(entry.options)) { + stored.set(entry.name, entry.options); + } + } + } + return { + ...config, + backends: config.backends.map((backend) => + mapSecrets(backend, (value, field) => { + if (value !== MASKED_SETTING_VALUE) return value; + const storedValue = stored.get(backend.name)?.[field]; + if (typeof storedValue !== 'string' || storedValue === '') { + throw new StorageBackendError( + `re-enter the secret '${field}' for '${backend.name}' — a mask can never become a stored value (new or renamed backend)`, + ); + } + return storedValue; + }), + ), + }; +} + +/** GET masking: secret fields always render as the sentinel, never a stored value. */ +export function maskBackendOptions( + type: StorageBackendTypeId, + options: Record, +): Record { + const masked = { ...options }; + for (const field of storageSecretFields(type)) masked[field] = MASKED_SETTING_VALUE; + return masked; +} + +/** Audit redaction: secret values become '***' (names and shapes survive). */ +export function redactStorageSecrets(config: StorageConfig): StorageConfig { + return { + ...config, + backends: config.backends.map((backend) => mapSecrets(backend, () => '***')), + }; +} + +/** Decrypt secret fields for ephemeral use (test probes). Never persisted. */ +export function decryptBackendSecrets(backend: StorageBackend): StorageBackend { + return mapSecrets(backend, (value, field) => { + if (typeof value !== 'string' || value === '') return value; + const plain = decrypt_api_key(value); + if (plain === null) { + throw new StorageBackendError( + `${backend.type} backend '${backend.name}': could not decrypt '${field}' — was ENCRYPTION_KEY changed or the row edited by hand?`, + ); + } + return plain; + }); +} diff --git a/server/src/nest/storage/storage-stats.service.ts b/server/src/nest/storage/storage-stats.service.ts new file mode 100644 index 000000000..f36da67e7 --- /dev/null +++ b/server/src/nest/storage/storage-stats.service.ts @@ -0,0 +1,68 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { STORAGE_CATEGORIES, storageUsageSchema, type StorageUsage } from '@trek/shared'; +import { DatabaseService } from '../database/database.service'; +import { StorageService } from './storage.service'; + +export class StatsBusyError extends Error {} + +export const USAGE_KEY = 'storage.usage'; + +/** + * Usage scan (backfill/stats spec): walks every served category through the + * facade — a mirrored category therefore lists its PRIMARY, the source of + * truth — and persists one JSON row with computedAt. Local backends pay one + * stat per file; s3 sizes ride the paginated listing. photos-google / + * photos-trek nest under the legacy photos/ prefix in mode A, so the legacy + * walk must exclude their subtrees to avoid double counting — the same + * google/ trek/ skip the backup walk uses. + */ +@Injectable() +export class StorageStatsService { + private readonly logger = new Logger(StorageStatsService.name); + private scanning = false; + + constructor( + private readonly storage: StorageService, + private readonly db: DatabaseService, + ) {} + + async scan(): Promise { + if (this.scanning) throw new StatsBusyError('a usage scan is already running'); + this.scanning = true; + try { + const categories = {} as StorageUsage['categories']; + for (const category of STORAGE_CATEGORIES) { + let objects = 0; + let bytes = 0; + for await (const stat of this.storage.list(category)) { + objects += 1; + bytes += stat.size; + } + categories[category] = { objects, bytes }; + } + const legacyPhotos = { objects: 0, bytes: 0 }; + for await (const stat of this.storage.list('photos')) { + if (stat.key.startsWith('google/') || stat.key.startsWith('trek/')) continue; + legacyPhotos.objects += 1; + legacyPhotos.bytes += stat.size; + } + const usage: StorageUsage = { computedAt: Date.now(), categories, legacyPhotos }; + this.db.run('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)', USAGE_KEY, JSON.stringify(usage)); + return usage; + } finally { + this.scanning = false; + } + } + + /** The stored row, parsed; null when absent or unparseable (logged, never a 500). */ + readUsage(): StorageUsage | null { + const row = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', USAGE_KEY); + if (!row) return null; + try { + return storageUsageSchema.parse(JSON.parse(row.value)); + } catch { + this.logger.warn('stored storage.usage row is unparseable — treating as never computed'); + return null; + } + } +} diff --git a/server/src/nest/storage/storage-upload.factory.ts b/server/src/nest/storage/storage-upload.factory.ts new file mode 100644 index 000000000..efbd16776 --- /dev/null +++ b/server/src/nest/storage/storage-upload.factory.ts @@ -0,0 +1,70 @@ +/** + * Multer options factory for storage-backed uploads. + * + * Every disk-backed upload route consumes this through its module's + * MulterModule.registerAsync DI factory: multer spools the request body into + * the category backend's own spool dir (same volume as the destination), and + * the handler commits with storage.put(category, file.filename, + * { tmpPath: file.path }) — an atomic same-volume rename. + * + * The factory is deliberately ignorant of allowlists and per-route quirks: + * fileFilter closures are caller-owned and passed through untouched. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { randomUUID } from 'crypto'; +import { diskStorage } from 'multer'; +import type { Options } from 'multer'; +import type { Request } from 'express'; +import { StorageService } from './storage.service'; +import type { StorageCategory } from './storage.types'; + +export interface StorageUploadOptions { + /** + * Constant for single-category modules; a per-request/per-file resolver for + * modules whose routes feed different categories (collections: fieldname + * 'image' → 'places', else 'covers'). + */ + category: StorageCategory | ((req: Request, file: Express.Multer.File) => StorageCategory); + /** → limits.fileSize */ + maxSize: number; + /** Caller-built closure, passed through untouched. */ + fileFilter?: Options['fileFilter']; + /** + * Returns the full final filename. Default: `${randomUUID()}${extname}` — + * extension case-preserved, no fallback for extensionless names (parity with + * the pre-factory call sites). + */ + filename?: (req: Request, file: Express.Multer.File) => string; + /** Always 'utf8' since fix #3; the option remains for explicit callers. */ + defParamCharset?: 'utf8'; +} + +export function buildStorageUploadOptions(storage: StorageService, opts: StorageUploadOptions) { + const resolveCategory = + typeof opts.category === 'function' ? opts.category : () => opts.category as StorageCategory; + const makeName = + opts.filename ?? + ((_req: Request, file: Express.Multer.File) => `${randomUUID()}${path.extname(file.originalname)}`); + return { + storage: diskStorage({ + // Resolved per request on purpose (registry no-cached-refs rule): a + // reload() that remaps the category must be picked up by the next upload. + destination: (req, file, cb) => { + try { + const dir = storage.spoolDirFor(resolveCategory(req, file)); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + cb(null, dir); + } catch (err) { + cb(err as Error, ''); + } + }, + filename: (req, file, cb) => cb(null, makeName(req, file)), + }), + limits: { fileSize: opts.maxSize }, + ...(opts.fileFilter ? { fileFilter: opts.fileFilter } : {}), + // Uniform since fix #3: non-ASCII original filenames decode as UTF-8 on + // every upload route (previously only trip files + collab opted in). + defParamCharset: opts.defParamCharset ?? 'utf8', + }; +} diff --git a/server/src/nest/storage/storage-usage-scan.job.ts b/server/src/nest/storage/storage-usage-scan.job.ts new file mode 100644 index 000000000..956a6ce52 --- /dev/null +++ b/server/src/nest/storage/storage-usage-scan.job.ts @@ -0,0 +1,29 @@ +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { StorageStatsService } from './storage-stats.service'; + +/** Nightly usage scan — 04:15, off the backup and thumb-sweep hours. */ +@Injectable() +export class StorageUsageScanJob implements OnApplicationBootstrap { + private readonly logger = new Logger(StorageUsageScanJob.name); + + constructor( + private readonly registrar: CronRegistrarService, + private readonly stats: StorageStatsService, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + this.registrar.register('storage-usage-scan', '15 4 * * *', () => { + void this.tick(); + }); + } + + async tick(): Promise { + try { + await this.stats.scan(); + } catch (err) { + this.logger.error(`usage scan failed: ${err instanceof Error ? err.message : String(err)}`); + } + } +} diff --git a/server/src/nest/storage/storage.module.ts b/server/src/nest/storage/storage.module.ts new file mode 100644 index 000000000..ad8fb644d --- /dev/null +++ b/server/src/nest/storage/storage.module.ts @@ -0,0 +1,41 @@ +import { Module } from '@nestjs/common'; +import { AppConfigModule } from '../app-config/app-config.module'; +import { AuditModule } from '../audit/audit.module'; +import { SchedulingModule } from '../scheduling/scheduling.module'; +import { StorageEventsService } from './storage-events.service'; +import { StorageJobsService } from './storage-jobs.service'; +import { StorageRegistryService } from './storage-registry.service'; +import { StorageService } from './storage.service'; +import { StorageAdminService } from './storage-admin.service'; +import { StorageAdminController } from './storage-admin.controller'; +import { StorageStatsService } from './storage-stats.service'; +import { StorageUsageScanJob } from './storage-usage-scan.job'; + +/** + * Storage container: registry (config), facade (byte-paths), admin surface. + * AuditModule feeds the write audits. AuthModule is deliberately NOT imported + * here — AuthModule itself imports StorageModule (avatar uploads), so the + * reverse import is a real module cycle (Nest resolves it as `imports[1] is + * undefined`, not a clean forwardRef case). JwtAuthGuard/AdminGuard need no + * provider from AuthModule to begin with — they carry no constructor + * dependencies, so `@UseGuards(JwtAuthGuard, AdminGuard)` instantiates them + * directly, the same as BackupModule/BackupController (which also sits behind + * StorageModule) already does. StorageRegistryService stays UNEXPORTED — the + * admin controller reaches it as a same-module provider, and nothing outside + * may cache drivers or trigger reloads. + */ +@Module({ + imports: [AppConfigModule, AuditModule, SchedulingModule], + controllers: [StorageAdminController], + providers: [ + StorageRegistryService, + StorageService, + StorageAdminService, + StorageEventsService, + StorageJobsService, + StorageStatsService, + StorageUsageScanJob, + ], + exports: [StorageService, StorageEventsService], +}) +export class StorageModule {} diff --git a/server/src/nest/storage/storage.service.ts b/server/src/nest/storage/storage.service.ts new file mode 100644 index 000000000..4c4f21e65 --- /dev/null +++ b/server/src/nest/storage/storage.service.ts @@ -0,0 +1,370 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { pipeline } from 'node:stream/promises'; +import type { Readable } from 'node:stream'; +import { Injectable } from '@nestjs/common'; +import type { Response } from 'express'; +import { contentTypeFor } from './content-type'; +import type { ReplicaFailure } from './drivers/mirror.driver'; +import { decideRange, isNotModified, validatorsFor, type ServingHeaders, type Validators } from './http-serving'; +import { assertValidKey } from './storage-keys'; +import { StorageRegistryService } from './storage-registry.service'; +import { + isClientAbortError, + StorageNotFoundError, + type ByteRange, + type LocalTempFile, + type ObjectStat, + type PutOptions, + type ServedCategory, + type StorageDriver, +} from './storage.types'; + +export interface SendOptions { + contentType?: string; + disposition?: string; +} + +interface Resolved { + driver: StorageDriver; + keyPrefix: string; + key: string; +} + +/** + * The facade all call sites use — and the ONLY entry point: callers address + * storage exclusively as (category, name) on this injected service. They + * never import a driver, never hold a driver or backend reference, and never + * know which instance serves a category — resolution happens inside the + * registry on every call. That is what makes backend reconfiguration a + * config-plus-reload() operation instead of a code change. + * + * `sendToResponse` and `withLocalFile` are the two helpers that make a remote + * backend a driver-only change later: every byte-consumer goes through one of + * them, and each owns the local-fast-path vs stream branch exactly once. + */ +@Injectable() +export class StorageService { + constructor(private readonly registry: StorageRegistryService) {} + + /** Driver-agnostic global scratch space (data/tmp). */ + tempDir(): string { + return this.registry.tempDir(); + } + + /** The category backend's same-volume spool dir, else tempDir(). Resolved per call. */ + spoolDirFor(category: ServedCategory): string { + return this.registry.resolve(category).driver.getSpoolDir?.() ?? this.tempDir(); + } + + // The read/write methods are async so key-validation failures surface as + // rejections, never synchronous throws — callers treat storage uniformly. + + async put( + category: ServedCategory, + name: string, + source: Readable | LocalTempFile, + opts?: PutOptions, + ): Promise { + const { driver, key } = this.resolve(category, name); + return driver.put(key, source, opts); + } + + async getStream( + category: ServedCategory, + name: string, + range?: ByteRange, + ): Promise<{ stream: Readable; stat: ObjectStat }> { + const { driver, key } = this.resolve(category, name); + return driver.getStream(key, range); + } + + async stat(category: ServedCategory, name: string): Promise { + const { driver, key } = this.resolve(category, name); + return driver.stat(key); + } + + async exists(category: ServedCategory, name: string): Promise { + return (await this.stat(category, name)) !== null; + } + + async delete(category: ServedCategory, name: string): Promise { + const { driver, key } = this.resolve(category, name); + return driver.delete(key); + } + + /** Names in and out are category-relative — the key prefix stays a registry detail. */ + async *list(category: ServedCategory, subPrefix = ''): AsyncIterable { + const { driver, keyPrefix } = this.registry.resolve(category); + for await (const stat of driver.list(keyPrefix + subPrefix)) { + yield { ...stat, key: stat.key.slice(keyPrefix.length) }; + } + } + + /** + * Serve an object into an Express response. Local backends use the + * root-relative res.sendFile form — absolute paths resolve against the + * rewritten req.url under the ExpressAdapter and spuriously 404 + * (files-download.controller.ts) — which brings etag/range/conditional-GET + * for free. Path-less (remote) drivers, and a local path whose file has + * vanished from disk (checked via `localPathIfPresent`, never trusted from + * `getLocalPath` alone), both fall through to the stream branch — for a + * MirrorDriver that reaches a replica that still holds the object, for a + * plain LocalDriver a genuine miss (`sendStreamed`'s pre-flight stat throws + * StorageNotFoundError, same as always — one extra driver call). A miss + * throws StorageNotFoundError: the caller owns the route's miss contract + * (404 envelope, 204-empty, or next()). + */ + async sendToResponse(category: ServedCategory, name: string, res: Response, opts?: SendOptions): Promise { + const { driver, key } = this.resolve(category, name); + + const localPath = this.localPathIfPresent(driver, key); + if (localPath !== null) { + this.applyHeaders(res, opts); + await new Promise((resolve, reject) => { + res.sendFile(path.basename(localPath), { root: path.dirname(localPath) }, (err) => { + if (!err) { + resolve(); + return; + } + // Client gone / bytes already on the wire: nothing useful to send — + // mirrors res.sendFile's own default callback (express + // response.js), which is the exact contract + // storageStaticHandler (platform.routes.ts) already implements + // caller-side for the /uploads/* static mounts. + if (isClientAbortError(err)) { + resolve(); + return; + } + reject(err); + }); + }); + return; + } + + await this.sendStreamed(driver, key, res, opts); + } + + /** + * The proxying serve — every path-less backend, plus a local primary whose + * file has vanished. `res.sendFile` gives the local branch content-type + * sniffing, ETag/Last-Modified, conditional GET, Range and HEAD for free; + * nothing gives them to a stream, so this reproduces the same contract from + * ONE pre-flight `stat()` plus the request headers (decisions live in the + * pure http-serving.ts). + * + * That stat is what makes the rest possible and is deliberately paid on + * every request: it turns a miss into a throw before a single header is + * set (the caller's 404/204/next() contract), and one HeadObject then + * answers 304s, 416s, HEADs and suffix ranges without ever opening a body. + * It is also the single source of truth for the range math — the ranged + * `getStream` response is never re-consulted for the total size. + */ + private async sendStreamed(driver: StorageDriver, key: string, res: Response, opts?: SendOptions): Promise { + const stat = await driver.stat(key); + if (stat === null) throw new StorageNotFoundError(key); + // Express always populates res.req — the local branch's res.sendFile + // already depends on it — so serving needs no caller churn. + const req = res.req; + const headers = req.headers as ServingHeaders; + const validators = validatorsFor(stat); + + if (isNotModified(headers, validators)) { + // 304 carries validators and nothing else: no body, no Content-Length, + // and above all no getStream — the whole point of revalidating. + this.applyValidators(res, validators); + res.statusCode = 304; + res.end(); + return; + } + + const decision = decideRange(headers, stat.size, validators.etag); + if (decision.kind === 'unsatisfiable') { + res.statusCode = 416; + res.setHeader('Content-Range', `bytes */${stat.size}`); + res.end(); + return; + } + + const range = decision.kind === 'partial' ? { start: decision.start, end: decision.end } : undefined; + + // HEAD is answered entirely from the stat — fetching a body just to + // discard it would be one wasted round trip per request. + if (req.method === 'HEAD') { + this.stageServeHeaders(res, key, opts, validators, stat.size, range); + res.end(); + return; + } + + // The body opens BEFORE a single header is staged. getStream can still + // fail after a successful stat — a delete that raced it, a backend blip — + // and a caller that recovers on the SAME response object must find it + // pristine: trek-photo-cache.service.ts treats a pre-header + // StorageNotFoundError as a cache miss and photo-resolver.service.ts then + // re-serves through that very res. A 206 status and a Content-Range left + // behind by an attempt that never produced a byte would corrupt that + // retry. + const { stream } = await driver.getStream(key, range); + try { + this.stageServeHeaders(res, key, opts, validators, stat.size, range); + } catch (err) { + // The body is already open (an S3 keep-alive socket, an fd) and nothing + // has piped it yet, so nothing else ever will — destroy it before the + // throw escapes. Staging really can fail: res.setHeader raises + // ERR_INVALID_CHAR on a control character, and Content-Disposition is + // built from client-supplied filenames (files-download.controller.ts + // uses multer's originalname), so a name with a CR/LF reaches here. + stream.destroy(); + throw err; + } + try { + // pipeline (unlike the hand-rolled pipe+Promise it replaces) destroys + // BOTH ends on any failure, so a client abort no longer leaks the + // source stream (an S3 keep-alive socket, an open fd, …). + await pipeline(stream, res); + } catch (err) { + // Gate the swallow on headersSent, not just the error code: the source + // here is the driver's raw stream (e.g. the S3 SDK's HTTP response), + // which can itself throw ECONNRESET/ERR_STREAM_PREMATURE_CLOSE on a + // genuine network blip that has nothing to do with the browser client + // — before any byte has reached res. A pre-header failure of ANY code + // must still reject so the caller's miss contract (404/204/next()) + // runs; only a POST-header failure is the client walking away + // mid-download, and only then is an abort-like code safe to swallow. + if (!res.headersSent || !isClientAbortError(err)) throw err; + } + } + + /** + * Hand a consumer (Jimp, zip packing, Immich push, …) a real filesystem + * path: the local fast-path when the driver has one AND the file is + * actually on disk (`localPathIfPresent`), else download to tempDir() and + * clean up afterwards — the same fall-through `sendToResponse` uses, and + * exactly the branch a remote driver (or a local path gone missing) will + * exercise. + */ + async withLocalFile(category: ServedCategory, name: string, fn: (absPath: string) => Promise): Promise { + const { driver, key } = this.resolve(category, name); + + const localPath = this.localPathIfPresent(driver, key); + if (localPath !== null) { + return fn(localPath); + } + + const tmp = path.join(this.tempDir(), randomUUID()); + const { stream } = await driver.getStream(key); + await pipeline(stream, fs.createWriteStream(tmp)); + try { + return await fn(tmp); + } finally { + await fs.promises.rm(tmp, { force: true }); + } + } + + /** + * A narrow locality probe: the real filesystem path for the object, or + * null when there isn't one — a remote (path-less) driver, or a local + * driver whose file no longer exists on disk. The fs-existence check is + * the fail-safe half of the contract: a caller that trusts a stale path + * (deleted between listing and this call, or by another process) would + * otherwise silently skip or corrupt-read the object instead of falling + * back to the stream path. Same spirit as spoolDirFor — exposes a path, + * never a driver. + */ + async getLocalPathOrNull(category: ServedCategory, name: string): Promise { + const { driver, key } = this.resolve(category, name); + return this.localPathIfPresent(driver, key); + } + + /** Replica failures from mirror backends — logged there, surfaced here. */ + health(): { replicaFailures: readonly ReplicaFailure[] } { + return { replicaFailures: this.registry.replicaFailures() }; + } + + /** + * Re-read storage config from `app_settings` and swap the registry's + * resolved backends/categories — the narrow passthrough a restore uses + * after reopening the DB (backup.impl.ts's restoreFromZip), so rehydrated + * bytes land where the RESTORED config says rather than the stale + * pre-restore one. Same shape as health(): StorageRegistryService stays + * unexported from the module, this facade is the only way in. + */ + reloadConfig(): void { + this.registry.reload(); + } + + private resolve(category: ServedCategory, name: string): Resolved { + const { driver, keyPrefix } = this.registry.resolve(category); + const key = keyPrefix + name; + assertValidKey(key); + return { driver, keyPrefix, key }; + } + + /** + * The shared local-fast-path gate for sendToResponse/withLocalFile/ + * getLocalPathOrNull: a driver's `getLocalPath` alone is never trusted as + * proof the object is at that path — composite drivers (MirrorDriver) + * resolve it against the primary ONLY, and even a plain local driver's + * path can vanish between listing and this call. Null here means "no + * usable local path" for any reason; callers fall through to the stream + * branch, which is what actually consults a MirrorDriver's replicas. + */ + private localPathIfPresent(driver: StorageDriver, key: string): string | null { + const localPath = driver.getLocalPath?.(key) ?? null; + if (localPath === null) return null; + return fs.existsSync(localPath) ? localPath : null; + } + + private applyHeaders(res: Response, opts?: SendOptions): void { + if (opts?.contentType) res.setHeader('Content-Type', opts.contentType); + if (opts?.disposition) res.setHeader('Content-Disposition', opts.disposition); + } + + /** + * Everything the 200/206 answer needs, staged in one place so the HEAD + * branch and the streaming branch cannot drift apart. + * + * Content-Type precedence is three-way and load-bearing: an explicit + * `opts.contentType` wins; else a type the CALLER already staged on the + * response survives — trek-photo-cache.service.ts sets the real type before + * calling because its objects are extension-less `.bin` blobs, and the local + * res.sendFile branch has always preserved a pre-set type; else derive from + * the key. Only the last case is new behaviour, and it exists because there + * is no content sniffing here — no type at all makes a browser download an + * image instead of showing it. + */ + private stageServeHeaders( + res: Response, + key: string, + opts: SendOptions | undefined, + validators: Validators, + size: number, + range?: { start: number; end: number }, + ): void { + // Truthiness (not `??`) mirrors applyHeaders: a blank contentType is + // "unset", never an empty Content-Type header. + if (opts?.contentType) { + res.setHeader('Content-Type', opts.contentType); + } else if (!res.getHeader('Content-Type')) { + res.setHeader('Content-Type', contentTypeFor(key)); + } + if (opts?.disposition) res.setHeader('Content-Disposition', opts.disposition); + res.setHeader('Accept-Ranges', 'bytes'); + this.applyValidators(res, validators); + + if (range) { + res.statusCode = 206; + res.setHeader('Content-Range', `bytes ${range.start}-${range.end}/${size}`); + res.setHeader('Content-Length', String(range.end - range.start + 1)); + } else { + res.setHeader('Content-Length', String(size)); + } + } + + /** Either half may be absent — see validatorsFor's degraded (mtimeMs 0) case. */ + private applyValidators(res: Response, validators: Validators): void { + if (validators.etag) res.setHeader('ETag', validators.etag); + if (validators.lastModified) res.setHeader('Last-Modified', validators.lastModified); + } +} diff --git a/server/src/nest/storage/storage.types.ts b/server/src/nest/storage/storage.types.ts new file mode 100644 index 000000000..d1c8af861 --- /dev/null +++ b/server/src/nest/storage/storage.types.ts @@ -0,0 +1,178 @@ +import type { Readable } from 'node:stream'; + +/** + * The storage abstraction's contract surface (spec: + * docs/superpowers/specs/2026-07-20-storage-backend-abstraction-design.md). + * + * S3 constraints are baked into the driver contract, not the implementation: + * keys are opaque POSIX-style strings, no rename/append/partial writes, no + * directory semantics. Adding a backend later means one driver file that + * passes the contract suite (tests/unit/nest/storage/storage-driver.contract.ts) + * — zero changes to controllers or services. + */ + +// Relocated to shared (the wire schema needs it; a second copy is forbidden). +// Re-exported so every existing `./storage.types` import keeps working. +export { STORAGE_CATEGORIES, type StorageCategory } from '@trek/shared'; +import { STORAGE_CATEGORIES as CONFIGURABLE_CATEGORIES, type StorageCategory as ConfigurableCategory } from '@trek/shared'; + +/** + * Categories the byte-paths serve. `photos` is the legacy shared photo + * directory (/uploads/photos): earlier TREK versions wrote place photos there + * and its objects are still served and backed up, but it is NOT configurable — + * it left STORAGE_CATEGORIES (category-labels spec) and always resolves to + * uploads-local (following a name-override, never a category entry). + */ +export const SERVED_CATEGORIES = [...CONFIGURABLE_CATEGORIES, 'photos'] as const; +export type ServedCategory = ConfigurableCategory | 'photos'; + +export interface ObjectStat { + key: string; + size: number; + mtimeMs: number; + /** + * The backend's own entity tag when it has one — S3 returns it on every + * HEAD/GET. Always a wire-ready HTTP entity-tag (quoted, `W/`-prefixed if + * weak): the opaque value is the backend's, but a driver may re-add the + * quotes its SDK stripped (see s3.driver.ts's normalizeEtag). Optional + * because a filesystem has no such concept — the serving path then derives + * express/send's weak `W/"-"` tag instead (http-serving.ts). + * Never synthesized by a driver, so "absent" always means "the backend + * didn't say". + */ + etag?: string; +} + +/** Inclusive byte range, `fs.createReadStream` semantics; `end` omitted = to EOF. */ +export interface ByteRange { + start: number; + end?: number; +} + +/** Ignored by local drivers; a future S3 driver needs it for the PUT. */ +export interface PutOptions { + contentType?: string; +} + +/** + * A temp file whose OWNERSHIP transfers to `put()`: the driver moves or + * consumes it (rename into place locally; upload-then-delete remotely). + * Callers must not touch the path afterwards. + */ +export interface LocalTempFile { + tmpPath: string; +} + +export function isLocalTempFile(source: Readable | LocalTempFile): source is LocalTempFile { + return typeof (source as LocalTempFile).tmpPath === 'string'; +} + +/** + * A socket-facing error code/syscall that LOOKS like "the client walked away + * or the pipe broke" — but is not, by itself, proof of that: the same codes + * (ECONNRESET, ERR_STREAM_PREMATURE_CLOSE) can come from a driver's own + * source stream failing for a real reason (e.g. the S3 SDK's HTTP response + * dropping on a genuine network blip), independent of the browser client. + * Callers MUST additionally gate on `res.headersSent` before treating a + * match as safe to swallow: a pre-header failure is always a real failure + * (the caller's miss contract — 404/204/next() — must still run), and only a + * POST-header match is the client walking away mid-download. See + * storage.service.ts's sendToResponse (the remote pipeline() branch gates + * this way explicitly; the local res.sendFile branch's callback only ever + * fires post-attempt, so it swallows unconditionally) and the maps/share + * place-photo proxies, which gate the same way around their own pipeline() + * call. + */ +export function isClientAbortError(err: unknown): boolean { + const e = err as { code?: string; syscall?: string } | null | undefined; + if (!e || typeof e !== 'object') return false; + return ( + e.code === 'ECONNABORTED' || + e.code === 'ECONNRESET' || + e.code === 'ERR_STREAM_PREMATURE_CLOSE' || + e.syscall === 'write' + ); +} + +export interface StorageDriver { + /** Backend instance name, e.g. 'uploads-local'. */ + readonly id: string; + /** Atomic: spool + single rename locally; single/multipart PUT remotely. */ + put(key: string, source: Readable | LocalTempFile, opts?: PutOptions): Promise; + getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }>; + stat(key: string): Promise; + /** Idempotent; resolving on a missing key is success. */ + delete(key: string): Promise; + list(prefix: string): AsyncIterable; + /** Fast-path for local drivers; `null`/absent is exactly the remote-driver branch every caller must handle. */ + getLocalPath?(key: string): string | null; + /** Same-filesystem spool dir (LocalDriver: `/.tmp`), or null for drivers with no local filesystem. */ + getSpoolDir?(): string | null; +} + +/** + * Driver wrappers (S3Driver.wrap) keep the underlying failure as `cause` and + * out of `message` — right for logs, useless anywhere the message is shown to + * an admin (probe results, replica-failure notifications), where "put failed" + * must say WHY (ECONNREFUSED vs NoSuchBucket). Flattens the cause chain, + * appending an S3-style `code` when it isn't already in that link's message. + */ +export function describeError(err: unknown): string { + const parts: string[] = []; + const seen = new Set(); + for (let cur: unknown = err; cur != null && !seen.has(cur); ) { + seen.add(cur); + const message = cur instanceof Error ? cur.message : String(cur); + const code = (cur as { code?: unknown }).code; + parts.push(typeof code === 'string' && code && !message.includes(code) ? `${message} (${code})` : message); + cur = cur instanceof Error ? (cur as { cause?: unknown }).cause : undefined; + } + return parts.filter(Boolean).join(': '); +} + +/** + * Typed error set so route handlers map storage failures to their existing + * bespoke envelopes without string-matching driver internals. + */ +export class StorageBackendError extends Error { + constructor( + message: string, + readonly cause?: unknown, + ) { + super(message); + this.name = 'StorageBackendError'; + } +} + +/** + * Optimistic-concurrency refusal (audit #7): the submitted config's `version` + * no longer matches the stored counter — something else (a config save, or a + * category migration's flip) wrote since the client last read state. Mapped + * to a 409 ahead of the blanket 400 (storage-admin.controller.ts). + */ +export class StorageConflictError extends Error { + constructor( + readonly currentVersion: number, + readonly submittedVersion: number, + ) { + super( + `storage settings changed since this form was loaded (current version ${currentVersion}, submitted ${submittedVersion}) — reload and reapply your edits`, + ); + this.name = 'StorageConflictError'; + } +} + +export class StorageNotFoundError extends Error { + constructor(readonly key: string) { + super(`storage object not found: ${key}`); + this.name = 'StorageNotFoundError'; + } +} + +/** Serving slices map this to a miss (404/next()), never a 500. */ +export class StorageInvalidKeyError extends Error { + constructor(readonly key: string) { + super(`invalid storage key: ${key}`); + this.name = 'StorageInvalidKeyError'; + } +} diff --git a/server/src/nest/trips/trips.controller.ts b/server/src/nest/trips/trips.controller.ts index cae1d961b..36b6bd7c4 100644 --- a/server/src/nest/trips/trips.controller.ts +++ b/server/src/nest/trips/trips.controller.ts @@ -20,11 +20,10 @@ import { FileInterceptor } from '@nestjs/platform-express'; import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; import type { Request, Response } from 'express'; -import { diskStorage } from 'multer'; +import type { Options } from 'multer'; import path from 'path'; -import fs from 'fs'; -import { v4 as uuidv4 } from 'uuid'; import type { ActiveTripResponse } from '@trek/shared'; +import { StorageService } from '../storage/storage.service'; import type { User } from '../../types'; import { TripsService } from './trips.service'; import { JwtAuthGuard } from '../auth/jwt-auth.guard'; @@ -38,23 +37,16 @@ import { UnsplashService } from '../unsplash/unsplash.service'; import { CalendarService } from '../calendar/calendar.service'; import { TripReadModelService } from '../trip-read-model/trip-read-model.service'; -const MAX_COVER_SIZE = 20 * 1024 * 1024; -const coversDir = path.join(__dirname, '../../../uploads/covers'); -const COVER_UPLOAD = { - storage: diskStorage({ - destination: (_req, _file, cb) => { - if (!fs.existsSync(coversDir)) fs.mkdirSync(coversDir, { recursive: true }); - cb(null, coversDir); - }, - filename: (_req, file, cb) => cb(null, `${uuidv4()}${path.extname(file.originalname)}`), - }), - limits: { fileSize: MAX_COVER_SIZE }, - fileFilter: (_req: Request, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => { - const ext = path.extname(file.originalname).toLowerCase(); - const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; - if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true); - else cb(new Error('Only jpg, png, gif, webp images allowed'), false); - }, +export const MAX_COVER_SIZE = 20 * 1024 * 1024; +// Still needed by the Unsplash cover download (a raw-fs writer until the +// caches/downloads slice); the multer destination moved to the storage spool. +// Consumed by trips.module.ts's MulterModule factory. Quirk preserved on +// purpose: a plain Error without statusCode maps to 500, not 400 (parity). +export const TRIP_COVER_FILE_FILTER: Options['fileFilter'] = (_req, file, cb) => { + const ext = path.extname(file.originalname).toLowerCase(); + const allowed = ['.jpg', '.jpeg', '.png', '.gif', '.webp']; + if (file.mimetype.startsWith('image/') && !file.mimetype.includes('svg') && allowed.includes(ext)) cb(null, true); + else cb(new Error('Only jpg, png, gif, webp images allowed')); }; const toDateStr = (d: Date) => d.toISOString().slice(0, 10); @@ -74,7 +66,7 @@ const addDays = (d: Date, n: number) => { const r = new Date(d); r.setDate(r.get export class TripsController { // calendar last: the hand-wired construction sites in the tests stay positional, // so a new dependency does not touch the ones that never reach the ICS route. - constructor(private readonly trips: TripsService, private readonly audit: AuditService, private readonly env: RuntimeEnvService, private readonly unsplash: UnsplashService, private readonly calendar: CalendarService, private readonly readModel: TripReadModelService) {} + constructor(private readonly trips: TripsService, private readonly audit: AuditService, private readonly env: RuntimeEnvService, private readonly unsplash: UnsplashService, private readonly calendar: CalendarService, private readonly readModel: TripReadModelService, private readonly storage: StorageService) {} @Get() list(@CurrentUser() user: User, @Query('archived') archived?: string) { @@ -161,7 +153,7 @@ export class TripsController { // it into uploads/covers so the cover survives offline + CDN link-rot (#1277). if (this.unsplash.isUnsplashCoverUrl(body.cover_image)) { try { - const filename = await this.unsplash.saveUnsplashCover(body.cover_image, coversDir); + const filename = await this.unsplash.saveUnsplashCover(body.cover_image); body.cover_image = `/uploads/covers/${filename}`; } catch (e) { console.error('Unsplash cover download failed:', e); @@ -174,7 +166,7 @@ export class TripsController { try { const result = await this.trips.update(id, user.id, body, user.role); if (body.cover_image !== undefined && body.cover_image !== oldCover) { - this.trips.deleteOldCover(oldCover); + await this.trips.deleteOldCover(oldCover); } if (Object.keys(result.changes).length > 0) { this.audit.writeAudit({ userId: user.id, action: 'trip.update', ip: getClientIp(req), details: { tripId: Number(id), trip: result.newTitle, ...(result.ownerEmail ? { owner: result.ownerEmail } : {}), ...result.changes } }); @@ -194,8 +186,8 @@ export class TripsController { } @Post(':id/cover') - @UseInterceptors(FileInterceptor('cover', COVER_UPLOAD)) - cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) { + @UseInterceptors(FileInterceptor('cover')) + async cover(@CurrentUser() user: User, @Param('id') id: string, @UploadedFile() file: Express.Multer.File | undefined) { if (isDemoWriteBlocked(this.env, user.email)) { throw new HttpException(DEMO_WRITE_ERROR, 403); } @@ -213,7 +205,10 @@ export class TripsController { if (!file) { throw new HttpException({ error: 'No image uploaded' }, 400); } - this.trips.deleteOldCover(trip.cover_image); + // Commit the spooled upload to its final storage location (atomic + // same-volume rename) before anything references the final path. + await this.storage.put('covers', file.filename, { tmpPath: file.path }); + await this.trips.deleteOldCover(trip.cover_image); const coverUrl = `/uploads/covers/${file.filename}`; this.trips.updateCoverImage(id, coverUrl); return { cover_image: coverUrl }; diff --git a/server/src/nest/trips/trips.module.ts b/server/src/nest/trips/trips.module.ts index 2b678c21b..a849ab1ac 100644 --- a/server/src/nest/trips/trips.module.ts +++ b/server/src/nest/trips/trips.module.ts @@ -27,11 +27,28 @@ import { TripMembersModule } from '../trip-members/trip-members.module'; import { TripReadModelModule } from '../trip-read-model/trip-read-model.module'; import { AddonsModule } from '../addons/addons.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; +import { MulterModule } from '@nestjs/platform-express'; +import { StorageModule } from '../storage/storage.module'; +import { StorageService } from '../storage/storage.service'; +import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; +import { MAX_COVER_SIZE, TRIP_COVER_FILE_FILTER } from './trips.controller'; /** Trips aggregate root (C1 — Phase 3). Uses exact strangler prefixes so it does * not capture the nested sub-domain mounts (collab, files, ...). */ @Module({ - imports: [McpSharedModule, TodoModule, PackingModule, FilesModule, ReservationsModule, DaysModule, PermissionsModule, AuditModule, BudgetModule, CollabModule, VacayModule, PlacesModule, AuthModule, AppConfigModule, UnsplashModule, RealtimeModule, PluginGuardsModule, AddonsModule, TripMembershipModule, CalendarModule, AccommodationsModule, TripMembersModule, TripReadModelModule], + imports: [ + MulterModule.registerAsync({ + imports: [StorageModule], + inject: [StorageService], + useFactory: (storage: StorageService) => + buildStorageUploadOptions(storage, { + category: 'covers', + maxSize: MAX_COVER_SIZE, + fileFilter: TRIP_COVER_FILE_FILTER, + }), + }), + StorageModule, + McpSharedModule, TodoModule, PackingModule, FilesModule, ReservationsModule, DaysModule, PermissionsModule, AuditModule, BudgetModule, CollabModule, VacayModule, PlacesModule, AuthModule, AppConfigModule, UnsplashModule, RealtimeModule, PluginGuardsModule, AddonsModule, TripMembershipModule, CalendarModule, AccommodationsModule, TripMembersModule, TripReadModelModule], controllers: [TripsController], providers: [TripsService, TripsMcp, TripPromptsMcp, TripsRpc], // Exported for FeedsModule (ICS feeds) and PluginsModule (RPC host injection). diff --git a/server/src/nest/trips/trips.service.ts b/server/src/nest/trips/trips.service.ts index 032c38b15..be1f14b1e 100644 --- a/server/src/nest/trips/trips.service.ts +++ b/server/src/nest/trips/trips.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import path from 'path'; -import fs from 'fs'; import { DatabaseService } from '../database/database.service'; import type { ActiveTrip, TrekWsPayload, TrekWsTripEventName } from '@trek/shared'; import { RealtimeService } from '../realtime/realtime.service'; @@ -11,6 +10,7 @@ import { BudgetService } from '../budget/budget.service'; import { ReservationsService } from '../reservations/reservations.service'; import { VacayService } from '../vacay/vacay.service'; import { UnsplashService } from '../unsplash/unsplash.service'; +import { StorageService } from '../storage/storage.service'; import { NotFoundError, ValidationError } from '../common/domain-errors'; export const MS_PER_DAY = 86400000; @@ -146,6 +146,7 @@ export class TripsService { private readonly vacay: VacayService, private readonly realtime: RealtimeService, private readonly unsplash: UnsplashService, + private readonly storage: StorageService, ) {} private get db() { @@ -499,17 +500,15 @@ export class TripsService { // ── Cover image ─────────────────────────────────────────────────────────── - deleteOldCover(coverImage: string | null | undefined): void { + async deleteOldCover(coverImage: string | null | undefined): Promise { if (!coverImage) return; - // cover_image is client-supplied, so treat it as untrusted: covers live in - // uploads/covers as a flat filename — use basename() and confine the unlink - // to that directory. (dist/nest/trips is one level deeper than the legacy - // dist/services home, hence the extra '..' — same resolved directory.) - const coversDir = path.resolve(__dirname, '../../../uploads/covers'); - const resolvedPath = path.resolve(path.join(coversDir, path.basename(coverImage))); - if (resolvedPath.startsWith(coversDir + path.sep) && fs.existsSync(resolvedPath)) { - fs.unlinkSync(resolvedPath); - } + // cover_image is client-supplied, so treat it as untrusted: covers are flat + // filenames in the 'covers' category — basename() confines the delete to + // it, and central key validation rejects anything hostile (swallowed like + // the old containment guard; an external https URL is likewise a no-op). + await this.storage.delete('covers', path.basename(coverImage)).catch(() => { + /* external URL or already gone */ + }); } updateCoverImage(tripId: string | number, coverUrl: string): void { diff --git a/server/src/nest/unsplash/unsplash.module.ts b/server/src/nest/unsplash/unsplash.module.ts index 3e73f7034..0fa967257 100644 --- a/server/src/nest/unsplash/unsplash.module.ts +++ b/server/src/nest/unsplash/unsplash.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { UnsplashService } from './unsplash.service'; import { AppConfigModule } from '../app-config/app-config.module'; +import { StorageModule } from '../storage/storage.module'; /** Unsplash cover search and download. No controller of its own — trips and * places both reach it. Deliberately NOT @Global, matching PermissionsModule: @@ -8,7 +9,7 @@ import { AppConfigModule } from '../app-config/app-config.module'; * AppConfigModule is imported explicitly because @Global only applies to * modules that are in the graph, which a single-domain TestingModule is not. */ @Module({ - imports: [AppConfigModule], + imports: [AppConfigModule, StorageModule], providers: [UnsplashService], exports: [UnsplashService], }) diff --git a/server/src/nest/unsplash/unsplash.service.ts b/server/src/nest/unsplash/unsplash.service.ts index 73df46489..f3426c1eb 100644 --- a/server/src/nest/unsplash/unsplash.service.ts +++ b/server/src/nest/unsplash/unsplash.service.ts @@ -1,11 +1,11 @@ -import path from 'path'; -import fs from 'fs'; +import { Readable } from 'node:stream'; import { v4 as uuidv4 } from 'uuid'; import { Injectable } from '@nestjs/common'; import { safeFetch } from '../../utils/ssrfGuard'; import { resolveApiKey } from '../settings/instance-api-keys'; import { DatabaseService } from '../database/database.service'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { StorageService } from '../storage/storage.service'; interface UnsplashSearchResponse { results?: { @@ -50,6 +50,7 @@ export class UnsplashService { constructor( private readonly db: DatabaseService, private readonly env: RuntimeEnvService, + private readonly storage: StorageService, ) {} /** @@ -134,13 +135,14 @@ export class UnsplashService { } /** - * Download a chosen Unsplash cover from its CDN into destDir so the cover is - * stored locally (offline + CDN link-rot safe) instead of hot-linked. Only the - * Unsplash image CDN host is accepted, and the request goes through the SSRF - * guard. Returns the saved filename. Throws on a non-Unsplash host, a failed - * download, an unsupported content type, or an oversized image. + * Download a chosen Unsplash cover from its CDN into the 'covers' storage + * category so the cover is stored locally (offline + CDN link-rot safe) + * instead of hot-linked. Only the Unsplash image CDN host is accepted, and the + * request goes through the SSRF guard. Returns the saved filename. Throws on a + * non-Unsplash host, a failed download, an unsupported content type, or an + * oversized image — validation runs before any write. */ - async saveUnsplashCover(url: string, destDir: string): Promise { + async saveUnsplashCover(url: string): Promise { if (!this.isUnsplashCoverUrl(url)) throw new Error('Not an Unsplash image URL'); const res = await safeFetch(url); if (!res.ok) throw new Error(`Unsplash image download failed (HTTP ${res.status})`); @@ -149,9 +151,8 @@ export class UnsplashService { if (!ext) throw new Error(`Unsupported cover image type: ${type || 'unknown'}`); const buf = Buffer.from(await res.arrayBuffer()); if (buf.byteLength > MAX_COVER_BYTES) throw new Error('Cover image too large'); - if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true }); const filename = `${uuidv4()}${ext}`; - fs.writeFileSync(path.join(destDir, filename), buf); + await this.storage.put('covers', filename, Readable.from(buf)); return filename; } } diff --git a/server/tests/e2e/assignments.e2e.test.ts b/server/tests/e2e/assignments.e2e.test.ts index e6b5c1ba2..4a4976743 100644 --- a/server/tests/e2e/assignments.e2e.test.ts +++ b/server/tests/e2e/assignments.e2e.test.ts @@ -38,6 +38,9 @@ const { db } = vi.hoisted(() => { UNIQUE(assignment_id, user_id));`); tmp.exec(`CREATE TABLE tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, color TEXT, created_at TEXT);`); tmp.exec(`CREATE TABLE place_tags (place_id INTEGER NOT NULL, tag_id INTEGER NOT NULL);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/backup.e2e.test.ts b/server/tests/e2e/backup.e2e.test.ts index c6010c92d..16446d2c1 100644 --- a/server/tests/e2e/backup.e2e.test.ts +++ b/server/tests/e2e/backup.e2e.test.ts @@ -23,6 +23,9 @@ const { db } = vi.hoisted(() => { tmp.exec(`CREATE TABLE audit_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, user_id INTEGER, action TEXT NOT NULL, resource TEXT, details TEXT, ip TEXT);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); @@ -33,10 +36,10 @@ vi.mock('../../src/nest/audit/audit-log.logger', () => ({ LOG_LEVEL: 'error', lo const { backupSvc } = vi.hoisted(() => ({ backupSvc: { - listBackups: vi.fn(), createBackup: vi.fn(), restoreFromZip: vi.fn(), parseAutoBackupBody: vi.fn(), - deleteBackup: vi.fn(), isValidBackupFilename: vi.fn(), backupFilePath: vi.fn(), - backupFileExists: vi.fn(), checkRateLimit: vi.fn(), getUploadTmpDir: () => '/tmp', BACKUP_RATE_WINDOW: 3600000, - MAX_BACKUP_UPLOAD_SIZE: 1024, + listBackups: vi.fn(), createBackup: vi.fn(), restoreFromZip: vi.fn(), restoreBackup: vi.fn(), + parseAutoBackupBody: vi.fn(), deleteBackup: vi.fn(), isValidBackupFilename: vi.fn(), + backupFileExists: vi.fn(), sendBackupToResponse: vi.fn(), checkRateLimit: vi.fn(), + BACKUP_RATE_WINDOW: 3600000, MAX_BACKUP_UPLOAD_SIZE: 1024, }, })); vi.mock('../../src/nest/backup/backup.impl', () => backupSvc); diff --git a/server/tests/e2e/collab.e2e.test.ts b/server/tests/e2e/collab.e2e.test.ts index d5cd3e40c..067cb5130 100644 --- a/server/tests/e2e/collab.e2e.test.ts +++ b/server/tests/e2e/collab.e2e.test.ts @@ -50,6 +50,9 @@ const { db } = vi.hoisted(() => { tmp.exec(`CREATE TABLE trip_files (id INTEGER PRIMARY KEY AUTOINCREMENT, trip_id INTEGER NOT NULL, note_id INTEGER, filename TEXT NOT NULL, original_name TEXT NOT NULL, file_size INTEGER, mime_type TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/days.e2e.test.ts b/server/tests/e2e/days.e2e.test.ts index eb196377a..ed1e6d7c5 100644 --- a/server/tests/e2e/days.e2e.test.ts +++ b/server/tests/e2e/days.e2e.test.ts @@ -54,6 +54,9 @@ const { db } = vi.hoisted(() => { metadata TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP);`); tmp.exec(`CREATE TABLE reservation_endpoints (id INTEGER PRIMARY KEY AUTOINCREMENT, reservation_id INTEGER NOT NULL, local_date TEXT);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/feeds.e2e.test.ts b/server/tests/e2e/feeds.e2e.test.ts index 7b35ad7c6..219706150 100644 --- a/server/tests/e2e/feeds.e2e.test.ts +++ b/server/tests/e2e/feeds.e2e.test.ts @@ -31,6 +31,9 @@ const { db } = vi.hoisted(() => { tmp.exec(`CREATE TABLE trips (id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, title TEXT, is_archived INTEGER NOT NULL DEFAULT 0, start_date TEXT, end_date TEXT, feed_token TEXT);`); tmp.exec(`CREATE TABLE trip_members (trip_id INTEGER NOT NULL, user_id INTEGER NOT NULL);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/files.e2e.test.ts b/server/tests/e2e/files.e2e.test.ts index 3afb7a2b0..7b9353a03 100644 --- a/server/tests/e2e/files.e2e.test.ts +++ b/server/tests/e2e/files.e2e.test.ts @@ -37,6 +37,9 @@ const { db } = vi.hoisted(() => { tmp.exec('CREATE TABLE places (id INTEGER PRIMARY KEY AUTOINCREMENT, trip_id INTEGER);'); tmp.exec('CREATE TABLE days (id INTEGER PRIMARY KEY AUTOINCREMENT, trip_id INTEGER);'); tmp.exec('CREATE TABLE day_assignments (id INTEGER PRIMARY KEY AUTOINCREMENT, day_id INTEGER);'); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/journey.e2e.test.ts b/server/tests/e2e/journey.e2e.test.ts index dba7633a1..dacb7a6f0 100644 --- a/server/tests/e2e/journey.e2e.test.ts +++ b/server/tests/e2e/journey.e2e.test.ts @@ -20,6 +20,9 @@ const { db } = vi.hoisted(() => { tmp.exec('PRAGMA journal_mode = WAL'); tmp.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, email TEXT NOT NULL UNIQUE, role TEXT NOT NULL DEFAULT 'user', password_version INTEGER NOT NULL DEFAULT 0);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/memories.e2e.test.ts b/server/tests/e2e/memories.e2e.test.ts index 3822789d0..6d2422482 100644 --- a/server/tests/e2e/memories.e2e.test.ts +++ b/server/tests/e2e/memories.e2e.test.ts @@ -23,6 +23,9 @@ const { db } = vi.hoisted(() => { tmp.exec('PRAGMA journal_mode = WAL'); tmp.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, email TEXT NOT NULL UNIQUE, role TEXT NOT NULL DEFAULT 'user', password_version INTEGER NOT NULL DEFAULT 0);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/notifications.e2e.test.ts b/server/tests/e2e/notifications.e2e.test.ts index 13f13a3ce..e0724ea30 100644 --- a/server/tests/e2e/notifications.e2e.test.ts +++ b/server/tests/e2e/notifications.e2e.test.ts @@ -44,6 +44,9 @@ const { db } = vi.hoisted(() => { is_read INTEGER DEFAULT 0, created_at DATETIME DEFAULT CURRENT_TIMESTAMP );`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/oidc.e2e.test.ts b/server/tests/e2e/oidc.e2e.test.ts index 8b8720c49..df91f24f5 100644 --- a/server/tests/e2e/oidc.e2e.test.ts +++ b/server/tests/e2e/oidc.e2e.test.ts @@ -22,11 +22,15 @@ vi.mock('../../src/app-config', async (importOriginal) => { const { db } = vi.hoisted(() => { // eslint-disable-next-line @typescript-eslint/no-require-imports const Database = require('better-sqlite3'); - // Bare :memory: db — nothing in these flows queries it (the db-touching - // OidcService methods and the toggles are stubbed on the container - // instances below), the container just needs a connection to construct the - // oidc/auth/permissions/atlas providers. - return { db: new Database(':memory:') }; + const tmp = new Database(':memory:'); + // Otherwise-bare :memory: db — nothing in these flows queries it (the + // db-touching OidcService methods and the toggles are stubbed on the + // container instances below), the container just needs a connection to + // construct the oidc/auth/permissions/atlas providers. app_settings is the + // one real exception: StorageRegistryService (behind AuthModule → + // StorageModule) reads it at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); + return { db: tmp }; }); vi.mock('../../src/db/database', () => ({ db, diff --git a/server/tests/e2e/packing.e2e.test.ts b/server/tests/e2e/packing.e2e.test.ts index 72ce206f4..1f9539438 100644 --- a/server/tests/e2e/packing.e2e.test.ts +++ b/server/tests/e2e/packing.e2e.test.ts @@ -91,6 +91,9 @@ const { db } = vi.hoisted(() => { name TEXT NOT NULL, sort_order INTEGER NOT NULL DEFAULT 0 );`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/place-enrichment.e2e.test.ts b/server/tests/e2e/place-enrichment.e2e.test.ts index c88dd2f9d..a23305f44 100644 --- a/server/tests/e2e/place-enrichment.e2e.test.ts +++ b/server/tests/e2e/place-enrichment.e2e.test.ts @@ -190,8 +190,8 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => it('200 with the stored credit for a cached picture', async () => { const photoCache = app.get(PlacePhotoCacheService); - vi.spyOn(photoCache, 'get').mockReturnValueOnce({ - photoUrl: '/x', filePath: '/tmp/x', attribution: 'Alice · CC BY-SA 4.0', + vi.spyOn(photoCache, 'get').mockResolvedValueOnce({ + photoUrl: '/x', attribution: 'Alice · CC BY-SA 4.0', }); const res = await request(server) diff --git a/server/tests/e2e/places.e2e.test.ts b/server/tests/e2e/places.e2e.test.ts index abaed821b..c7bafbe54 100644 --- a/server/tests/e2e/places.e2e.test.ts +++ b/server/tests/e2e/places.e2e.test.ts @@ -50,6 +50,9 @@ const { db } = vi.hoisted(() => { // Deleting a place takes its linked expense with it (#1298). tmp.exec(`CREATE TABLE budget_items (id INTEGER PRIMARY KEY AUTOINCREMENT, trip_id INTEGER NOT NULL, name TEXT, total_price REAL DEFAULT 0, place_id INTEGER, reservation_id INTEGER);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/share.e2e.test.ts b/server/tests/e2e/share.e2e.test.ts index 4131bfc33..09ffd59ac 100644 --- a/server/tests/e2e/share.e2e.test.ts +++ b/server/tests/e2e/share.e2e.test.ts @@ -46,9 +46,9 @@ import { PermissionsService } from '../../src/nest/permissions/permissions.servi let checkPermission: MockInstance; // Overridden in the container rather than path-mocked: the cache is a provider now. -const serveFilePath = vi.fn(); +// serveKey hands the controller a bare photos-google storage name (slice 3). +const serveKey = vi.fn(); -import os from 'node:os'; import path from 'node:path'; import fs from 'node:fs'; import { createTables } from '../../src/db/schema'; @@ -66,7 +66,7 @@ describe('Share-link e2e (real auth guard + real SQL over temp SQLite)', () => { async function build() { const moduleRef = await Test.createTestingModule({ imports: [DatabaseModule, ShareModule] }) - .overrideProvider(PlacePhotoCacheService).useValue({ serveFilePath }) + .overrideProvider(PlacePhotoCacheService).useValue({ serveKey }) .compile(); const nest = moduleRef.createNestApplication(); nest.use(cookieParser()); @@ -97,7 +97,7 @@ describe('Share-link e2e (real auth guard + real SQL over temp SQLite)', () => { db.exec('DELETE FROM trips'); tripId = Number(db.prepare('INSERT INTO trips (user_id, title) VALUES (1, ?)').run('Trip').lastInsertRowid); checkPermission.mockReturnValue(true); - serveFilePath.mockReset(); + serveKey.mockReset(); }); afterAll(async () => { @@ -166,29 +166,36 @@ describe('Share-link e2e (real auth guard + real SQL over temp SQLite)', () => { }); describe('public place-photo proxy (/api/shared/:token/place-photo/:placeId/bytes)', () => { - const photoFile = path.join(os.tmpdir(), 'trek-share-photo.e2e.jpg'); + // The controller streams (category 'photos-google', name) through the real + // storage facade, so the fixture has to live in the real photos-google root. + const photoName = 'trek-share-photo.e2e.jpg'; + const photoFile = path.join(__dirname, '../../uploads/photos/google', photoName); const photoBytes = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]); // JPEG-ish header - beforeAll(() => fs.writeFileSync(photoFile, photoBytes)); + beforeAll(() => { + fs.mkdirSync(path.dirname(photoFile), { recursive: true }); + fs.writeFileSync(photoFile, photoBytes); + }); afterAll(() => { try { fs.unlinkSync(photoFile); } catch { /* ignore */ } }); it('streams cached bytes with no cookie (unguarded) for a valid token + place', async () => { const created = await request(server).post(`/api/trips/${tripId}/share-link`).set('Cookie', sessionCookie(1)).send({}); db.prepare('INSERT INTO places (trip_id, name, image_url) VALUES (?, ?, ?)') .run(tripId, 'Louvre', '/api/maps/place-photo/ChIJabc/bytes'); - serveFilePath.mockReturnValueOnce(photoFile); + serveKey.mockReturnValueOnce(photoName); const res = await request(server).get(`/api/shared/${created.body.token}/place-photo/ChIJabc/bytes`); expect(res.status).toBe(200); expect(res.headers['content-type']).toContain('image/jpeg'); expect(res.headers['cache-control']).toContain('immutable'); expect(Buffer.from(res.body)).toEqual(photoBytes); - expect(serveFilePath).toHaveBeenCalledWith('ChIJabc'); + expect(serveKey).toHaveBeenCalledWith('ChIJabc'); }); - it('404 when the token/place does not resolve to a cached photo', async () => { + it('answers an empty 204 when the token/place does not resolve to a cached photo', async () => { const res = await request(server).get('/api/shared/bad/place-photo/ChIJabc/bytes'); - expect(res.status).toBe(404); - expect(res.body).toEqual({ error: 'Photo not cached' }); + expect(res.status).toBe(204); + expect(res.headers['cache-control']).toBe('no-store'); + expect(res.text ?? '').toBe(''); }); }); }); diff --git a/server/tests/e2e/storage-admin.e2e.test.ts b/server/tests/e2e/storage-admin.e2e.test.ts new file mode 100644 index 000000000..03a82af78 --- /dev/null +++ b/server/tests/e2e/storage-admin.e2e.test.ts @@ -0,0 +1,383 @@ +/** + * Storage admin e2e — exercises the migrated /api/admin/storage surface + * (StorageAdminController) through the real JwtAuthGuard + AdminGuard + + * ManagedGuard against a temp SQLite db. DI-native: no service mock, so the + * registry's real boot/seed/reload pipeline and StorageAdminService run for + * real. Covers auth (401), the admin gate (403), managed-mode refusal (403, + * the first e2e to assert it — ManagedGuard is otherwise only wired in + * AppModule), the GET/PUT/test happy paths, the 400 envelope for both a + * semantic registry refusal and a Zod pipe rejection, and secret masking/ + * encryption/redaction (including without an explicit ENCRYPTION_KEY). + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import cookieParser from 'cookie-parser'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { Server } from 'http'; +import { APP_GUARD } from '@nestjs/core'; +import { DatabaseModule } from '../../src/nest/database/database.module'; +import { StorageModule } from '../../src/nest/storage/storage.module'; +import { ManagedGuard } from '../../src/nest/common/managed.guard'; +import { Test } from '@nestjs/testing'; +import { seedUser, sessionCookie } from './harness'; + +const { db } = vi.hoisted(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const Database = require('better-sqlite3'); + const tmp = new Database(':memory:'); + tmp.exec('PRAGMA journal_mode = WAL'); + // `users` carries the columns listUsers/createUser/updateUser select, plus the + // is_guest flag the #1362 COALESCE guards read (admin.e2e.test.ts DDL). + tmp.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, + email TEXT NOT NULL UNIQUE, role TEXT NOT NULL DEFAULT 'user', password_version INTEGER NOT NULL DEFAULT 0, + password_hash TEXT, avatar TEXT, is_guest INTEGER DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_login DATETIME);`); + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); + // Slim audit_log mirror (no FKs), same shape as admin.e2e.test.ts. + tmp.exec(`CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + user_id INTEGER, action TEXT NOT NULL, resource TEXT, details TEXT, ip TEXT);`); + return { db: tmp }; +}); + +vi.mock('../../src/db/database', () => ({ db, closeDb: () => {}, reinitialize: () => {} })); +// The audit domain is DI-native: writeAudit runs for real against the temp db's +// audit_log table; only the file logger is silenced. +vi.mock('../../src/nest/audit/audit-log.logger', () => ({ LOG_LEVEL: 'error', logInfo: vi.fn(), logDebug: vi.fn(), logError: vi.fn(), logWarn: vi.fn() })); +// apiKeyCrypto imports ENCRYPTION_KEY from here for the cipher; the raw +// process.env var is managed separately in beforeAll/afterAll for the +// no-explicit-key case (STORE2E-007). JWT_SECRET must also be +// supplied — jwt-verify.ts (JwtAuthGuard) and the harness's signSession both +// import it from this same module, so mocking the module wholesale requires +// keeping both consistent. +vi.mock('../../src/config', () => ({ ENCRYPTION_KEY: 'e2e-storage-key', JWT_SECRET: 'e2e-storage-jwt-secret' })); + +import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; +import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; + +describe('Storage admin e2e (real auth + admin guard + managed guard + temp SQLite)', () => { + let server: Server; + let app: Awaited>; + let adminCookie: string; + let userCookie: string; + + async function build() { + const moduleRef = await Test.createTestingModule({ + imports: [DatabaseModule, StorageModule], + providers: [{ provide: APP_GUARD, useClass: ManagedGuard }], + }).compile(); + const nest = moduleRef.createNestApplication(); + nest.use(cookieParser()); + nest.useGlobalPipes(new ZodValidationPipe()); + nest.useGlobalFilters(new TrekExceptionFilter()); + await nest.init(); + return nest; + } + + beforeAll(async () => { + process.env.ENCRYPTION_KEY = 'e2e-storage-key'; + seedUser(db as never, { id: 1 }); + seedUser(db as never, { id: 2, role: 'admin', email: 'e2e-storage-admin@example.test' }); + userCookie = sessionCookie(1); + adminCookie = sessionCookie(2); + app = await build(); + server = app.getHttpServer(); + }); + + // Clears the persisted rows only — NOT the registry's in-memory last-good + // snapshot (StorageRegistryService keeps state across requests within one + // `app`, and this suite builds `app` once in beforeAll). A test added after + // STORE2E-004 that expects to see fresh defaults must call GET (which + // renders the registry's live snapshot) rather than assume the deleted rows + // reset it — the registry only re-reads app_settings on `reload()`/init. + beforeEach(() => { + db.exec("DELETE FROM app_settings WHERE key LIKE 'storage.%'"); + db.exec('DELETE FROM audit_log'); + }); + + afterAll(async () => { + await app.close(); + delete process.env.ENCRYPTION_KEY; + }); + + it('STORE2E-001 401 without a session', async () => { + expect((await request(server).get('/api/admin/storage')).status).toBe(401); + // PUT stays behind the same guard chain — pins that the guards are declared + // class-level (JwtAuthGuard + AdminGuard on the controller), not re-added + // per handler where a future handler could forget them. + expect((await request(server).put('/api/admin/storage').send({ backends: [], categories: {} })).status).toBe(401); + expect( + (await request(server).post('/api/admin/storage/test').send({ backend: { name: 'x', type: 'local', options: { root: '/tmp' } } })).status, + ).toBe(401); + }); + + it('STORE2E-002 403 for a non-admin', async () => { + const res = await request(server).get('/api/admin/storage').set('Cookie', userCookie); + expect(res.status).toBe(403); + expect(res.body).toEqual({ error: 'Admin access required' }); + }); + + it('STORE2E-003 GET renders the effective defaults world', async () => { + const res = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(res.status).toBe(200); + const names = (res.body.backends as Array<{ name: string; source: string }>).map((b) => [b.name, b.source]); + expect(names).toEqual(expect.arrayContaining([['uploads-local', 'built-in'], ['backups-local', 'built-in']])); + expect(Object.keys(res.body.categories)).toHaveLength(8); + expect(res.body.seedFilePresent).toBe(false); + expect(res.body.health).toEqual({ replicaFailures: [] }); + }); + + it('STORE2E-004 PUT persists, answers the fresh world, audits with redacted secrets', async () => { + const nasRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-nas-')); + const body = { + backends: [ + { name: 'nas-backups', type: 'local', options: { root: nasRoot } }, + { + name: 'off-box', + type: 's3', + options: { endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: 'sk-e2e' }, + }, + ], + categories: { backups: 'nas-backups' }, + version: 0, + }; + const res = await request(server).put('/api/admin/storage').set('Cookie', adminCookie).send(body); + expect(res.status).toBe(200); + const offBox = (res.body.backends as Array<{ name: string; source: string; options: Record }>).find( + (b) => b.name === 'off-box', + )!; + expect(offBox.source).toBe('settings'); + expect(offBox.options.secretAccessKey).toBe('••••••••'); // masked, never echoed + expect(res.body.categories.backups).toEqual({ backend: 'nas-backups', source: 'settings' }); + + const row = db.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as { value: string }; + expect(row.value).toContain('enc:v1:'); + expect(row.value).not.toContain('sk-e2e'); + + const audit = db.prepare("SELECT details FROM audit_log WHERE action = 'admin.storage_update'").get() as { details: string }; + expect(audit.details).toContain('***'); + expect(audit.details).not.toContain('sk-e2e'); + }); + + it('STORE2E-005 PUT with a semantic violation → 400 with the registry message verbatim', async () => { + const res = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ backends: [], categories: { backups: 'nope' }, version: 0 }); + expect(res.status).toBe(400); + expect(res.body).toEqual({ error: "category 'backups' maps to unknown backend 'nope'" }); + }); + + it('STORE2E-006 PUT with an unknown top-level key → 400 from the Zod pipe (reserved readOnly door)', async () => { + const res = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ backends: [], categories: {}, version: 0, readOnly: true }); + expect(res.status).toBe(400); + expect(res.body.error).toBeDefined(); + }); + + it('STORE2E-007 PUT with a plaintext secret and no explicit ENCRYPTION_KEY saves, encrypted at rest', async () => { + delete process.env.ENCRYPTION_KEY; + try { + const res = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ + backends: [ + { + name: 'off-box', + type: 's3', + options: { endpoint: 'http://127.0.0.1:9000', bucket: 'trek', accessKeyId: 'ak', secretAccessKey: 'sk' }, + }, + ], + categories: {}, + version: 0, + }); + expect(res.status).toBe(200); + // The secret never comes back and never persists in the clear. + const offBox = (res.body.backends as Array<{ name: string; options: Record }>).find( + (b) => b.name === 'off-box', + )!; + expect(offBox.options.secretAccessKey).not.toBe('sk'); + const row = db.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as { value: string }; + expect(row.value).not.toContain('"sk"'); + expect(row.value).toContain('enc:v1:'); + } finally { + process.env.ENCRYPTION_KEY = 'e2e-storage-key'; + } + }); + + it('STORE2E-008 managed mode refuses the whole surface with the standard body', async () => { + process.env.TREK_MANAGED = 'true'; + try { + const res = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(res.status).toBe(403); + expect(res.body).toEqual({ + error: 'This is configured by the operator of this instance.', + code: 'MANAGED_FORBIDDEN', + }); + } finally { + delete process.env.TREK_MANAGED; + } + }); + + it('STORE2E-009 POST /test probes a local candidate and answers 200 with per-target results', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-probe-')); + const res = await request(server) + .post('/api/admin/storage/test') + .set('Cookie', adminCookie) + .send({ backend: { name: 'cand', type: 'local', options: { root } } }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ ok: true, targets: [{ name: 'cand', ok: true }] }); + const audit = db.prepare("SELECT details FROM audit_log WHERE action = 'admin.storage_test'").get() as { details: string }; + expect(JSON.parse(audit.details)).toMatchObject({ backend: 'cand', type: 'local', ok: true }); + }); + + it('STORE2E-010 migration moves a category end to end', async () => { + const destRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-migration-dest-')); + const put = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ backends: [{ name: 'dest', type: 'local', options: { root: destRoot } }], categories: {}, version: 0 }); + expect(put.status).toBe(200); + + // 'journey' is empty (no objects planted), so the migration is valid and instant. + const start = await request(server) + .post('/api/admin/storage/migrations') + .set('Cookie', adminCookie) + .send({ category: 'journey', to: 'dest' }); + expect(start.status).toBe(200); + expect(start.body).toEqual({ started: true }); + + let status: { status: string } | undefined; + let stateBody: { categories: Record } | undefined; + for (let i = 0; i < 50; i++) { + const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + stateBody = state.body as typeof stateBody; + status = (state.body.migrations as Array<{ category: string; status: string }>).find( + (m) => m.category === 'journey', + ); + if (status && status.status !== 'running') break; + await new Promise((r) => setTimeout(r, 100)); + } + expect(status).toMatchObject({ status: 'done' }); + expect(stateBody!.categories.journey).toEqual({ backend: 'dest', source: 'settings' }); + const audit = db.prepare("SELECT details FROM audit_log WHERE action = 'admin.storage_migration'").get() as { details: string }; + expect(JSON.parse(audit.details)).toEqual({ category: 'journey', to: 'dest' }); + }); + + it('STORE2E-011 backfill guards: 401 anon, 404 non-mirror, 409 while running is covered by unit — here the 404', async () => { + expect((await request(server).post('/api/admin/storage/backends/x/backfill')).status).toBe(401); + const res = await request(server).post('/api/admin/storage/backends/uploads-local/backfill').set('Cookie', adminCookie); + expect(res.status).toBe(404); + expect(res.body.error).toContain('not a mirror'); + }); + + it('STORE2E-012 backfill happy path copies pre-existing objects and surfaces in state until done', async () => { + // Route backups through a mirror with a local replica, then plant a + // pre-mirror object directly on the primary. + const nasRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-nas-')); + // Override backups-local to a tmp root: WITHOUT this the built-in default + // is the repo's real server/data/backups directory — never write there. + const backupsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-backups-')); + const put = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ + backends: [ + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + { name: 'nas', type: 'local', options: { root: nasRoot } }, + { name: 'm', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ], + categories: { backups: 'm' }, + version: 0, + }); + expect(put.status).toBe(200); + fs.writeFileSync(path.join(backupsRoot, 'pre-mirror.zip'), 'oldbytes'); + + const start = await request(server).post('/api/admin/storage/backends/m/backfill').set('Cookie', adminCookie); + expect(start.status).toBe(200); + expect(start.body).toEqual({ started: true }); + + // Poll state until the job finishes (tiny local copy — a few ticks). + let status: { status: string } | undefined; + for (let i = 0; i < 50; i++) { + const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + status = (state.body.backfills as Array<{ backend: string; status: string }>).find((b) => b.backend === 'm'); + if (status && status.status !== 'running') break; + await new Promise((r) => setTimeout(r, 100)); + } + expect(status).toMatchObject({ status: 'done' }); + expect(fs.existsSync(path.join(nasRoot, 'pre-mirror.zip'))).toBe(true); + const audit = db.prepare("SELECT details FROM audit_log WHERE action = 'admin.storage_backfill'").get() as { details: string }; + expect(JSON.parse(audit.details)).toMatchObject({ backend: 'm' }); + }); + + it('STORE2E-013 cancel 404s with no active run; stats refresh returns real numbers and audits', async () => { + expect((await request(server).delete('/api/admin/storage/backends/m/backfill').set('Cookie', adminCookie)).status).toBe(404); + const res = await request(server).post('/api/admin/storage/stats/refresh').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(res.body.computedAt).toBeGreaterThan(0); + expect(res.body.categories.backups.objects).toBeGreaterThanOrEqual(1); // pre-mirror.zip at least + const audit = db.prepare("SELECT COUNT(*) AS n FROM audit_log WHERE action = 'admin.storage_stats_refresh'").get() as { n: number }; + expect(audit.n).toBe(1); + }); + + it('STORE2E-014 regression (audit #7): a migration flip while the admin form is open makes the stale save 409, and the flip survives', async () => { + const destRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-e2e-migration-dest2-')); + const put = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ backends: [{ name: 'dest', type: 'local', options: { root: destRoot } }], categories: {}, version: 0 }); + expect(put.status).toBe(200); + + // The admin opens the form: their draft is built at THIS version, before + // the migration below flips anything. + const loaded = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + const staleVersion = loaded.body.version as number; + expect(staleVersion).toBe(1); + expect(loaded.body.categories.journey).toEqual({ backend: 'uploads-local', source: 'default' }); + + // Meanwhile: a category migration flips 'journey' to 'dest' (empty + // category, so it completes instantly) — the same write path + // StorageRegistryService.assignCategory uses, bumping the version. + const start = await request(server) + .post('/api/admin/storage/migrations') + .set('Cookie', adminCookie) + .send({ category: 'journey', to: 'dest' }); + expect(start.status).toBe(200); + let status: { status: string } | undefined; + for (let i = 0; i < 50; i++) { + const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + status = (state.body.migrations as Array<{ category: string; status: string }>).find( + (m) => m.category === 'journey', + ); + if (status && status.status !== 'running') break; + await new Promise((r) => setTimeout(r, 100)); + } + expect(status).toMatchObject({ status: 'done' }); + + // The admin's still-open form — unaware of the flip, and carrying no + // opinion of its own on 'journey' — submits its (now stale) save. + const staleSave = await request(server) + .put('/api/admin/storage') + .set('Cookie', adminCookie) + .send({ + backends: [{ name: 'dest', type: 'local', options: { root: destRoot } }], + categories: {}, + version: staleVersion, + }); + expect(staleSave.status).toBe(409); + expect(staleSave.body.error).toContain('storage settings changed since this form was loaded'); + + // The flip survives — the stale save never silently reverted 'journey' + // back to its old default. + const after = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(after.body.categories.journey).toEqual({ backend: 'dest', source: 'settings' }); + expect(after.body.version).toBeGreaterThan(staleVersion); + }); +}); diff --git a/server/tests/e2e/tags.e2e.test.ts b/server/tests/e2e/tags.e2e.test.ts index 4eed950aa..51b9512ab 100644 --- a/server/tests/e2e/tags.e2e.test.ts +++ b/server/tests/e2e/tags.e2e.test.ts @@ -26,6 +26,9 @@ const { db } = vi.hoisted(() => { color TEXT DEFAULT '#10b981', created_at DATETIME DEFAULT CURRENT_TIMESTAMP );`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/todo.e2e.test.ts b/server/tests/e2e/todo.e2e.test.ts index 2bddd5456..585125d11 100644 --- a/server/tests/e2e/todo.e2e.test.ts +++ b/server/tests/e2e/todo.e2e.test.ts @@ -43,6 +43,9 @@ const { db } = vi.hoisted(() => { user_id INTEGER NOT NULL, UNIQUE(trip_id, category_name, user_id) );`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/transit.e2e.test.ts b/server/tests/e2e/transit.e2e.test.ts index db397dce6..709cd8bc2 100644 --- a/server/tests/e2e/transit.e2e.test.ts +++ b/server/tests/e2e/transit.e2e.test.ts @@ -18,6 +18,9 @@ const { db } = vi.hoisted(() => { tmp.exec('PRAGMA journal_mode = WAL'); tmp.exec(`CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL, email TEXT NOT NULL UNIQUE, role TEXT NOT NULL DEFAULT 'user', password_version INTEGER NOT NULL DEFAULT 0);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); vi.mock('../../src/db/database', () => ({ db, closeDb: () => {}, reinitialize: () => {} })); diff --git a/server/tests/e2e/trips.e2e.test.ts b/server/tests/e2e/trips.e2e.test.ts index f9a04e9ce..504f8857c 100644 --- a/server/tests/e2e/trips.e2e.test.ts +++ b/server/tests/e2e/trips.e2e.test.ts @@ -96,6 +96,9 @@ const { db } = vi.hoisted(() => { tmp.exec(`CREATE TABLE audit_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, user_id INTEGER, action TEXT NOT NULL, resource TEXT, details TEXT, ip TEXT);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/e2e/vacay.e2e.test.ts b/server/tests/e2e/vacay.e2e.test.ts index 0b5fdff96..acddcf551 100644 --- a/server/tests/e2e/vacay.e2e.test.ts +++ b/server/tests/e2e/vacay.e2e.test.ts @@ -64,6 +64,9 @@ const { db } = vi.hoisted(() => { user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, year_type TEXT NOT NULL DEFAULT 'calendar', year_start_month INTEGER NOT NULL DEFAULT 1, year_start_day INTEGER NOT NULL DEFAULT 1, hire_date TEXT);`); + // StorageRegistryService (behind StorageModule, now in this module chain) reads + // this at onModuleInit. + tmp.exec('CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT);'); return { db: tmp }; }); diff --git a/server/tests/helpers/mcp-test-controllers.ts b/server/tests/helpers/mcp-test-controllers.ts index ba4d6434e..a0bad3369 100644 --- a/server/tests/helpers/mcp-test-controllers.ts +++ b/server/tests/helpers/mcp-test-controllers.ts @@ -74,6 +74,8 @@ import { makeNotificationsService, makeNotificationPreferencesService } from './ import { AddonsService } from '../../src/nest/addons/addons.service'; import { notificationsStub } from './notifications'; import { EphemeralTokenService } from '../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../src/nest/files/allowed-file-types.service'; +import { makeStorageFixture } from './storage-fixture'; /** * Hand-wired counterpart of the boot-time discovery in McpRegistryService, @@ -84,6 +86,7 @@ import { EphemeralTokenService } from '../../src/nest/auth/ephemeral-token.servi */ export function createMcpTestRegistry(): McpRegistry { const dbService = new DatabaseService(db); + const generalStorage = makeStorageFixture('').storage; const permissionsService = new PermissionsService(dbService); // Same argument list as auth.bridge.ts. AtlasService used to sit in third // place; when getTravelStats moved onto AtlasService itself the edge was @@ -102,15 +105,16 @@ export function createMcpTestRegistry(): McpRegistry { new UserCleanupService(dbService, budgetService), new MailerService(dbService), new EphemeralTokenService(), + new AllowedFileTypesService(dbService), ); const queryHelpersService = new QueryHelpersService(dbService); const daysService = new DaysService(dbService, permissionsService, realtimeService, queryHelpersService); const todoService = new TodoService(dbService, permissionsService, realtimeService); const packingService = new PackingService(dbService, permissionsService, realtimeService, notificationsStub()); - const collabService = new CollabService(dbService, permissionsService, realtimeService, notificationsStub()); + const collabService = new CollabService(dbService, permissionsService, realtimeService, notificationsStub(), generalStorage); // Exactly one instance, shared by maps, places and share: its stampede guard // and its on-disk set only work if all three readers see the same maps. - const placePhotoCache = new PlacePhotoCacheService(dbService, new RuntimeEnvService()); + const placePhotoCache = new PlacePhotoCacheService(dbService, makeStorageFixture('photos/google/').storage); const mapsService = new MapsService(dbService, placePhotoCache); const journeyDomain = new JourneyDomainService(dbService, realtimeService, new TrekPhotosRepository(dbService)); // The last three were previously omitted, which left them `undefined` at @@ -119,9 +123,10 @@ export function createMcpTestRegistry(): McpRegistry { // includes `src`, so nothing here is typechecked; pass them for real. const placesService = new PlacesService( dbService, permissionsService, realtimeService, mapsService, queryHelpersService, - new UnsplashService(dbService, new RuntimeEnvService()), + new UnsplashService(dbService, new RuntimeEnvService(), generalStorage), placePhotoCache, journeyDomain, + generalStorage, ); const reservationsService = new ReservationsService(dbService, permissionsService, budgetService, realtimeService, notificationsStub(), new ReservationsReadRepository(dbService)); const accommodationsService = new AccommodationsService(dbService, permissionsService, realtimeService); @@ -134,12 +139,13 @@ export function createMcpTestRegistry(): McpRegistry { budgetService, new VacayService(dbService, realtimeService, notificationsStub()), realtimeService, - new UnsplashService(dbService, new RuntimeEnvService()), + new UnsplashService(dbService, new RuntimeEnvService(), generalStorage), + generalStorage, ); const readModelService = new TripReadModelService( dbService, membersService, daysService, accommodationsService, budgetService, packingService, reservationsService, collabService, placesService, todoService, - new FilesService(dbService, permissionsService, realtimeService, new EphemeralTokenService()), + new FilesService(dbService, permissionsService, realtimeService, new EphemeralTokenService(), generalStorage), ); const calendarService = new CalendarService(dbService, reservationsService); // The nine addon-gated surfaces read their toggle off an injected service now @@ -172,7 +178,7 @@ export function createMcpTestRegistry(): McpRegistry { new ShareMcp(new ShareService(dbService, new SettingsService(dbService), permissionsService, queryHelpersService, placePhotoCache), authService, guards), new MapsMcp(mapsService), new PlacesMcp(placesService, mapsService, dbService, authService, journeyDomain, assignmentsService, guards), - new CollectionsMcp(new CollectionsService(dbService, permissionsService, realtimeService, notificationsStub()), dbService, authService, addonsService), + new CollectionsMcp(new CollectionsService(dbService, permissionsService, realtimeService, notificationsStub(), generalStorage), dbService, authService, addonsService), new TransitMcp(new TransitService(), daysService, reservationsService, dbService, authService, guards), new AtlasMcp(new AtlasService(dbService), addonsService, authService), new JourneyMcp(journeyDomain, new JourneyShareService(dbService, journeyDomain), addonsService, authService), diff --git a/server/tests/helpers/plugin-host.ts b/server/tests/helpers/plugin-host.ts index 9cd78c18a..600a1da2e 100644 --- a/server/tests/helpers/plugin-host.ts +++ b/server/tests/helpers/plugin-host.ts @@ -70,6 +70,7 @@ import { UnsplashService } from '../../src/nest/unsplash/unsplash.service'; import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import { TrekPhotosRepository } from '../../src/nest/photos/trek-photos.repository'; import { RuntimeEnvService } from '../../src/nest/app-config/runtime-env.service'; +import { makeStorageFixture } from './storage-fixture'; /** * Hand-wired counterpart of the PluginsModule DI graph for no-Nest tests @@ -82,6 +83,7 @@ import { RuntimeEnvService } from '../../src/nest/app-config/runtime-env.service * container would discover, handed to the host factory as a registry. */ export function createPluginRpcHostFactory(dbs: DatabaseService): PluginRpcHostFactory { + const generalStorage = makeStorageFixture('').storage; const permissions = new PermissionsService(dbs); const exchangeRates = new ExchangeRatesService(); const realtime = new RealtimeService(); @@ -90,16 +92,16 @@ export function createPluginRpcHostFactory(dbs: DatabaseService): PluginRpcHostF const queryHelpers = new QueryHelpersService(dbs); const todos = new TodoService(dbs, permissions, realtime); const packing = new PackingService(dbs, permissions, realtime, notificationsStub()); - const files = new FilesService(dbs, permissions, realtime, new EphemeralTokenService()); + const files = new FilesService(dbs, permissions, realtime, new EphemeralTokenService(), generalStorage); const reservations = new ReservationsService(dbs, permissions, budget, realtime, notificationsStub(), new ReservationsReadRepository(dbs)); - const collab = new CollabService(dbs, permissions, realtime, notificationsStub()); + const collab = new CollabService(dbs, permissions, realtime, notificationsStub(), generalStorage); const vacay = new VacayService(dbs, realtime, notificationsStub()); const days = new DaysService(dbs, permissions, realtime, queryHelpers); - const photoCache = new PlacePhotoCacheService(dbs, new RuntimeEnvService()); - const unsplash = new UnsplashService(dbs, new RuntimeEnvService()); + const photoCache = new PlacePhotoCacheService(dbs, makeStorageFixture('photos/google/').storage); + const unsplash = new UnsplashService(dbs, new RuntimeEnvService(), generalStorage); const journey = new JourneyDomainService(dbs, realtime, new TrekPhotosRepository(dbs)); - const places = new PlacesService(dbs, permissions, realtime, new MapsService(dbs, photoCache), queryHelpers, unsplash, photoCache, journey); - const collections = new CollectionsService(dbs, permissions, realtime, notificationsStub()); + const places = new PlacesService(dbs, permissions, realtime, new MapsService(dbs, photoCache), queryHelpers, unsplash, photoCache, journey, generalStorage); + const collections = new CollectionsService(dbs, permissions, realtime, notificationsStub(), generalStorage); const atlas = new AtlasService(dbs); const dayNotes = new DayNotesService(dbs, permissions, realtime); const assignments = new AssignmentsService(dbs, permissions, realtime, queryHelpers, journey); @@ -108,7 +110,7 @@ export function createPluginRpcHostFactory(dbs: DatabaseService): PluginRpcHostF const llmConfig = new LlmConfigResolver(new SettingsService(dbs), dbs, addons); const oauth = new PluginOAuthService(dbs); const accommodations = new AccommodationsService(dbs, permissions, realtime); - const trips = new TripsService(dbs, reservations, days, permissions, budget, vacay, realtime, unsplash); + const trips = new TripsService(dbs, reservations, days, permissions, budget, vacay, realtime, unsplash, generalStorage); const members = new TripMembersService(dbs, budget, new UserCleanupService(dbs, budget), permissions, realtime, notificationsStub()); const guards = new PluginGuards(dbs, permissions, addons); @@ -120,7 +122,7 @@ export function createPluginRpcHostFactory(dbs: DatabaseService): PluginRpcHostF new TodoRpc(todos, realtime, guards), new DayNotesRpc(dayNotes, realtime, guards), new PackingRpc(packing, realtime, guards), - new FilesRpc(files, realtime, dbs, guards), + new FilesRpc(files, realtime, dbs, guards, generalStorage), new PlacesRpc(places, journey, realtime, guards), new DaysRpc(days, realtime, guards), new AccommodationsRpc(accommodations, realtime, guards), diff --git a/server/tests/helpers/rpc-host-deps.ts b/server/tests/helpers/rpc-host-deps.ts index d32590540..f62dbd24d 100644 --- a/server/tests/helpers/rpc-host-deps.ts +++ b/server/tests/helpers/rpc-host-deps.ts @@ -67,7 +67,7 @@ export function allRpcControllers(): object[] { new TodoRpc(anyService(), anyService(), anyService()), new DayNotesRpc(anyService(), anyService(), anyService()), new PackingRpc(anyService(), anyService(), anyService()), - new FilesRpc(anyService(), anyService(), anyService(), anyService()), + new FilesRpc(anyService(), anyService(), anyService(), anyService(), anyService()), new PlacesRpc(anyService(), anyService(), anyService(), anyService()), new DaysRpc(anyService(), anyService(), anyService()), new ItineraryRpc(anyService(), anyService(), anyService()), diff --git a/server/tests/helpers/storage-fixture.ts b/server/tests/helpers/storage-fixture.ts new file mode 100644 index 000000000..7123b4fe7 --- /dev/null +++ b/server/tests/helpers/storage-fixture.ts @@ -0,0 +1,41 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { LocalDriver } from '../../src/nest/storage/drivers/local.driver'; +import type { StorageRegistryService, ResolvedCategory } from '../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../src/nest/storage/storage.service'; + +/** + * A real StorageService facade over a stub registry + a REAL LocalDriver + * rooted in a throwaway mkdtemp dir (the storage.service.test.ts makeFixture + * pattern, shared). Every category resolves to the same driver + keyPrefix — + * exactly what a single-category service test needs, and the two + * TREK_PLACE_PHOTO_DIR modes are just two prefixes ('photos/google/' vs ''). + * The real mode flip lives in the registry and is pinned by its own tests. + */ +export interface StorageFixture { + storage: StorageService; + /** Driver root — objects land at root/. */ + root: string; + cleanup(): void; +} + +export function makeStorageFixture(keyPrefix: string): StorageFixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-fx-')); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-fx-tmp-')); + const driver = new LocalDriver({ id: 'fixture-local', root }); + driver.init({ cleanSpool: true }); + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix, backendName: 'fixture-local' }), + tempDir: () => tempDir, + replicaFailures: () => [], + } as unknown as StorageRegistryService; + return { + storage: new StorageService(registry), + root, + cleanup: () => { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(tempDir, { recursive: true, force: true }); + }, + }; +} diff --git a/server/tests/integration/backup.test.ts b/server/tests/integration/backup.test.ts index c462e79d2..3cd70dfbe 100644 --- a/server/tests/integration/backup.test.ts +++ b/server/tests/integration/backup.test.ts @@ -59,9 +59,9 @@ vi.mock('../../src/nest/backup/backup.impl', async () => { created_at: new Date().toISOString(), }), restoreFromZip: vi.fn().mockResolvedValue({ success: true }), + restoreBackup: vi.fn().mockResolvedValue({ success: true }), deleteBackup: vi.fn().mockReturnValue(undefined), backupFileExists: vi.fn().mockReturnValue(false), - backupFilePath: vi.fn().mockReturnValue('/tmp/test-backup.zip'), // Keep checkRateLimit from actual so rate-limit tests work correctly checkRateLimit: vi.fn().mockReturnValue(true), }; @@ -85,6 +85,7 @@ import { resetTestDb, resetRateLimits } from '../helpers/test-db'; import { createAdmin, createUser } from '../helpers/factories'; import { authCookie } from '../helpers/auth'; import * as backupService from '../../src/nest/backup/backup.impl'; +import { DEFAULT_BACKUPS_ROOT } from '../../src/nest/storage/storage-paths'; import fs from 'fs'; import path from 'path'; import os from 'os'; @@ -200,36 +201,41 @@ describe('Backup security', () => { // --------------------------------------------------------------------------- describe('Backup download', () => { - let tmpFile: string; + // Serving goes through the real StorageService now (sendBackupToResponse is + // NOT overridden in the impl mock), so the fixture must live in the real + // backups root. Only this one file is cleaned up — never the whole dir. + const downloadFixture = 'backup-2026-04-06T12-00-00.zip'; + const downloadFixturePath = path.join(DEFAULT_BACKUPS_ROOT, downloadFixture); beforeEach(() => { - // Create a real temporary file that Express can stream back - tmpFile = path.join(os.tmpdir(), `test-backup-${Date.now()}.zip`); - fs.writeFileSync(tmpFile, 'fake zip content'); - vi.mocked(backupService.backupFileExists).mockReturnValue(true); - vi.mocked(backupService.backupFilePath).mockReturnValue(tmpFile); + vi.mocked(backupService.backupFileExists).mockResolvedValue(true); }); afterAll(() => { - try { fs.unlinkSync(tmpFile); } catch {} + try { fs.unlinkSync(downloadFixturePath); } catch {} }); it('BACKUP-INT-001 — GET /backup/download/:filename returns 200 with content-disposition', async () => { const { user: admin } = createAdmin(testDb); - const filename = 'backup-2026-04-06T12-00-00.zip'; + // (Re)write the fixture immediately before the request that serves it. + fs.mkdirSync(DEFAULT_BACKUPS_ROOT, { recursive: true }); + fs.writeFileSync(downloadFixturePath, 'fake zip content'); const res = await request(app) - .get(`/api/backup/download/${filename}`) + .get(`/api/backup/download/${downloadFixture}`) .set('Cookie', authCookie(admin.id)); expect(res.status).toBe(200); expect(res.headers['content-disposition']).toMatch(/attachment/i); - expect(res.headers['content-disposition']).toContain(filename); + expect(res.headers['content-disposition']).toContain(downloadFixture); + // superagent has no parser for application/zip, so assert the byte count + // rather than the (unbuffered) body. + expect(res.headers['content-length']).toBe(String('fake zip content'.length)); }); it('BACKUP-INT-002 — GET /backup/download/:filename returns 400 for invalid filename', async () => { const { user: admin } = createAdmin(testDb); - vi.mocked(backupService.backupFileExists).mockReturnValue(false); + vi.mocked(backupService.backupFileExists).mockResolvedValue(false); const res = await request(app) .get('/api/backup/download/not-a-valid-name.tar.gz') @@ -241,7 +247,7 @@ describe('Backup download', () => { it('BACKUP-INT-003 — GET /backup/download/:filename returns 404 when file not found', async () => { const { user: admin } = createAdmin(testDb); - vi.mocked(backupService.backupFileExists).mockReturnValue(false); + vi.mocked(backupService.backupFileExists).mockResolvedValue(false); const res = await request(app) .get('/api/backup/download/backup-2026-04-06T12-00-00.zip') @@ -261,8 +267,8 @@ describe('Backup restore', () => { const { user: admin } = createAdmin(testDb); const filename = 'backup-2026-04-06T12-00-00.zip'; - vi.mocked(backupService.backupFileExists).mockReturnValue(true); - vi.mocked(backupService.restoreFromZip).mockResolvedValue({ success: true }); + vi.mocked(backupService.backupFileExists).mockResolvedValue(true); + vi.mocked(backupService.restoreBackup).mockResolvedValue({ success: true }); const res = await request(app) .post(`/api/backup/restore/${filename}`) @@ -270,12 +276,13 @@ describe('Backup restore', () => { expect(res.status).toBe(200); expect(res.body.success).toBe(true); + expect(vi.mocked(backupService.restoreBackup)).toHaveBeenCalledWith(expect.anything(), filename); }); it('BACKUP-INT-005 — POST /backup/restore/:filename returns 404 when backup not found', async () => { const { user: admin } = createAdmin(testDb); - vi.mocked(backupService.backupFileExists).mockReturnValue(false); + vi.mocked(backupService.backupFileExists).mockResolvedValue(false); const res = await request(app) .post('/api/backup/restore/backup-2026-04-06T12-00-00.zip') @@ -296,12 +303,12 @@ describe('Backup restore', () => { expect([400, 404]).toContain(res.status); }); - it('BACKUP-INT-007 — POST /backup/restore/:filename returns 400 when restoreFromZip reports failure', async () => { + it('BACKUP-INT-007 — POST /backup/restore/:filename returns 400 when the restore reports failure', async () => { const { user: admin } = createAdmin(testDb); const filename = 'backup-2026-04-06T12-00-00.zip'; - vi.mocked(backupService.backupFileExists).mockReturnValue(true); - vi.mocked(backupService.restoreFromZip).mockResolvedValue({ + vi.mocked(backupService.backupFileExists).mockResolvedValue(true); + vi.mocked(backupService.restoreBackup).mockResolvedValue({ success: false, error: 'Invalid backup: travel.db not found', status: 400, @@ -325,7 +332,7 @@ describe('Backup delete', () => { const { user: admin } = createAdmin(testDb); const filename = 'backup-2026-04-06T12-00-00.zip'; - vi.mocked(backupService.backupFileExists).mockReturnValue(true); + vi.mocked(backupService.backupFileExists).mockResolvedValue(true); vi.mocked(backupService.deleteBackup).mockReturnValue(undefined); const res = await request(app) @@ -334,13 +341,14 @@ describe('Backup delete', () => { expect(res.status).toBe(200); expect(res.body.success).toBe(true); - expect(vi.mocked(backupService.deleteBackup)).toHaveBeenCalledWith(filename); + // First arg is the app's real StorageService (BackupService forwards it). + expect(vi.mocked(backupService.deleteBackup)).toHaveBeenCalledWith(expect.anything(), filename); }); it('BACKUP-INT-009 — DELETE /backup/:filename returns 404 when not found', async () => { const { user: admin } = createAdmin(testDb); - vi.mocked(backupService.backupFileExists).mockReturnValue(false); + vi.mocked(backupService.backupFileExists).mockResolvedValue(false); const res = await request(app) .delete('/api/backup/backup-2026-04-06T12-00-00.zip') diff --git a/server/tests/integration/collab.test.ts b/server/tests/integration/collab.test.ts index bf3a9de67..44a72066f 100644 --- a/server/tests/integration/collab.test.ts +++ b/server/tests/integration/collab.test.ts @@ -264,6 +264,81 @@ describe('Collab notes', () => { expect(del.body.success).toBe(true); }); + it('COLLAB-P01 — attachment filename is stored bare (category + name; no files/ prefix)', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const create = await request(app) + .post(`/api/trips/${trip.id}/collab/notes`) + .set('Cookie', authCookie(user.id)) + .send({ title: 'Bare name' }); + const noteId = create.body.note.id; + + const upload = await request(app) + .post(`/api/trips/${trip.id}/collab/notes/${noteId}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', FIXTURE_PDF); + expect(upload.status).toBe(201); + + const list = await request(app) + .get(`/api/trips/${trip.id}/collab/notes`) + .set('Cookie', authCookie(user.id)); + const note = list.body.notes.find((n: any) => n.id === noteId); + expect(note.attachments[0].filename).toMatch(/^[0-9a-f-]{36}\.pdf$/); + expect(note.attachments[0].url).toMatch(/^\/api\/trips\/\d+\/files\/\d+\/download$/); + // The bytes live in uploads/files under the bare name. + expect(fs.existsSync(path.join(uploadsDir, note.attachments[0].filename))).toBe(true); + }); + + it('COLLAB-P02 — deleting a note file removes its bytes from uploads/files', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const create = await request(app) + .post(`/api/trips/${trip.id}/collab/notes`) + .set('Cookie', authCookie(user.id)) + .send({ title: 'Delete file bytes' }); + const noteId = create.body.note.id; + + const upload = await request(app) + .post(`/api/trips/${trip.id}/collab/notes/${noteId}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', FIXTURE_PDF); + const fileId = upload.body.file.id; + const diskName = path.basename(upload.body.file.filename); + expect(fs.existsSync(path.join(uploadsDir, diskName))).toBe(true); + + const del = await request(app) + .delete(`/api/trips/${trip.id}/collab/notes/${noteId}/files/${fileId}`) + .set('Cookie', authCookie(user.id)); + expect(del.status).toBe(200); + expect(fs.existsSync(path.join(uploadsDir, diskName))).toBe(false); + }); + + it('COLLAB-P03 — deleting a note removes its attachments’ bytes from uploads/files', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const create = await request(app) + .post(`/api/trips/${trip.id}/collab/notes`) + .set('Cookie', authCookie(user.id)) + .send({ title: 'Delete note bytes' }); + const noteId = create.body.note.id; + + const upload = await request(app) + .post(`/api/trips/${trip.id}/collab/notes/${noteId}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', FIXTURE_PDF); + const diskName = path.basename(upload.body.file.filename); + expect(fs.existsSync(path.join(uploadsDir, diskName))).toBe(true); + + const del = await request(app) + .delete(`/api/trips/${trip.id}/collab/notes/${noteId}`) + .set('Cookie', authCookie(user.id)); + expect(del.status).toBe(200); + expect(fs.existsSync(path.join(uploadsDir, diskName))).toBe(false); + }); + it('COLLAB-028 — uploaded note file URL uses authenticated download path, not /uploads/', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); diff --git a/server/tests/integration/collections.test.ts b/server/tests/integration/collections.test.ts new file mode 100644 index 000000000..e9187b24a --- /dev/null +++ b/server/tests/integration/collections.test.ts @@ -0,0 +1,156 @@ +/** + * Collections upload integration tests (COLL-P01…P05). + * + * Cover + place-image upload parity, written BEFORE the storage-upload swap: + * statuses, bodies, on-disk layout — including the cover filter's plain-Error + * 500 quirk (shared with the trip cover config) and the place-image filter's + * statusCode-400 contract. + */ +import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest'; +import request from 'supertest'; +import type { Application } from 'express'; +import type { INestApplication } from '@nestjs/common'; +import path from 'path'; +import fs from 'fs'; + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('PRAGMA busy_timeout = 5000'); + const mock = { + db, + closeDb: () => {}, + reinitialize: () => {}, + canAccessTrip: (tripId: any, userId: number) => + db.prepare(`SELECT t.id, t.user_id FROM trips t LEFT JOIN trip_members m ON m.trip_id = t.id AND m.user_id = ? WHERE t.id = ? AND (t.user_id = ? OR m.user_id IS NOT NULL)`).get(userId, tripId, userId), + isOwner: (tripId: any, userId: number) => + !!db.prepare('SELECT id FROM trips WHERE id = ? AND user_id = ?').get(tripId, userId), + }; + return { testDb: db, dbMock: mock }; +}); + +vi.mock('../../src/db/database', () => dbMock); +vi.mock('../../src/config', () => ({ + JWT_SECRET: 'test-jwt-secret-for-trek-testing-only', + ENCRYPTION_KEY: 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2', + updateJwtSecret: () => {}, + SESSION_DURATION: '24h', + SESSION_DURATION_MS: 86400000, + SESSION_DURATION_SECONDS: 86400, + DEFAULT_LANGUAGE: 'en', +})); +vi.mock('../../src/websocket', () => ({ broadcast: vi.fn(), broadcastToUser: vi.fn(), getOnlineUserIds: vi.fn(() => []) })); + +import { buildApp } from '../../src/bootstrap'; +import { createTables } from '../../src/db/schema'; +import { runMigrations } from '../../src/db/migrations'; +import { resetTestDb, resetRateLimits } from '../helpers/test-db'; +import { createUser } from '../helpers/factories'; +import { authCookie } from '../helpers/auth'; + +let nestApp: INestApplication; +let app: Application; +const FIXTURE_IMG = path.join(__dirname, '../fixtures/small-image.jpg'); +const coversDir = path.join(__dirname, '../../uploads/covers'); +const placesDir = path.join(__dirname, '../../uploads/places'); + +function createCollection(ownerId: number): number { + return Number(testDb.prepare("INSERT INTO collections (owner_id, name) VALUES (?, 'C')").run(ownerId).lastInsertRowid); +} + +function createCollectionPlace(collectionId: number, ownerId: number): number { + return Number(testDb.prepare("INSERT INTO collection_places (collection_id, owner_id, name) VALUES (?, ?, 'P')").run(collectionId, ownerId).lastInsertRowid); +} + +beforeAll(async () => { + createTables(testDb); + runMigrations(testDb); + nestApp = await buildApp(); + app = nestApp.getHttpAdapter().getInstance(); +}); + +beforeEach(() => { + resetTestDb(testDb); + resetRateLimits(nestApp); + // Enable the collections addon (the controller sits behind AddonGuard). + testDb.prepare( + "INSERT OR REPLACE INTO addons (id, name, description, type, icon, enabled, sort_order) VALUES ('collections', 'Collections', 'Saved places', 'global', 'Bookmark', 1, 40)" + ).run(); +}); + +afterAll(async () => { + await nestApp.close(); + testDb.close(); + fs.rmSync(coversDir, { recursive: true, force: true }); + fs.rmSync(placesDir, { recursive: true, force: true }); +}); + +describe('Collection cover upload', () => { + it('COLL-P01 — cover upload stores /uploads/covers/ and writes the file', async () => { + const { user } = createUser(testDb); + const collectionId = createCollection(user.id); + + const res = await request(app) + .post(`/api/addons/collections/${collectionId}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG, 'cover.png'); + expect(res.status).toBe(201); + expect(res.body.cover_image).toMatch(/^\/uploads\/covers\/[0-9a-f-]{36}\.png$/); + const diskName = res.body.cover_image.replace('/uploads/covers/', ''); + expect(fs.existsSync(path.join(coversDir, diskName))).toBe(true); + }); + + it('COLL-P02 — no file → 400 "No image uploaded"', async () => { + const { user } = createUser(testDb); + const collectionId = createCollection(user.id); + + const res = await request(app) + .post(`/api/addons/collections/${collectionId}/cover`) + .set('Cookie', authCookie(user.id)); + expect(res.status).toBe(400); + expect(res.body.error).toBe('No image uploaded'); + }); + + it('COLL-P03 — non-image cover is 500 (plain-Error filter quirk — pinned, do not "fix")', async () => { + const { user } = createUser(testDb); + const collectionId = createCollection(user.id); + + const res = await request(app) + .post(`/api/addons/collections/${collectionId}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', Buffer.from('plain text'), { filename: 'doc.txt', contentType: 'text/plain' }); + expect(res.status).toBe(500); + }); +}); + +describe('Collection place image upload', () => { + it('COLL-P04 — place image upload stores /uploads/places/ and writes the file', async () => { + const { user } = createUser(testDb); + const collectionId = createCollection(user.id); + const placeId = createCollectionPlace(collectionId, user.id); + + const res = await request(app) + .post(`/api/addons/collections/places/${placeId}/image`) + .set('Cookie', authCookie(user.id)) + .attach('image', FIXTURE_IMG, 'photo.jpg'); + expect(res.status).toBe(200); + expect(res.body.image_url).toMatch(/^\/uploads\/places\/[0-9a-f-]{36}\.jpg$/); + const diskName = res.body.image_url.replace('/uploads/places/', ''); + expect(fs.existsSync(path.join(placesDir, diskName))).toBe(true); + }); + + it('COLL-P05 — non-image place upload is 400 with the bespoke message', async () => { + const { user } = createUser(testDb); + const collectionId = createCollection(user.id); + const placeId = createCollectionPlace(collectionId, user.id); + + const res = await request(app) + .post(`/api/addons/collections/places/${placeId}/image`) + .set('Cookie', authCookie(user.id)) + .attach('image', Buffer.from('%PDF-1.4'), { filename: 'doc.pdf', contentType: 'application/pdf' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Only jpg, png, gif, webp images allowed'); + }); +}); diff --git a/server/tests/integration/files.test.ts b/server/tests/integration/files.test.ts index 32733332f..fda7f26bd 100644 --- a/server/tests/integration/files.test.ts +++ b/server/tests/integration/files.test.ts @@ -133,6 +133,57 @@ describe('Upload file', () => { } }); + it('FILE-P01 — stored filename is a bare uuid name and the bytes land in uploads/files', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await uploadFile(trip.id, user.id, FIXTURE_PDF); + expect(res.status).toBe(201); + expect(res.body.file.filename).toMatch(/^[0-9a-f-]{36}\.pdf$/); + expect(fs.existsSync(path.join(uploadsDir, res.body.file.filename))).toBe(true); + // Nothing left behind in the spool after a successful commit. + const spoolDir = path.join(__dirname, '../../uploads/.tmp'); + if (fs.existsSync(spoolDir)) { + expect(fs.readdirSync(spoolDir)).not.toContain(res.body.file.filename); + } + }); + + it('FILE-P02 — non-video over 50 MB is rejected with no bytes left anywhere', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const BIG = 51 * 1024 * 1024; + // Other suites share the real uploads/ tree when the whole tier runs in + // parallel, so the residue checks key on this upload's unique size rather + // than comparing whole-directory listings. + const bigFilesIn = (dir: string) => + fs.existsSync(dir) + ? fs.readdirSync(dir).filter((f) => { + try { return fs.statSync(path.join(dir, f)).size === BIG; } catch { return false; } + }) + : []; + const res = await request(app) + .post(`/api/trips/${trip.id}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', Buffer.alloc(BIG), { filename: 'big.pdf', contentType: 'application/pdf' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('File is too large'); + expect(bigFilesIn(uploadsDir)).toEqual([]); + expect(bigFilesIn(path.join(__dirname, '../../uploads/.tmp'))).toEqual([]); + }); + + it('FILE-P03 — non-ASCII original filenames are preserved (defParamCharset utf8)', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', Buffer.from('utf8 name test'), { filename: 'résumé.pdf', contentType: 'application/pdf' }); + expect(res.status).toBe(201); + expect(res.body.file.original_name).toBe('résumé.pdf'); + }); + it('FILE-021 — non-member cannot upload file', async () => { const { user: owner } = createUser(testDb); const { user: other } = createUser(testDb); @@ -514,4 +565,64 @@ describe('File download', () => { .set('Cookie', `trek_session=${token}`); expect(dl.status).toBe(200); }); + + it('FILE-P04 — download serves the exact bytes with Content-Type, ETag, and Range support', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + const upload = await uploadFile(trip.id, user.id, FIXTURE_PDF); + const fileId = upload.body.file.id; + const fixtureBytes = fs.readFileSync(FIXTURE_PDF); + + const dl = await request(app) + .get(`/api/trips/${trip.id}/files/${fileId}/download`) + .set('Cookie', authCookie(user.id)); + expect(dl.status).toBe(200); + expect(dl.headers['content-type']).toBe('application/pdf'); + expect(dl.headers.etag).toMatch(/^W\/"[0-9a-f]+-[0-9a-f]+"$/); + expect(Buffer.from(dl.body).equals(fixtureBytes)).toBe(true); + + const ranged = await request(app) + .get(`/api/trips/${trip.id}/files/${fileId}/download`) + .set('Cookie', authCookie(user.id)) + .set('Range', 'bytes=0-3'); + expect(ranged.status).toBe(206); + expect(ranged.headers['content-range']).toBe(`bytes 0-3/${fixtureBytes.length}`); + }); + + it('FILE-P05 — Apple Wallet passes are served inline with the canonical MIME types', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + for (const [ext, mime] of [ + ['pkpass', 'application/vnd.apple.pkpass'], + ['pkpasses', 'application/vnd.apple.pkpasses'], + ] as const) { + const upload = await request(app) + .post(`/api/trips/${trip.id}/files`) + .set('Cookie', authCookie(user.id)) + .attach('file', Buffer.from('PK-wallet-bytes'), `pass.${ext}`); + expect(upload.status).toBe(201); + + const dl = await request(app) + .get(`/api/trips/${trip.id}/files/${upload.body.file.id}/download`) + .set('Cookie', authCookie(user.id)); + expect(dl.status).toBe(200); + expect(dl.headers['content-type']).toBe(mime); + expect(dl.headers['content-disposition']).toBe(`inline; filename="pass.${ext}"`); + } + }); + + it('FILE-P06 — a DB row whose bytes are gone answers 404 File not found', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + const upload = await uploadFile(trip.id, user.id, FIXTURE_PDF); + const fileId = upload.body.file.id; + fs.rmSync(path.join(uploadsDir, upload.body.file.filename), { force: true }); + + const dl = await request(app) + .get(`/api/trips/${trip.id}/files/${fileId}/download`) + .set('Cookie', authCookie(user.id)); + expect(dl.status).toBe(404); + expect(dl.body).toEqual({ error: 'File not found' }); + }); }); diff --git a/server/tests/integration/journey.test.ts b/server/tests/integration/journey.test.ts index af2c9f11c..421b07751 100644 --- a/server/tests/integration/journey.test.ts +++ b/server/tests/integration/journey.test.ts @@ -1008,3 +1008,154 @@ describe('Photo upload validation', () => { expect(res.body.error).toBe('No files uploaded'); }); }); + +// ───────────────────────────────────────────────────────────────────────────── +// Upload parity (JOURNEY-P01…P09) — entry photos, gallery photos, video+poster, +// journey cover. Written BEFORE the storage-upload swap to pin behavior. +// ───────────────────────────────────────────────────────────────────────────── + +describe('Journey upload parity', () => { + const fsMod = require('fs') as typeof import('fs'); + const pathMod = require('path') as typeof import('path'); + const FIXTURE_IMG = pathMod.join(__dirname, '../fixtures/small-image.jpg'); + const journeyDir = pathMod.join(__dirname, '../../uploads/journey'); + + afterAll(() => { + fsMod.rmSync(journeyDir, { recursive: true, force: true }); + }); + + it('JOURNEY-P01 — entry photo upload stores journey/ with a lowercased extension', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + const entry = createJourneyEntry(testDb, journey.id, user.id, { entry_date: '2026-04-01' }); + + const res = await request(app) + .post(`/api/journeys/entries/${entry.id}/photos`) + .set('Cookie', authCookie(user.id)) + .attach('photos', FIXTURE_IMG, 'PHOTO.JPG'); + expect(res.status).toBe(201); + expect(res.body.photos).toHaveLength(1); + expect(res.body.photos[0].file_path).toMatch(/^journey\/[0-9a-f-]{36}\.jpg$/); + const diskName = res.body.photos[0].file_path.replace(/^journey\//, ''); + expect(fsMod.existsSync(pathMod.join(journeyDir, diskName))).toBe(true); + }); + + it('JOURNEY-P02 — extensionless originalname falls back to .jpg (wildcard allowlist)', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + const entry = createJourneyEntry(testDb, journey.id, user.id, { entry_date: '2026-04-01' }); + // The admin allowlist rejects the empty extension before the filename + // fallback can run; the wildcard makes the fallback reachable so it is + // actually pinned here. + testDb.prepare("INSERT OR REPLACE INTO app_settings (key, value) VALUES ('allowed_file_types', '*')").run(); + + const res = await request(app) + .post(`/api/journeys/entries/${entry.id}/photos`) + .set('Cookie', authCookie(user.id)) + .attach('photos', fsMod.readFileSync(FIXTURE_IMG), { filename: 'noext', contentType: 'image/jpeg' }); + expect(res.status).toBe(201); + expect(res.body.photos[0].file_path).toMatch(/^journey\/[0-9a-f-]{36}\.jpg$/); + }); + + it('JOURNEY-P03 — non-image entry upload is rejected 400', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + const entry = createJourneyEntry(testDb, journey.id, user.id, { entry_date: '2026-04-01' }); + + const res = await request(app) + .post(`/api/journeys/entries/${entry.id}/photos`) + .set('Cookie', authCookie(user.id)) + .attach('photos', Buffer.from('plain text'), { filename: 'doc.txt', contentType: 'text/plain' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('Only image files are allowed'); + }); + + it('JOURNEY-P04 — gallery photo upload stores journey/ paths', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + + const res = await request(app) + .post(`/api/journeys/${journey.id}/gallery/photos`) + .set('Cookie', authCookie(user.id)) + .attach('photos', FIXTURE_IMG); + expect(res.status).toBe(201); + expect(res.body.photos.length).toBeGreaterThan(0); + }); + + it('JOURNEY-P05 — non-contributor gallery upload is 403 and files remain on disk (no cleanup today)', async () => { + const { user: owner } = createUser(testDb); + const { user: other } = createUser(testDb); + const journey = createJourney(testDb, owner.id); + + const before = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + const res = await request(app) + .post(`/api/journeys/${journey.id}/gallery/photos`) + .set('Cookie', authCookie(other.id)) + .attach('photos', FIXTURE_IMG); + expect(res.status).toBe(403); + expect(res.body.error).toBe('Not allowed'); + const after = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + expect(after).toBe(before + 1); + }); + + it('JOURNEY-P06 — video+poster upload: poster is ALWAYS stored as .jpg (stored-XSS pin)', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + + const res = await request(app) + .post(`/api/journeys/${journey.id}/gallery/video`) + .set('Cookie', authCookie(user.id)) + .attach('video', Buffer.from('fake video bytes'), { filename: 'clip.MP4', contentType: 'video/mp4' }) + .attach('poster', fsMod.readFileSync(FIXTURE_IMG), { filename: 'poster.png', contentType: 'image/png' }); + expect(res.status).toBe(201); + expect(res.body.photos).toHaveLength(1); + expect(res.body.photos[0].file_path).toMatch(/^journey\/[0-9a-f-]{36}\.mp4$/); + expect(res.body.photos[0].thumbnail_path).toMatch(/^journey\/[0-9a-f-]{36}\.jpg$/); + }); + + it('JOURNEY-P07 — poster-only video upload is 400 with no files left anywhere', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + + const before = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + const res = await request(app) + .post(`/api/journeys/${journey.id}/gallery/video`) + .set('Cookie', authCookie(user.id)) + .attach('poster', fsMod.readFileSync(FIXTURE_IMG), { filename: 'poster.png', contentType: 'image/png' }); + expect(res.status).toBe(400); + expect(res.body.error).toBe('No video uploaded'); + const after = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + expect(after).toBe(before); + }); + + it('JOURNEY-P08 — non-contributor video upload is 403 and BOTH files are removed', async () => { + const { user: owner } = createUser(testDb); + const { user: other } = createUser(testDb); + const journey = createJourney(testDb, owner.id); + + const before = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + const res = await request(app) + .post(`/api/journeys/${journey.id}/gallery/video`) + .set('Cookie', authCookie(other.id)) + .attach('video', Buffer.from('fake video bytes'), { filename: 'clip.mp4', contentType: 'video/mp4' }) + .attach('poster', fsMod.readFileSync(FIXTURE_IMG), { filename: 'poster.png', contentType: 'image/png' }); + expect(res.status).toBe(403); + expect(res.body.error).toBe('Not allowed'); + const after = fsMod.existsSync(journeyDir) ? fsMod.readdirSync(journeyDir).length : 0; + expect(after).toBe(before); + }); + + it('JOURNEY-P09 — cover upload stores journey/ into cover_image', async () => { + const { user } = createUser(testDb); + const journey = createJourney(testDb, user.id); + + const res = await request(app) + .post(`/api/journeys/${journey.id}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG); + expect(res.status).toBe(200); + expect(res.body.cover_image).toMatch(/^journey\/[0-9a-f-]{36}\.jpg$/); + const diskName = res.body.cover_image.replace(/^journey\//, ''); + expect(fsMod.existsSync(pathMod.join(journeyDir, diskName))).toBe(true); + }); +}); diff --git a/server/tests/integration/leg-mode-incoming.test.ts b/server/tests/integration/leg-mode-incoming.test.ts index 7aa60de59..8f864fcb9 100644 --- a/server/tests/integration/leg-mode-incoming.test.ts +++ b/server/tests/integration/leg-mode-incoming.test.ts @@ -35,48 +35,32 @@ describe('incoming_leg_transport_mode migration', () => { // The case above starts from an empty database and runs every migration, so it // passes wherever a migration sits in the array. - it('the trailing migration lands on a database that is one version behind', () => { + it('every unreleased migration replays cleanly on a database that is behind', () => { // What actually has to hold: existing installs replay only the slots above - // their schema_version, so a migration inserted mid-array is silently - // skipped on every one of them. Undoing what the LAST migration did and - // rewinding one version is the only way to prove it still runs. + // their schema_version, and an install may skip releases — so every + // migration that has not shipped yet must be REPLAY-SAFE on a schema where + // it already applied: guard DDL (CREATE ... IF NOT EXISTS, pragma column + // checks before ALTER TABLE) and make data transforms idempotent (WHERE + // guards, UPDATE OR IGNORE). This test rewinds a fully migrated database + // to the last released version and replays everything above it; a slot + // that is not replay-safe throws here. // - // >>> Appending a migration? Re-point the "undo" below at whatever yours - // >>> does. That is the whole maintenance cost of this guard. The current - // >>> trailing slot clears the GB rows out of place_regions, because the - // >>> atlas bundle now carries Great Britain at county/borough level and - // >>> the cached GB-ENG rows would otherwise never be re-derived (#1974). + // >>> Appending a migration? Nothing to change here — just write it + // >>> replay-safe; this replay tells you if it is not. + // >>> Cutting a release? Bump RELEASED_VERSION to the version it ships. + const RELEASED_VERSION = 189; + const upgraded = new Database(':memory:'); upgraded.exec('PRAGMA foreign_keys = ON'); createTables(upgraded); runMigrations(upgraded); const { version } = upgraded.prepare('SELECT version FROM schema_version').get() as { version: number }; + expect(version).toBeGreaterThanOrEqual(RELEASED_VERSION); - // Put back exactly what that migration exists to remove: a stale cache row - // pointing at the constituent-country polygon, plus one for another country - // that must survive untouched. - upgraded.exec(` - INSERT INTO users (id, username, email, password_hash) VALUES (900, 'u', 'u@e.test', 'x'); - INSERT INTO trips (id, user_id, title) VALUES (900, 900, 'Trip'); - INSERT INTO places (id, trip_id, name) VALUES (9001, 900, 'Camden Market'); - INSERT INTO places (id, trip_id, name) VALUES (9002, 900, 'Sagrada Familia'); - INSERT INTO place_regions (place_id, country_code, region_code, region_name) - VALUES (9001, 'GB', 'GB-ENG', 'England'), (9002, 'ES', 'ES-CT', 'Catalunya'); - `); + upgraded.prepare('UPDATE schema_version SET version = ?').run(RELEASED_VERSION); + runMigrations(upgraded); // a throw = some trailing migration is not replay-safe - upgraded.prepare('UPDATE schema_version SET version = ?').run(version - 1); - - runMigrations(upgraded); - - // The assertion that proves the trailing slot ran. - expect( - upgraded.prepare("SELECT place_id FROM place_regions WHERE country_code = 'GB'").get() - ).toBeUndefined(); - // ...and that it kept to its own country. - expect( - upgraded.prepare("SELECT region_code FROM place_regions WHERE place_id = 9002").get() - ).toEqual({ region_code: 'ES-CT' }); expect(upgraded.prepare('SELECT version FROM schema_version').get()).toEqual({ version }); upgraded.close(); }); diff --git a/server/tests/integration/places.test.ts b/server/tests/integration/places.test.ts index 7223d6ce7..583cc3565 100644 --- a/server/tests/integration/places.test.ts +++ b/server/tests/integration/places.test.ts @@ -1046,7 +1046,12 @@ describe('Custom place image upload', () => { .set('Cookie', authCookie(user.id)) .attach('image', FIXTURE_JPEG); expect(res.status).toBe(200); - expect(res.body.place.image_url).toMatch(/^\/uploads\/places\//); + expect(res.body.place.image_url).toMatch(/^\/uploads\/places\/[0-9a-f-]{36}\./); + // The bytes land at the final uploads/places path under the bare uuid name. + const fsMod = require('fs') as typeof import('fs'); + const pathMod = require('path') as typeof import('path'); + const diskName = res.body.place.image_url.replace('/uploads/places/', ''); + expect(fsMod.existsSync(pathMod.join(__dirname, '../../uploads/places', diskName))).toBe(true); const cleared = await request(app) .put(`/api/trips/${trip.id}/places/${place.id}`) diff --git a/server/tests/integration/profile.test.ts b/server/tests/integration/profile.test.ts index 4a71a82c0..4b79ab541 100644 --- a/server/tests/integration/profile.test.ts +++ b/server/tests/integration/profile.test.ts @@ -108,6 +108,32 @@ describe('Avatar', () => { expect(typeof res.body.avatar_url).toBe('string'); }); + it('PROFILE-P01 — avatar bytes land at uploads/avatars under a bare uuid name', async () => { + const fsMod = require('fs') as typeof import('fs'); + const pathMod = require('path') as typeof import('path'); + const avatarsDir = pathMod.join(__dirname, '../../uploads/avatars'); + const { user } = createUser(testDb); + + const res = await request(app) + .post('/api/auth/avatar') + .set('Cookie', authCookie(user.id)) + .attach('avatar', FIXTURE_JPEG); + expect(res.status).toBe(200); + const first = pathMod.basename(res.body.avatar_url); + expect(first).toMatch(/^[0-9a-f-]{36}\.jpg$/); + expect(fsMod.existsSync(pathMod.join(avatarsDir, first))).toBe(true); + + // Re-upload removes the previous file (saveAvatar's cleanup branch). + const second = await request(app) + .post('/api/auth/avatar') + .set('Cookie', authCookie(user.id)) + .attach('avatar', FIXTURE_JPEG); + expect(second.status).toBe(200); + const secondName = pathMod.basename(second.body.avatar_url); + expect(fsMod.existsSync(pathMod.join(avatarsDir, secondName))).toBe(true); + expect(fsMod.existsSync(pathMod.join(avatarsDir, first))).toBe(false); + }); + it('PROFILE-003 — uploading non-image (PDF) is rejected', async () => { const { user } = createUser(testDb); const res = await request(app) diff --git a/server/tests/integration/security.test.ts b/server/tests/integration/security.test.ts index 6241c53fb..ceac29db1 100644 --- a/server/tests/integration/security.test.ts +++ b/server/tests/integration/security.test.ts @@ -221,7 +221,7 @@ describe('File download path traversal', () => { const res = await request(app) .get(`/api/trips/${trip.id}/files/${fileId}/download`) .set(authHeader(user.id)); - // resolveFilePath strips traversal via path.basename; normalized file does not exist in uploads + // path.basename() strips traversal in the download controller; normalized file does not exist in uploads expect(res.status).not.toBe(200); }); }); diff --git a/server/tests/integration/share.test.ts b/server/tests/integration/share.test.ts index b8484d8cf..6cccb1eb6 100644 --- a/server/tests/integration/share.test.ts +++ b/server/tests/integration/share.test.ts @@ -51,13 +51,27 @@ import { createUser, createTrip, addTripMember, createDay, createPlace, createDa import { authCookie } from '../helpers/auth'; import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import { DatabaseService } from '../../src/nest/database/database.service'; -import { RuntimeEnvService } from '../../src/nest/app-config/runtime-env.service'; import { db as sharedDb } from '../../src/db/database'; +import { LocalDriver } from '../../src/nest/storage/drivers/local.driver'; +import { StorageService } from '../../src/nest/storage/storage.service'; +import type { StorageRegistryService, ResolvedCategory } from '../../src/nest/storage/storage-registry.service'; +import { DEFAULT_UPLOADS_ROOT, GLOBAL_TEMP_DIR } from '../../src/nest/storage/storage-paths'; // A real instance over the same connection the app uses — these cases write a -// cache entry and then read it back through the HTTP route. -const placePhotoCache = new PlacePhotoCacheService(new DatabaseService(sharedDb), new RuntimeEnvService()); +// cache entry and then read it back through the HTTP route, so the stub +// storage must be rooted where the app's registry serves 'photos-google' +// from (mode A: uploads/photos/google). +const uploadsDriver = new LocalDriver({ id: 'share-test-local', root: DEFAULT_UPLOADS_ROOT }); +uploadsDriver.init({ ensurePrefixes: ['photos/google/'], cleanSpool: false }); +const testStorage = new StorageService({ + resolve: (): ResolvedCategory => ({ driver: uploadsDriver, keyPrefix: 'photos/google/', backendName: 'share-test-local' }), + tempDir: () => GLOBAL_TEMP_DIR, + replicaFailures: () => [], +} as unknown as StorageRegistryService); +const placePhotoCache = new PlacePhotoCacheService(new DatabaseService(sharedDb), testStorage); import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; let nestApp: INestApplication; let app: Application; @@ -561,8 +575,8 @@ describe('Shared trip — place photos in shared links (issue #1100)', () => { it('SHARE-018 — public proxy streams cached bytes for a valid token + place (no cookie)', async () => { const { token } = await setupSharedPlaceWithPhoto(); - const cached = await placePhotoCache.put(PLACE_ID, photoBytes, null); - cachedFilePath = cached.filePath; + await placePhotoCache.put(PLACE_ID, photoBytes, null); + cachedFilePath = path.join(DEFAULT_UPLOADS_ROOT, 'photos/google', `${crypto.createHash('sha1').update(PLACE_ID).digest('hex')}.jpg`); const res = await request(app).get(`/api/shared/${token}/place-photo/${encodeURIComponent(PLACE_ID)}/bytes`); expect(res.status).toBe(200); @@ -570,24 +584,30 @@ describe('Shared trip — place photos in shared links (issue #1100)', () => { expect(Buffer.from(res.body)).toEqual(photoBytes); }); - it('SHARE-019 — public proxy 404s for a placeId not in the shared trip', async () => { + // Every miss shape answers the same empty 204 (fix: mirrors the maps proxy, + // #1727 extended to shared pages). The service still collapses "not in + // trip" / "bad token" / "map disabled" into one indistinguishable null, so + // no authorization detail leaks through the status change. + it('SHARE-019 — public proxy answers an empty 204 for a placeId not in the shared trip', async () => { const { token } = await setupSharedPlaceWithPhoto(); const res = await request(app).get(`/api/shared/${token}/place-photo/ChIJnotInTrip/bytes`); - expect(res.status).toBe(404); - expect(res.body).toEqual({ error: 'Photo not cached' }); + expect(res.status).toBe(204); + expect(res.headers['cache-control']).toBe('no-store'); + expect(res.text ?? '').toBe(''); }); - it('SHARE-020 — public proxy 404s for an invalid token', async () => { + it('SHARE-020 — public proxy answers an empty 204 for an invalid token', async () => { await setupSharedPlaceWithPhoto(); const res = await request(app).get(`/api/shared/bad-token/place-photo/${encodeURIComponent(PLACE_ID)}/bytes`); - expect(res.status).toBe(404); - expect(res.body).toEqual({ error: 'Photo not cached' }); + expect(res.status).toBe(204); + expect(res.headers['cache-control']).toBe('no-store'); + expect(res.text ?? '').toBe(''); }); // Regression — GHSA-9hc8 sibling: place photos are part of the map/itinerary, - // so the proxy must 404 when the owner disabled the map, even with a valid - // token + cached bytes. - it('SHARE-025 — public place-photo proxy 404s when share_map=false', async () => { + // so the proxy must stream nothing when the owner disabled the map, even + // with a valid token + cached bytes. + it('SHARE-025 — public place-photo proxy streams nothing when share_map=false', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Hidden Photo Place' }); @@ -596,11 +616,11 @@ describe('Shared trip — place photos in shared links (issue #1100)', () => { .post(`/api/trips/${trip.id}/share-link`) .set('Cookie', authCookie(user.id)) .send({ share_map: false }); - const cached = await placePhotoCache.put(PLACE_ID, photoBytes, null); - cachedFilePath = cached.filePath; + await placePhotoCache.put(PLACE_ID, photoBytes, null); + cachedFilePath = path.join(DEFAULT_UPLOADS_ROOT, 'photos/google', `${crypto.createHash('sha1').update(PLACE_ID).digest('hex')}.jpg`); const res = await request(app).get(`/api/shared/${token}/place-photo/${encodeURIComponent(PLACE_ID)}/bytes`); - expect(res.status).toBe(404); - expect(res.body).toEqual({ error: 'Photo not cached' }); + expect(res.status).toBe(204); + expect(res.text ?? '').toBe(''); }); }); diff --git a/server/tests/integration/storage-restore-reload.test.ts b/server/tests/integration/storage-restore-reload.test.ts new file mode 100644 index 000000000..7ddcd12df --- /dev/null +++ b/server/tests/integration/storage-restore-reload.test.ts @@ -0,0 +1,122 @@ +/** + * Task C6 (audit #4, major) — a restored archive carries its OWN + * `app_settings` storage.* rows, which can name a different category + * assignment than the pre-restore install. Before this fix, + * `restoreFromZip` never reloaded the storage registry after reopening the + * DB, so uploads rehydration ran against the STALE pre-restore driver/ + * category map — files landed on the wrong backend. + * + * `backup.impl.test.ts` (BACKUP-045f / BACKUP-061f) pins the *ordering* + * inside `restoreFromZip` with a stubbed StorageService: reinitialize() → + * storage.reloadConfig() → rehydration, and that reload is skipped when + * reopen fails. This test pins the other half — that `reloadConfig()` + * genuinely causes a changed category assignment to take effect for real + * byte placement — using the REAL StorageRegistryService/StorageService/ + * DatabaseService trio (same pattern as storage-registry.service.test.ts), + * with two real LocalDriver backends over real tmp directories, so the + * category under test ('files', reassigned between nas-a/nas-b) never + * touches a production path. Note: registry construction itself still + * mkdirs the built-in uploads-local/backups-local roots (server/uploads, + * server/data/backups) as an unconditional side effect of onModuleInit() — + * accepted precedent from storage-registry.service.test.ts, harmless (mkdir + * -p on an existing dir), and orthogonal to what this test actually exercises. + */ +import { describe, it, expect, beforeAll, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const Database = require('better-sqlite3'); + +import { createTables } from '../../src/db/schema'; +import { runMigrations } from '../../src/db/migrations'; +import { DatabaseService } from '../../src/nest/database/database.service'; +import type { RuntimeEnvService } from '../../src/nest/app-config/runtime-env.service'; +import { StorageEventsService } from '../../src/nest/storage/storage-events.service'; +import { BACKENDS_KEY, CATEGORIES_KEY, StorageRegistryService } from '../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../src/nest/storage/storage.service'; + +const testDb = new Database(':memory:'); +testDb.exec('PRAGMA journal_mode = WAL'); +testDb.exec('PRAGMA foreign_keys = ON'); +const db = new DatabaseService(testDb); + +beforeAll(() => { + createTables(testDb); + runMigrations(testDb); +}); + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-restore-reload-')); + tmpDirs.push(dir); + return dir; +} + +function envStub(): RuntimeEnvService { + return { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; +} + +function setSetting(key: string, value: unknown): void { + testDb + .prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)') + .run(key, JSON.stringify(value)); +} + +afterEach(() => { + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); + testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run(); +}); + +describe('C6 — restore reloads the storage registry (audit #4)', () => { + it("reloadConfig() picks up the restored archive's category assignment before any byte is rehydrated", async () => { + const rootA = makeTmpDir(); + const rootB = makeTmpDir(); + + // Pre-restore world: 'files' resolves to backend nas-a. + setSetting(BACKENDS_KEY, [ + { name: 'nas-a', type: 'local', options: { root: rootA } }, + { name: 'nas-b', type: 'local', options: { root: rootB } }, + ]); + setSetting(CATEGORIES_KEY, { files: 'nas-a' }); + + const registry = new StorageRegistryService(db, envStub(), new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + + expect(registry.resolve('files').backendName).toBe('nas-a'); + + // A pre-restore object lives under nas-a. + const preSrc = path.join(makeTmpDir(), 'old.txt'); + fs.writeFileSync(preSrc, 'pre-restore bytes'); + await storage.put('files', 'old.txt', { tmpPath: preSrc }); + expect(fs.existsSync(path.join(rootA, 'files', 'old.txt'))).toBe(true); + + // Simulate what restoreFromZip's DB swap + reinitialize() actually does in + // production: the restored archive's travel.db is now live, and IT names a + // different backend for 'files' (an admin on the source install pointed + // 'files' at a different backend before taking that backup). + setSetting(CATEGORIES_KEY, { files: 'nas-b' }); + + // Audit #4, pre-fix: the registry is still holding the pre-restore driver + // map at this point — resolve() is stale until something calls reload(). + expect(registry.resolve('files').backendName).toBe('nas-a'); + + // The fix under test: StorageService.reloadConfig(), the narrow passthrough + // restoreFromZip now calls right after reinitialize() and right before + // rehydration (see backup.impl.ts). + storage.reloadConfig(); + + expect(registry.resolve('files').backendName).toBe('nas-b'); + + // Rehydration (the same storage.put({ tmpPath }) shape backup.impl.ts's + // rehydrateUploads uses for every extracted upload entry) now lands under + // the RESTORED config's backend, never the stale pre-restore one. + const restoredSrc = path.join(makeTmpDir(), 'restored.txt'); + fs.writeFileSync(restoredSrc, 'restored bytes'); + await storage.put('files', 'restored.txt', { tmpPath: restoredSrc }); + + expect(fs.existsSync(path.join(rootB, 'files', 'restored.txt'))).toBe(true); + expect(fs.existsSync(path.join(rootA, 'files', 'restored.txt'))).toBe(false); + }); +}); diff --git a/server/tests/integration/storage-stream-abort.test.ts b/server/tests/integration/storage-stream-abort.test.ts new file mode 100644 index 000000000..c0eb85d84 --- /dev/null +++ b/server/tests/integration/storage-stream-abort.test.ts @@ -0,0 +1,347 @@ +/** + * Real-socket repro for the audit's open question ("does a client abort + * mid-download take the process down?") — task C3. Everything else touching + * this fix is exercised at the unit level (storage.service.test.ts, + * maps.controller.test.ts, share.controller.test.ts, exception-filter.test.ts) + * against mocked streams; this suite is the one place a *real* TCP client + * destroys a *real* connection mid-transfer, so the sendFile ECONNABORTED + * path and the pipeline() premature-close path are proven against actual + * socket/eos semantics, not a mock's approximation of them. + * + * A bare express() + http.Server wraps StorageService directly — no need to + * boot the full Nest app for this, since sendToResponse is the one place + * every byte-serving caller (files-download, journey-public, + * photo-resolver, trek-photo-cache, backup, /uploads/photos) funnels + * through, per storage.service.ts's own docblock. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import express from 'express'; +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +import { LocalDriver } from '../../src/nest/storage/drivers/local.driver'; +import type { StorageRegistryService, ResolvedCategory } from '../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../src/nest/storage/storage.service'; +import type { ObjectStat, StorageDriver } from '../../src/nest/storage/storage.types'; + +/** Trickles chunks on a timer so a real client has time to abort mid-stream + * regardless of machine/CI speed — the S3-network-latency stand-in. */ +class SlowReadable extends Readable { + private sent = 0; + private destroyedFlag = false; + constructor(private readonly totalChunks = 40) { + super(); + } + override _read(): void { + if (this.sent >= this.totalChunks) { + this.push(null); + return; + } + this.sent++; + setTimeout(() => this.push(Buffer.alloc(2048, 'x')), 5); + } + override _destroy(err: Error | null, cb: (err: Error | null) => void): void { + this.destroyedFlag = true; + cb(err); + } + get wasDestroyed(): boolean { + return this.destroyedFlag; + } +} + +/** Fails before ever pushing a byte — the S3-SDK-network-blip stand-in: a + * source error with NO client involved at all, before any header has gone + * out. Used to prove a pre-header failure still rejects (the caller's + * miss/error contract must run), even when the failure carries a code that + * looks abort-like (ECONNRESET). */ +class FailingBeforeFirstByteReadable extends Readable { + override _read(): void { + process.nextTick(() => this.destroy(Object.assign(new Error('simulated S3 network blip'), { code: 'ECONNRESET' }))); + } +} + +function makeRemoteFixture(): { storage: StorageService; lastStream: () => SlowReadable | undefined } { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-abort-')); + let last: SlowReadable | undefined; + const stat: ObjectStat = { key: 'remote.bin', size: 40 * 2048, mtimeMs: 1 }; + const driver: StorageDriver = { + id: 'stub-remote-abort', + put: async () => undefined, + getStream: async () => { + last = new SlowReadable(); + return { stream: last, stat }; + }, + stat: async () => stat, + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix: '', backendName: 'stub-remote-abort' }), + tempDir: () => tmpDir, + replicaFailures: () => [], + } as unknown as StorageRegistryService; + return { storage: new StorageService(registry), lastStream: () => last }; +} + +function makeFailingRemoteFixture(): { storage: StorageService } { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-abort-fail-')); + const stat: ObjectStat = { key: 'remote.bin', size: 40 * 2048, mtimeMs: 1 }; + const driver: StorageDriver = { + id: 'stub-remote-fail', + put: async () => undefined, + getStream: async () => ({ stream: new FailingBeforeFirstByteReadable(), stat }), + stat: async () => stat, + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix: '', backendName: 'stub-remote-fail' }), + tempDir: () => tmpDir, + replicaFailures: () => [], + } as unknown as StorageRegistryService; + return { storage: new StorageService(registry) }; +} + +function makeLocalFixture(): { storage: StorageService; root: string } { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-abort-local-')); + const driver = new LocalDriver({ id: 'stub-local-abort', root }); + driver.init({ cleanSpool: true }); + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix: '', backendName: 'stub-local-abort' }), + tempDir: () => root, + replicaFailures: () => [], + } as unknown as StorageRegistryService; + return { storage: new StorageService(registry), root }; +} + +describe('storage stream abort — real socket (C3 repro)', () => { + let unhandled: unknown[] = []; + const onUnhandled = (err: unknown) => unhandled.push(err); + + beforeEach(() => { + unhandled = []; + process.on('unhandledRejection', onUnhandled); + }); + + afterEach(() => { + process.off('unhandledRejection', onUnhandled); + }); + + describe('remote (pipeline) branch', () => { + let server: http.Server; + let baseUrl: string; + let fx: ReturnType; + let outcome: { settled: 'resolved' | 'rejected' | 'pending'; error?: unknown }; + + beforeAll(async () => { + fx = makeRemoteFixture(); + const app = express(); + app.get('/slow', (_req, res) => { + outcome = { settled: 'pending' }; + fx.storage + .sendToResponse('files', 'remote.bin', res) + .then(() => { + outcome = { settled: 'resolved' }; + }) + .catch((error: unknown) => { + outcome = { settled: 'rejected', error }; + }); + }); + server = http.createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + it('resolves cleanly (no 500, no unhandled rejection) and destroys the source stream (no S3 socket leak)', async () => { + await new Promise((resolve, reject) => { + const req = http.get(`${baseUrl}/slow`, (res) => { + res.once('data', () => { + // Abort mid-transfer: headers + at least one chunk are already + // on the wire, the SlowReadable is still trickling out the rest. + req.destroy(); + }); + res.on('error', () => {}); // expected once we destroy the socket + }); + req.on('error', () => {}); // ECONNRESET after our own destroy — expected + req.on('close', () => resolve()); + setTimeout(() => reject(new Error('timed out waiting for the aborted request to close')), 5000); + }); + + // Give the server's pipeline() rejection handler a tick to settle and + // the source's _destroy to run. + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(outcome.settled).toBe('resolved'); // swallowed, not a 500/rejection + expect(outcome.error).toBeUndefined(); + expect(fx.lastStream()?.wasDestroyed).toBe(true); // the leak fix + expect(unhandled).toEqual([]); // the audit's "does it take the process down" question + + // The process/server is still alive and serving — prove it with one + // more full, un-aborted request. + const finalBody: string = await new Promise((resolve, reject) => { + http + .get(`${baseUrl}/slow`, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + res.on('error', reject); + }) + .on('error', reject); + }); + expect(finalBody.length).toBe(40 * 2048); + }, 10000); + }); + + // CRITICAL regression case (task C3 review): a source failure BEFORE any + // byte reaches the client — no client abort involved at all — must still + // reject sendToResponse over a real connection, so the route's own + // miss/error handling actually runs instead of silently answering a + // truncated 200 with an empty body. + // + // Note on what's actually observable here: node:stream/promises' + // `pipeline()` destroys EVERY stream in the pipeline on any single + // failure — including the destination (`res`) — as part of its own + // cleanup, confirmed empirically (`dst.destroyed === true` by the time + // pipeline's rejection is caught, even for a purely source-side, pre-any- + // write error). So a caller's own fallback write attempted from its catch + // block (`res.status(502).json(...)`) is reliably a safe, silent no-op — + // Node's OutgoingMessage swallows writes after destroy rather than + // throwing — not a delivered response body. What this test can and does + // pin: the promise REJECTS (the miss/error contract genuinely runs, + // instead of being silently swallowed as if it were a client abort), that + // safe no-op write doesn't crash the process or the request, and the + // server survives to serve the next, unrelated request correctly. + describe('remote (pipeline) branch — pre-header source failure', () => { + let server: http.Server; + let baseUrl: string; + let outcome: { settled: 'resolved' | 'rejected' | 'pending'; error?: unknown }; + + beforeAll(async () => { + const fx = makeFailingRemoteFixture(); + const app = express(); + app.get('/broken', (_req, res) => { + outcome = { settled: 'pending' }; + fx.storage + .sendToResponse('files', 'remote.bin', res) + .then(() => { + outcome = { settled: 'resolved' }; + }) + .catch((error: unknown) => { + outcome = { settled: 'rejected', error }; + // The caller's own miss/error contract: prove it actually RUNS + // (as opposed to sendToResponse silently resolving and this + // branch never executing at all) — even though, per the note + // above, the write itself is a no-op onto an already-destroyed + // response. + if (!res.headersSent) res.status(502).json({ error: 'upstream failure' }); + }); + }); + app.get('/ok', (_req, res) => res.status(200).json({ ok: true })); + server = http.createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + it('rejects (the miss/error contract genuinely fires) instead of silently resolving as a fake empty success', async () => { + // The connection ends one way or another (a delivered body, or the + // destroyed-response no-op surfacing as a raw close) — either way, + // within the timeout, proving the server isn't stuck. + await new Promise((resolve, reject) => { + const req = http.get(`${baseUrl}/broken`, (res) => { + res.resume(); + res.on('end', () => resolve()); + res.on('error', () => resolve()); // a destroyed-response close is expected here + }); + req.on('error', () => resolve()); // socket hang up is expected here too + setTimeout(() => reject(new Error('timed out — the server got stuck')), 5000); + }); + + expect(outcome.settled).toBe('rejected'); // NOT silently swallowed as a client abort + expect((outcome.error as { code?: string } | undefined)?.code).toBe('ECONNRESET'); + expect(unhandled).toEqual([]); + + // The process/server survived and correctly serves an unrelated, + // ordinary request afterward — the real "did this take the process + // down" proof. + const ok: { status: number; body: unknown } = await new Promise((resolve, reject) => { + http + .get(`${baseUrl}/ok`, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode ?? 0, body: JSON.parse(Buffer.concat(chunks).toString('utf8')) })); + res.on('error', reject); + }) + .on('error', reject); + }); + expect(ok).toEqual({ status: 200, body: { ok: true } }); + }, 10000); + }); + + describe('local (res.sendFile) branch', () => { + let server: http.Server; + let baseUrl: string; + let fx: ReturnType; + let outcome: { settled: 'resolved' | 'rejected' | 'pending'; error?: unknown }; + + beforeAll(async () => { + fx = makeLocalFixture(); + // A large-enough file that writing it to the socket isn't instantaneous. + await fx.storage.put('files', 'big.bin', Readable.from(Buffer.alloc(20 * 1024 * 1024, 'y'))); + const app = express(); + app.get('/big', (_req, res) => { + outcome = { settled: 'pending' }; + fx.storage + .sendToResponse('files', 'big.bin', res) + .then(() => { + outcome = { settled: 'resolved' }; + }) + .catch((error: unknown) => { + outcome = { settled: 'rejected', error }; + }); + }); + server = http.createServer(app); + await new Promise((resolve) => server.listen(0, resolve)); + baseUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + it('resolves cleanly (no 500, no unhandled rejection) when the client aborts mid-download', async () => { + await new Promise((resolve, reject) => { + const req = http.get(`${baseUrl}/big`, (res) => { + res.once('data', () => { + req.destroy(); + }); + res.on('error', () => {}); + }); + req.on('error', () => {}); + req.on('close', () => resolve()); + setTimeout(() => reject(new Error('timed out waiting for the aborted request to close')), 5000); + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(outcome.settled).toBe('resolved'); + expect(outcome.error).toBeUndefined(); + expect(unhandled).toEqual([]); + }, 10000); + }); +}); diff --git a/server/tests/integration/trips.test.ts b/server/tests/integration/trips.test.ts index 4fd5d5c6e..b878a1eab 100644 --- a/server/tests/integration/trips.test.ts +++ b/server/tests/integration/trips.test.ts @@ -1279,3 +1279,84 @@ describe('Trip bundle', () => { expect(memberView.body.packingItems.map((i: { name: string }) => i.name)).toEqual(['Tent']); }); }); + +// ───────────────────────────────────────────────────────────────────────────── +// Cover upload parity (TRIP-P01…P05) — written BEFORE the storage-upload swap. +// ───────────────────────────────────────────────────────────────────────────── + +describe('Trip cover upload parity', () => { + const fsMod = require('fs') as typeof import('fs'); + const pathMod = require('path') as typeof import('path'); + const FIXTURE_IMG = pathMod.join(__dirname, '../fixtures/small-image.jpg'); + const coversDir = pathMod.join(__dirname, '../../uploads/covers'); + + afterAll(() => { + fsMod.rmSync(coversDir, { recursive: true, force: true }); + }); + + it('TRIP-P01 — cover upload returns /uploads/covers/ and writes the file', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG, 'cover.png'); + expect(res.status).toBe(201); + expect(res.body.cover_image).toMatch(/^\/uploads\/covers\/[0-9a-f-]{36}\.png$/); + const diskName = res.body.cover_image.replace('/uploads/covers/', ''); + expect(fsMod.existsSync(pathMod.join(coversDir, diskName))).toBe(true); + }); + + it('TRIP-P02 — re-upload deletes the previous cover file', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const first = await request(app) + .post(`/api/trips/${trip.id}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG, 'one.jpg'); + const firstName = first.body.cover_image.replace('/uploads/covers/', ''); + expect(fsMod.existsSync(pathMod.join(coversDir, firstName))).toBe(true); + + const second = await request(app) + .post(`/api/trips/${trip.id}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG, 'two.jpg'); + expect(second.status).toBe(201); + expect(fsMod.existsSync(pathMod.join(coversDir, firstName))).toBe(false); + }); + + it('TRIP-P03 — no file → 400 "No image uploaded"', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/cover`) + .set('Cookie', authCookie(user.id)); + expect(res.status).toBe(400); + expect(res.body.error).toBe('No image uploaded'); + }); + + it('TRIP-P04 — non-image upload is 500 (plain-Error filter quirk — pinned, do not "fix")', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/cover`) + .set('Cookie', authCookie(user.id)) + .attach('cover', Buffer.from('plain text'), { filename: 'doc.txt', contentType: 'text/plain' }); + expect(res.status).toBe(500); + }); + + it('TRIP-P05 — unknown trip → 404 "Trip not found"', async () => { + const { user } = createUser(testDb); + + const res = await request(app) + .post('/api/trips/999999/cover') + .set('Cookie', authCookie(user.id)) + .attach('cover', FIXTURE_IMG); + expect(res.status).toBe(404); + expect(res.body.error).toBe('Trip not found'); + }); +}); diff --git a/server/tests/integration/uploads-static.test.ts b/server/tests/integration/uploads-static.test.ts new file mode 100644 index 000000000..e96c0a60d --- /dev/null +++ b/server/tests/integration/uploads-static.test.ts @@ -0,0 +1,304 @@ +/** + * /uploads static-mount parity tests (UPLOADS-P01…P24). + * + * Written BEFORE the storage slice-3 swap of the four express.static mounts + * (avatars / covers / journey / places) for a storage-backed handler on the + * same URLs. P01–P21 pin today's express.static behavior — ETag / + * Last-Modified / conditional-GET / Range / HEAD, miss-falls-through to the + * Nest 404 envelope — and must stay green across the swap. P22–P24 are the + * approved deliberate deltas (D1–D3 in the slice-3 plan): RED against + * express.static, green against the storage handler. + * + * These routes are unauthenticated BY DESIGN (audit SEC-M9): server-chosen + * UUIDv4 filenames; gating them would break share-link trip cards, journey + * public pages, and email-embedded avatars. No auth setup here is deliberate. + */ +import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; +import request from 'supertest'; +import type { Application } from 'express'; +import type { INestApplication } from '@nestjs/common'; +import path from 'path'; +import fs from 'fs'; + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('PRAGMA busy_timeout = 5000'); + const mock = { + db, + closeDb: () => {}, + reinitialize: () => {}, + canAccessTrip: () => undefined, + isOwner: () => false, + }; + return { testDb: db, dbMock: mock }; +}); + +vi.mock('../../src/db/database', () => dbMock); +vi.mock('../../src/config', () => ({ + JWT_SECRET: 'test-jwt-secret-for-trek-testing-only', + ENCRYPTION_KEY: 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2', + updateJwtSecret: () => {}, + SESSION_DURATION: '24h', + SESSION_DURATION_MS: 86400000, + SESSION_DURATION_SECONDS: 86400, + DEFAULT_LANGUAGE: 'en', +})); +vi.mock('../../src/websocket', () => ({ broadcast: vi.fn(), broadcastToUser: vi.fn(), getOnlineUserIds: vi.fn(() => []) })); + +import { buildApp } from '../../src/bootstrap'; +import { createTables } from '../../src/db/schema'; +import { runMigrations } from '../../src/db/migrations'; + +let nestApp: INestApplication; +let app: Application; + +const uploadsDir = path.join(__dirname, '../../uploads'); +// 8 known bytes (the PNG magic) so Content-Length / Range math is exact. +const BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const NAME = 'uploads-static-parity.png'; + +// Every path this suite creates, for targeted afterAll cleanup — the uploads +// tree is shared with other integration workers, so no directory-wide rm of +// the category dirs themselves. +const created: string[] = []; + +function writeFixture(rel: string, bytes: Buffer = BYTES): void { + const abs = path.join(uploadsDir, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, bytes); + created.push(abs); +} + +beforeAll(async () => { + createTables(testDb); + runMigrations(testDb); + nestApp = await buildApp(); + app = nestApp.getHttpAdapter().getInstance(); + // Written AFTER buildApp(): LocalDriver init ensures + realpaths the + // category dirs at boot. + writeFixture(`avatars/${NAME}`); + writeFixture(`covers/${NAME}`); + writeFixture(`journey/thumbs/uploads-static-parity.jpg`); + writeFixture(`places/${NAME}`); + writeFixture('avatars/.uploads-static-parity-hidden'); + writeFixture('avatars/.tmp/uploads-static-parity-spooled.bin'); + fs.mkdirSync(path.join(uploadsDir, 'journey/parity-subdir'), { recursive: true }); +}); + +afterAll(async () => { + await nestApp.close(); + testDb.close(); + for (const abs of created) fs.rmSync(abs, { force: true }); + fs.rmSync(path.join(uploadsDir, 'avatars/.tmp'), { recursive: true, force: true }); + fs.rmSync(path.join(uploadsDir, 'journey/parity-subdir'), { recursive: true, force: true }); +}); + +describe('/uploads static parity — hit headers and bytes', () => { + it('UPLOADS-P01 — GET hit: 200 with express.static header set', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`); + expect(res.status).toBe(200); + expect(res.body.equals ? res.body.equals(BYTES) : Buffer.from(res.body).equals(BYTES)).toBe(true); + expect(res.headers.etag).toMatch(/^W\/"[0-9a-f]+-[0-9a-f]+"$/); + expect(res.headers['last-modified']).toBeDefined(); + expect(res.headers['cache-control']).toBe('public, max-age=0'); + expect(res.headers['accept-ranges']).toBe('bytes'); + expect(res.headers['content-type']).toBe('image/png'); + expect(res.headers['content-length']).toBe('8'); + }); + + it('UPLOADS-P02 — HEAD hit: 200, same headers, empty body', async () => { + const res = await request(app).head(`/uploads/avatars/${NAME}`); + expect(res.status).toBe(200); + expect(res.headers.etag).toMatch(/^W\/"[0-9a-f]+-[0-9a-f]+"$/); + expect(res.headers['content-length']).toBe('8'); + expect(res.headers['content-type']).toBe('image/png'); + expect(res.text ?? '').toBe(''); + }); + + it('UPLOADS-P19 — query string is ignored', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}?x=1`); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('8'); + }); + + it('UPLOADS-P20 — doubled leading slash still serves', async () => { + const res = await request(app).get(`/uploads/avatars//${NAME}`); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('8'); + }); +}); + +describe('/uploads static parity — conditional GET', () => { + it('UPLOADS-P03 — If-None-Match with the current ETag → 304', async () => { + const first = await request(app).get(`/uploads/avatars/${NAME}`); + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('If-None-Match', first.headers.etag); + expect(res.status).toBe(304); + expect(res.headers.etag).toBe(first.headers.etag); + expect(res.text ?? '').toBe(''); + }); + + it('UPLOADS-P04 — If-Modified-Since with the current Last-Modified → 304', async () => { + const first = await request(app).get(`/uploads/avatars/${NAME}`); + const res = await request(app) + .get(`/uploads/avatars/${NAME}`) + .set('If-Modified-Since', first.headers['last-modified']); + expect(res.status).toBe(304); + }); + + it('UPLOADS-P05 — stale If-None-Match → 200 full', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('If-None-Match', 'W/"0-0"'); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('8'); + }); + + it('UPLOADS-P06 — If-Match mismatch → 412', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('If-Match', '"mismatch"'); + expect(res.status).toBe(412); + }); +}); + +describe('/uploads static parity — Range', () => { + it('UPLOADS-P07 — single satisfiable range → 206 with Content-Range', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('Range', 'bytes=0-3'); + expect(res.status).toBe(206); + expect(res.headers['content-range']).toBe('bytes 0-3/8'); + expect(res.headers['content-length']).toBe('4'); + expect(Buffer.from(res.body).equals(BYTES.subarray(0, 4))).toBe(true); + }); + + it('UPLOADS-P08 — unsatisfiable range → 416 with Content-Range */size', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('Range', 'bytes=999-'); + expect(res.status).toBe(416); + expect(res.headers['content-range']).toBe('bytes */8'); + }); + + it('UPLOADS-P09 — multi-range is ignored → 200 full', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('Range', 'bytes=0-0,4-5'); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('8'); + }); + + it('UPLOADS-P10 — non-bytes range unit is ignored → 200 full', async () => { + const res = await request(app).get(`/uploads/avatars/${NAME}`).set('Range', 'apples=0-1'); + expect(res.status).toBe(200); + }); + + it('UPLOADS-P11 — stale If-Range drops the range → 200 full', async () => { + const res = await request(app) + .get(`/uploads/avatars/${NAME}`) + .set('Range', 'bytes=0-3') + .set('If-Range', 'W/"0-0"'); + expect(res.status).toBe(200); + expect(res.headers['content-length']).toBe('8'); + }); + + it('UPLOADS-P12 — fresh If-Range honors the range → 206', async () => { + const first = await request(app).get(`/uploads/avatars/${NAME}`); + const res = await request(app) + .get(`/uploads/avatars/${NAME}`) + .set('Range', 'bytes=0-3') + .set('If-Range', first.headers.etag); + expect(res.status).toBe(206); + }); +}); + +describe('/uploads static parity — miss falls through to the Nest 404 envelope', () => { + it('UPLOADS-P13 — GET miss → Nest envelope, not a bespoke 404', async () => { + const res = await request(app).get('/uploads/avatars/nope.png'); + expect(res.status).toBe(404); + expect(res.body).toEqual({ error: 'Cannot GET /uploads/avatars/nope.png' }); + }); + + it('UPLOADS-P14 — HEAD miss → 404, empty body', async () => { + const res = await request(app).head('/uploads/avatars/nope.png'); + expect(res.status).toBe(404); + expect(res.text ?? '').toBe(''); + }); + + it('UPLOADS-P15 — POST to a hit URL falls through (no 405)', async () => { + const res = await request(app).post(`/uploads/avatars/${NAME}`); + expect(res.status).toBe(404); + expect(res.body.error).toBe(`Cannot POST /uploads/avatars/${NAME}`); + }); + + it('UPLOADS-P16 — malformed percent-encoding falls through', async () => { + const res = await request(app).get('/uploads/avatars/%ZZ'); + expect(res.status).toBe(404); + expect(res.body.error).toMatch(/^Cannot GET /); + }); + + it('UPLOADS-P17 — traversal shapes fall through, raw and encoded', async () => { + const raw = await request(app).get('/uploads/avatars/../covers/x.png'); + expect(raw.status).toBe(404); + const encoded = await request(app).get('/uploads/avatars/..%2Fcovers%2Fx.png'); + expect(encoded.status).toBe(404); + }); + + it('UPLOADS-P18 — dotfile basename falls through', async () => { + const res = await request(app).get('/uploads/avatars/.uploads-static-parity-hidden'); + expect(res.status).toBe(404); + expect(res.body.error).toBe('Cannot GET /uploads/avatars/.uploads-static-parity-hidden'); + }); +}); + +describe('/uploads static parity — all four mounts', () => { + it('UPLOADS-P21 — covers/journey(nested)/places each serve a hit and 404 a miss', async () => { + // Fixtures are (re)written immediately before each request: other suites + // rm their whole category dir in afterAll (collections/trips → covers, + // collections → places, journey.test → journey), so a beforeAll-written + // file can vanish mid-run when the tier runs in parallel workers. + writeFixture(`covers/${NAME}`); + const coverHit = await request(app).get(`/uploads/covers/${NAME}`); + expect(coverHit.status).toBe(200); + const coverMiss = await request(app).get('/uploads/covers/nope.png'); + expect(coverMiss.status).toBe(404); + expect(coverMiss.body.error).toBe('Cannot GET /uploads/covers/nope.png'); + + // Nested name — journey thumbs live at journey/thumbs/.jpg and are + // served through the same mount; pins multi-segment storage keys. + writeFixture('journey/thumbs/uploads-static-parity.jpg'); + const journeyHit = await request(app).get('/uploads/journey/thumbs/uploads-static-parity.jpg'); + expect(journeyHit.status).toBe(200); + expect(journeyHit.headers['content-type']).toBe('image/jpeg'); + const journeyMiss = await request(app).get('/uploads/journey/thumbs/nope.jpg'); + expect(journeyMiss.status).toBe(404); + + writeFixture(`places/${NAME}`); + const placeHit = await request(app).get(`/uploads/places/${NAME}`); + expect(placeHit.status).toBe(200); + const placeMiss = await request(app).get('/uploads/places/nope.png'); + expect(placeMiss.status).toBe(404); + }); +}); + +describe('/uploads static — approved deliberate deltas (RED against express.static)', () => { + // D3 (slice-3 plan): send's legacy dotfiles mode only ignores dot BASENAMES, + // so a file under a dot-directory is servable today. Central storage key + // validation rejects every dot segment — the spec's ".tmp must remain + // unservable" pin. + it('UPLOADS-P22 — a file under a dot-directory is NOT servable', async () => { + const res = await request(app).get('/uploads/avatars/.tmp/uploads-static-parity-spooled.bin'); + expect(res.status).toBe(404); + expect(res.body.error).toBe('Cannot GET /uploads/avatars/.tmp/uploads-static-parity-spooled.bin'); + }); + + // D1/D2: express.static answers directory URLs with a 301 trailing-slash + // redirect (and an index.html lookup on the slashed form). The storage + // handler treats both as a plain miss. + it('UPLOADS-P23 — mount root (with and without slash) is a miss', async () => { + const bare = await request(app).get('/uploads/avatars'); + expect(bare.status).toBe(404); + expect(bare.body.error).toBe('Cannot GET /uploads/avatars'); + const slashed = await request(app).get('/uploads/avatars/'); + expect(slashed.status).toBe(404); + }); + + it('UPLOADS-P24 — a real subdirectory URL is a miss, not a redirect', async () => { + const res = await request(app).get('/uploads/journey/parity-subdir'); + expect(res.status).toBe(404); + expect(res.body.error).toBe('Cannot GET /uploads/journey/parity-subdir'); + }); +}); diff --git a/server/tests/unit/app-config/derive.test.ts b/server/tests/unit/app-config/derive.test.ts index c669de46e..7886637ed 100644 --- a/server/tests/unit/app-config/derive.test.ts +++ b/server/tests/unit/app-config/derive.test.ts @@ -12,6 +12,7 @@ import { deriveIntegrations, deriveBackup, deriveNet, + derivePaths, deriveAll, } from '../../../src/app-config/derive'; @@ -221,6 +222,15 @@ describe('deriveNet', () => { }); }); +describe('derivePaths', () => { + it('passes the path vars through raw — defaulting stays at the consumer', () => { + expect(derivePaths({ TREK_WIKI_DIR: '/w' }).wikiDir).toBe('/w'); + expect(derivePaths({ TREK_PLACE_PHOTO_DIR: '/p' }).placePhotoDir).toBe('/p'); + expect(derivePaths({}).wikiDir).toBeUndefined(); + expect(derivePaths({}).placePhotoDir).toBeUndefined(); + }); +}); + describe('deriveAll', () => { it('assembles every namespace', () => { const env = deriveAll({ PORT: '4000', DEMO_MODE: 'true' }); diff --git a/server/tests/unit/app-config/parsers.test.ts b/server/tests/unit/app-config/parsers.test.ts index 21b02dfb1..8070e11e5 100644 --- a/server/tests/unit/app-config/parsers.test.ts +++ b/server/tests/unit/app-config/parsers.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect } from 'vitest'; import { csvList, csvListFiltered, + nonNegativeIntOr, numberOr, parseBool, parseDurationMs, @@ -80,6 +81,19 @@ describe('positiveIntOr', () => { }); }); +describe('nonNegativeIntOr', () => { + it('accepts zero (0 disables retries) and positive integers', () => { + expect(nonNegativeIntOr('0', 1)).toBe(0); + expect(nonNegativeIntOr('3', 1)).toBe(3); + }); + it('falls back on unset, blank, negative, and non-numeric', () => { + expect(nonNegativeIntOr(undefined, 1)).toBe(1); + expect(nonNegativeIntOr(' ', 1)).toBe(1); + expect(nonNegativeIntOr('-1', 1)).toBe(1); + expect(nonNegativeIntOr('nope', 1)).toBe(1); + }); +}); + describe('csvList / csvListFiltered', () => { it('returns null when unset or empty', () => { expect(csvList(undefined)).toBeNull(); diff --git a/server/tests/unit/auto-backup.test.ts b/server/tests/unit/auto-backup.test.ts index 89b1e1bb2..248ca8912 100644 --- a/server/tests/unit/auto-backup.test.ts +++ b/server/tests/unit/auto-backup.test.ts @@ -65,10 +65,29 @@ import path from 'node:path'; import { AutoBackupJob } from '../../src/nest/backup/auto-backup.job'; import { createBackup } from '../../src/nest/backup/backup.impl'; import type { BackupService } from '../../src/nest/backup/backup.service'; +import type { StorageService } from '../../src/nest/storage/storage.service'; import type { CronRegistrarService } from '../../src/nest/scheduling/cron-registrar.service'; const liveDb = path.join(__dirname, '../../data', 'travel.db'); +// createBackup receives StorageService as a parameter (BackupService injects +// it). fs is mocked wholesale here, so a plain stub stands in for the backups +// backend: staging under /stub/spool, empty category listings, put/stat succeed. +const storageStub = vi.hoisted(() => ({}) as Record); +function resetStorageStub() { + Object.assign(storageStub, { + spoolDirFor: vi.fn(() => '/stub/spool'), + tempDir: vi.fn(() => '/stub/tmp'), + list: vi.fn(async function* () {}), + withLocalFile: vi.fn(async (_c: string, _k: string, fn: (p: string) => Promise) => fn('/stub/local/path')), + put: vi.fn(async () => {}), + stat: vi.fn(async (_c: string, key: string) => ({ key, size: 4096, mtimeMs: Date.parse('2026-04-27T02:00:00Z') })), + exists: vi.fn(async () => true), + delete: vi.fn(async () => {}), + }); +} +resetStorageStub(); + interface Registered { name: string; expr: string; @@ -88,9 +107,11 @@ function makeJob() { }; const job = new AutoBackupJob( // The same wiring the container does, with the real service function behind - // it — the run below is still the production code path. - { createBackup } as unknown as BackupService, + // it — the run below is still the production code path. The forward mirrors + // BackupService: inject the storage stub as the first argument. + { createBackup: (prefix?: 'backup' | 'auto-backup') => createBackup(storageStub as unknown as StorageService, prefix) } as unknown as BackupService, registrar as unknown as CronRegistrarService, + storageStub as unknown as StorageService, ); return { job, registered, registrar }; } @@ -121,6 +142,7 @@ describe('auto-backup run', () => { beforeEach(() => { vi.clearAllMocks(); + resetStorageStub(); // No ENCRYPTION_KEY in the env means the key file is the source of truth and // has to travel with the backup. delete process.env.ENCRYPTION_KEY; @@ -140,7 +162,7 @@ describe('auto-backup run', () => { expect(dbMock.db.exec).toHaveBeenCalledWith(expect.stringContaining('VACUUM INTO')); const dbEntry = archiveMock.file.mock.calls.find(([, opts]) => opts?.name === 'travel.db'); expect(dbEntry).toBeDefined(); - expect(dbEntry?.[0]).toContain('.travel-snap-auto-backup-'); + expect(dbEntry?.[0]).toContain('/stub/spool/travel-snap-auto-backup-'); expect(dbEntry?.[0]).not.toBe(liveDb); }); @@ -158,15 +180,21 @@ describe('auto-backup run', () => { it('keeps the auto-backup-*.zip naming so retention still prunes it', async () => { fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('backup-settings.json')); - fsMock.readdirSync.mockReturnValue(['auto-backup-2020-01-01T02-00-00.zip'] as unknown as string[]); + // An expired scheduled archive sits in the backups category (retention + // lists through storage now); the category enumeration stays empty. + storageStub.list = vi.fn((category: string) => + (async function* () { + if (category === 'backups') yield { key: 'auto-backup-2020-01-01T02-00-00.zip', size: 0, mtimeMs: 0 }; + })(), + ); stubArchiver(); await scheduledRun()(); expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringMatching(/^Auto-Backup created: auto-backup-[\dT-]+\.zip$/)); - // cleanupOldBackups(keep_days) still runs after the archive and only sees - // files it can match by prefix - expect(fsMock.unlinkSync).toHaveBeenCalledWith(expect.stringContaining('auto-backup-2020-01-01T02-00-00.zip')); + // cleanupOldBackups(storage, keep_days) still runs after the archive and + // only prunes objects it can match by prefix — through storage.delete + expect(storageStub.delete).toHaveBeenCalledWith('backups', 'auto-backup-2020-01-01T02-00-00.zip'); }); it('logs the failure, drops the partial zip and skips retention', async () => { @@ -178,16 +206,21 @@ describe('auto-backup run', () => { expect(logMock.logError).toHaveBeenCalledWith('Auto-Backup: disk full'); expect(logMock.logInfo).not.toHaveBeenCalledWith(expect.stringContaining('Auto-Backup created')); - expect(fsMock.unlinkSync).toHaveBeenCalledWith(expect.stringContaining('auto-backup-')); - // the snapshot scratch file is cleaned up too, and retention never runs - expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('.travel-snap-auto-backup-'), { force: true }); - expect(fsMock.readdirSync).not.toHaveBeenCalled(); + // nothing was committed (put commits by rename only on success); the + // half-built zip spool and snapshot scratch are cleaned up, and retention + // never runs + expect(storageStub.put).not.toHaveBeenCalled(); + expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('zip-build-auto-backup-'), { force: true }); + expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('travel-snap-auto-backup-'), { force: true }); + expect(storageStub.list).not.toHaveBeenCalledWith('backups'); + expect(storageStub.delete).not.toHaveBeenCalled(); }); }); describe('auto-backup scheduling (AutoBackupJob.start)', () => { beforeEach(() => { vi.clearAllMocks(); + resetStorageStub(); fsMock.existsSync.mockImplementation(() => false); fsMock.readFileSync.mockReturnValue('{}'); }); @@ -249,13 +282,14 @@ describe('auto-backup scheduling (AutoBackupJob.start)', () => { await (registered.at(-1)?.onTick as () => Promise)(); expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringContaining('Auto-Backup created')); - expect(fsMock.readdirSync).not.toHaveBeenCalled(); // cleanupOldBackups never ran + expect(storageStub.list).not.toHaveBeenCalledWith('backups'); // cleanupOldBackups never ran + expect(storageStub.delete).not.toHaveBeenCalled(); }); it('a non-Error rejection from createBackup is stringified into the failure log', async () => { const broken = { createBackup: vi.fn().mockRejectedValue('plain string') } as unknown as BackupService; const registrar = { isEnabled: vi.fn(() => true), register: vi.fn(() => true), unregister: vi.fn() }; - const job = new AutoBackupJob(broken, registrar as unknown as CronRegistrarService); + const job = new AutoBackupJob(broken, registrar as unknown as CronRegistrarService, storageStub as unknown as StorageService); await job.runBackup(); expect(logMock.logError).toHaveBeenCalledWith('Auto-Backup: plain string'); }); @@ -286,6 +320,7 @@ describe('auto-backup scheduling (AutoBackupJob.start)', () => { describe('BACKUP-047 updateAutoSettings', () => { beforeEach(() => { vi.clearAllMocks(); + resetStorageStub(); fsMock.existsSync.mockImplementation(() => false); fsMock.readFileSync.mockReturnValue('{}'); }); diff --git a/server/tests/unit/db/migrate-encryption-parity.test.ts b/server/tests/unit/db/migrate-encryption-parity.test.ts index 7b0a9584a..3f05bde1c 100644 --- a/server/tests/unit/db/migrate-encryption-parity.test.ts +++ b/server/tests/unit/db/migrate-encryption-parity.test.ts @@ -10,10 +10,16 @@ * key, decrypt_api_key returns null on the next read, and the instance quietly * behaves as if no key were configured. This pins the copy instead. */ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; +import Database from 'better-sqlite3'; import { INSTANCE_API_KEY_NAMES } from '../../../src/nest/settings/instance-api-keys'; +import { storageSecretFields } from '@trek/shared'; + +const SERVER_ROOT = path.join(__dirname, '..', '..', '..'); const script = fs.readFileSync( path.join(__dirname, '..', '..', '..', 'scripts', 'migrate-encryption.ts'), @@ -30,6 +36,17 @@ const appSettingsLoop = (() => { return script.slice(start, end); })(); +// The storage.backends section only — bounded by its own marker and the next +// one (`--- settings:`), so a field name appearing elsewhere in the script +// cannot satisfy the test. +const storageBackendsSection = (() => { + const start = script.indexOf('--- app_settings: storage.backends ---'); + expect(start).toBeGreaterThan(-1); + const end = script.indexOf('--- settings:', start); + expect(end).toBeGreaterThan(start); + return script.slice(start, end); +})(); + describe('migrate-encryption.ts app_settings parity', () => { it('ROTPAR-001: rotates every instance-wide API key', () => { for (const name of INSTANCE_API_KEY_NAMES) { @@ -46,4 +63,72 @@ describe('migrate-encryption.ts app_settings parity', () => { it('ROTPAR-003: the canonical list is not empty, so ROTPAR-001 cannot pass vacuously', () => { expect(INSTANCE_API_KEY_NAMES.length).toBeGreaterThan(0); }); + + it('ROTPAR-004: rotates every storage.backends secret field for s3', () => { + const fields = storageSecretFields('s3'); + expect(fields.length).toBeGreaterThan(0); + for (const field of fields) { + expect(storageBackendsSection).toContain(`'${field}'`); + } + }); +}); + +describe('migrate-encryption.ts storage.backends: malformed shape', () => { + let tmpDir: string; + let dbPath: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-migrate-encryption-')); + dbPath = path.join(tmpDir, 'travel.db'); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('ROTPAR-005: reports an error instead of silently skipping a non-array storage.backends value', () => { + const seed = new Database(dbPath); + seed.exec(` + CREATE TABLE users ( + id INTEGER PRIMARY KEY, + maps_api_key TEXT, unsplash_api_key TEXT, openweather_api_key TEXT, + immich_api_key TEXT, synology_password TEXT, synology_sid TEXT, + synology_did TEXT, mfa_secret TEXT + ); + CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE settings (user_id INTEGER, key TEXT, value TEXT); + CREATE TABLE trip_album_links (id INTEGER PRIMARY KEY, passphrase TEXT); + CREATE TABLE trek_photos (id INTEGER PRIMARY KEY, passphrase TEXT); + `); + // Valid JSON, but not an array — the malformed shape the fix must not + // silently skip. + seed.prepare('INSERT INTO app_settings (key, value) VALUES (?, ?)').run( + 'storage.backends', + JSON.stringify({ not: 'an array' }), + ); + seed.close(); + + let output: string; + let threw = false; + try { + const stdout = execFileSync(process.execPath, ['--import', 'tsx', 'scripts/migrate-encryption.ts'], { + cwd: SERVER_ROOT, + env: { ...process.env, DB_PATH: dbPath }, + input: 'old-key\nnew-key\nyes\n', + stdio: 'pipe', + encoding: 'utf8', + }); + output = stdout; + } catch (err) { + // A non-empty result.errors makes the script exit(1) — the SUT's + // documented posture for "some secrets could not be migrated". The + // error itself is written via console.warn (stderr), not stdout. + threw = true; + const e = err as { stdout?: string; stderr?: string }; + output = `${e.stdout ?? ''}${e.stderr ?? ''}`; + } + + expect(threw).toBe(true); + expect(output).toContain('app_settings.storage.backends: not an array — skipping'); + }, 30000); }); diff --git a/server/tests/unit/db/trip-files-prefix-migration.test.ts b/server/tests/unit/db/trip-files-prefix-migration.test.ts new file mode 100644 index 000000000..e47e4317b --- /dev/null +++ b/server/tests/unit/db/trip-files-prefix-migration.test.ts @@ -0,0 +1,43 @@ +/** + * Boot migration: strip the legacy 'files/' prefix from trip_files.filename. + * + * Collab note attachments historically stored 'files/' while the file + * manager stored bare names in the same column; the storage layer addresses + * objects as category + bare name, so existing prefixed rows are normalized + * once at boot (storage slice 2). + */ +import { describe, it, expect } from 'vitest'; +import Database from 'better-sqlite3'; +import { createTables } from '../../../src/db/schema'; +import { runMigrations } from '../../../src/db/migrations'; + +function makeDbWithRows(): Database.Database { + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA busy_timeout = 5000'); + db.exec('PRAGMA foreign_keys = ON'); + createTables(db); + // Minimal FK chain: user → trip → trip_files rows. + db.prepare("INSERT INTO users (id, username, email, password_hash) VALUES (1, 'u', 'u@example.test', 'x')").run(); + db.prepare("INSERT INTO trips (id, user_id, title) VALUES (1, 1, 'T')").run(); + db.prepare( + "INSERT INTO trip_files (trip_id, filename, original_name) VALUES (1, 'files/aaa.pdf', 'a.pdf'), (1, 'bbb.pdf', 'b.pdf')", + ).run(); + return db; +} + +describe('trip_files files/-prefix migration', () => { + it('strips the files/ prefix from legacy collab rows and leaves bare rows alone', () => { + const db = makeDbWithRows(); + try { + runMigrations(db); + const names = db + .prepare('SELECT filename FROM trip_files ORDER BY id') + .all() + .map((r) => (r as { filename: string }).filename); + expect(names).toEqual(['aaa.pdf', 'bbb.pdf']); + } finally { + db.close(); + } + }); +}); diff --git a/server/tests/unit/files/files.rpc.test.ts b/server/tests/unit/files/files.rpc.test.ts index 1eec851d1..8d8ad1f08 100644 --- a/server/tests/unit/files/files.rpc.test.ts +++ b/server/tests/unit/files/files.rpc.test.ts @@ -1,16 +1,17 @@ /** * The file plugin surface after it moved onto @PluginMethod. * - * Files are the only plugin surface that touches disk and the only one that uses - * three separate rights, so the cases here lean on the upload path: the extension - * blocklist, the size cap, the demo-mode refusal, and that each operation asks for - * its own permission rather than one shared domain action. + * Files are the only plugin surface with three separate rights (one per write + * operation), and both its byte-paths — read and write — go through the storage + * layer rather than touching disk directly. The cases here lean on the upload + * path: the extension blocklist, the size cap, the demo-mode refusal, that each + * operation asks for its own permission rather than one shared domain action, and + * that a create pins the validate -> put -> insert -> broadcast order so a failed + * put can never leave a DB row pointing at missing bytes. */ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { expectRegisteredProvider } from '../../helpers/module-providers'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; +import { Readable } from 'node:stream'; import { PluginRpcHost } from '../../../src/nest/plugins/host/rpc-host'; import { createTestPluginRegistry } from '../../../src/nest/plugins/host/rpc-kit/testing'; import { PluginGuards } from '../../../src/nest/plugins/host/plugin-guards.service'; @@ -21,6 +22,8 @@ import type { RealtimeService } from '../../../src/nest/realtime/realtime.servic import type { DatabaseService } from '../../../src/nest/database/database.service'; import type { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import type { AddonsService } from '../../../src/nest/addons/addons.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; +import { StorageNotFoundError, StorageInvalidKeyError } from '../../../src/nest/storage/storage.types'; import type { RpcRequest, RpcError, RpcResponse } from '../../../src/nest/plugins/protocol/envelope'; import { makeDeps } from '../../helpers/rpc-host-deps'; @@ -28,8 +31,6 @@ const req = (method: string, params: Record = {}): RpcRequest = const b64 = (s: string) => Buffer.from(s).toString('base64'); -let tmpDir: string; - /** File 2 sits on trip 1, which belongs to user 42. */ function build(opts: { file?: Record | undefined; foreign?: string | null; allow?: (a: string) => boolean } = {}) { const realtime = { broadcast: vi.fn() } as unknown as RealtimeService & { broadcast: ReturnType }; @@ -38,7 +39,6 @@ function build(opts: { file?: Record | undefined; foreign?: str getFileById: vi.fn((id: number) => id === 2 ? (opts.file ?? { filename: 'visa.pdf', original_name: 'visa.pdf', mime_type: 'application/pdf', file_size: 2, deleted_at: null }) : undefined, ), - resolveFilePath: vi.fn((name: string) => ({ resolved: path.join(tmpDir, name), safe: true })), findForeignLinkTarget: vi.fn(() => opts.foreign ?? null), createFile: vi.fn((_t: number, meta: Record) => ({ id: 130, ...meta })), createFileLink: vi.fn(() => [{ file_id: 2 }]), @@ -53,17 +53,18 @@ function build(opts: { file?: Record | undefined; foreign?: str checkPermission: vi.fn((action: string) => (opts.allow ? opts.allow(action) : true)), } as unknown as PermissionsService; const guards = new PluginGuards(db, permissions, { isAddonEnabled: vi.fn(() => true) } as unknown as AddonsService); - const rpc = new FilesRpc(files, realtime, db, guards); + const storage = { + getStream: vi.fn(async () => ({ + stream: Readable.from(Buffer.from('hi')), + stat: { key: 'visa.pdf', size: 2, mtimeMs: 0 }, + })), + put: vi.fn(async () => undefined), + } as unknown as StorageService & Record>; + const rpc = new FilesRpc(files, realtime, db, guards, storage); const host = (...grants: string[]) => new PluginRpcHost('p', new Set(grants), makeDeps(), createTestPluginRegistry([rpc])); - return { files, realtime, permissions, host }; + return { files, realtime, permissions, storage, host }; } -beforeEach(() => { - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-files-rpc-')); - fs.writeFileSync(path.join(tmpDir, 'visa.pdf'), 'hi'); -}); -afterEach(() => fs.rmSync(tmpDir, { recursive: true, force: true })); - describe('FilesRpc reads', () => { it('FILES-RPC-001 files.list is membership-checked and excludes the trash', async () => { const f = build(); @@ -93,16 +94,48 @@ describe('FilesRpc reads', () => { const f = build({ file: { filename: 'visa.pdf', original_name: 'visa.pdf', mime_type: null, file_size: 400 * 1024 * 1024, deleted_at: null } }); const res = (await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42)) as RpcError; expect(res.error.code).toBe('BAD_PARAMS'); - // The path is never even resolved for an oversized file. - expect(f.files.resolveFilePath).not.toHaveBeenCalled(); + // The storage layer is never even asked for an oversized file. + expect(f.storage.getStream).not.toHaveBeenCalled(); }); - it('FILES-RPC-005 a path outside the uploads root is refused', async () => { + it('FILES-RPC-005 a missing storage object gets the accessibility envelope, not a raw error', async () => { const f = build(); - (f.files.resolveFilePath as ReturnType).mockReturnValueOnce({ resolved: '/etc/passwd', safe: false }); + (f.storage.getStream as ReturnType).mockRejectedValueOnce(new StorageNotFoundError('files/visa.pdf')); const res = (await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42)) as RpcError; expect(res.error.message).toBe('file path is not accessible'); }); + + it('FILES-RPC-005d an invalid storage key gets the accessibility envelope, not a raw error', async () => { + const f = build(); + (f.storage.getStream as ReturnType).mockRejectedValueOnce(new StorageInvalidKeyError('..')); + const res = (await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42)) as RpcError; + expect(res.error.message).toBe('file path is not accessible'); + }); + + it('FILES-RPC-005b an object whose stat exceeds the cap is refused before buffering', async () => { + const f = build(); + const stream = Readable.from(Buffer.from('hi')); + (f.storage.getStream as ReturnType).mockResolvedValueOnce({ + stream, stat: { key: 'visa.pdf', size: 11 * 1024 * 1024, mtimeMs: 0 }, + }); + const res = (await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42)) as RpcError; + expect(res.error.code).toBe('BAD_PARAMS'); + expect(res.error.message).toBe('file too large to read (>10485760 bytes); use the download UI'); + expect(stream.destroyed).toBe(true); + }); + + it('FILES-RPC-005c a stream that outgrows its stat is aborted mid-read', async () => { + const f = build(); + // stat claims 2 bytes; the stream actually yields 10MB+1 — a lying driver + // must not push an oversized payload through the IPC pipe. + const big = Readable.from([Buffer.alloc(6 * 1024 * 1024), Buffer.alloc(6 * 1024 * 1024)]); + (f.storage.getStream as ReturnType).mockResolvedValueOnce({ + stream: big, stat: { key: 'visa.pdf', size: 2, mtimeMs: 0 }, + }); + const res = (await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42)) as RpcError; + expect(res.error.message).toBe('file too large to read'); + expect(big.destroyed).toBe(true); + }); }); describe('FilesRpc writes', () => { @@ -221,7 +254,8 @@ describe('FilesRpc writes', () => { findForeignLinkTarget: vi.fn(() => null), createFile: vi.fn(() => ({ id: 130 })), } as unknown as FilesService; - const rpc = new FilesRpc(files, { broadcast: vi.fn() } as unknown as RealtimeService, db, guards); + const storage = { getStream: vi.fn(), put: vi.fn(async () => undefined) } as unknown as StorageService; + const rpc = new FilesRpc(files, { broadcast: vi.fn() } as unknown as RealtimeService, db, guards, storage); const host = new PluginRpcHost('p', new Set(['db:write:files']), makeDeps(), createTestPluginRegistry([rpc])); const input = { name: 'a.pdf', content_base64: b64('x') }; const denied = (await host.dispatch(req('files.create', { tripId: 1, input }), 9)) as RpcError; @@ -247,6 +281,47 @@ describe('FilesRpc writes', () => { expect(f.files.createFile).toHaveBeenCalledWith(1, expect.anything(), 42, { place_id: '7', reservation_id: '9', description: 'd' }); }); + it('FILES-RPC-018 create puts the bytes into the storage layer before the DB row', async () => { + const f = build(); + const order: string[] = []; + (f.storage.put as ReturnType).mockImplementation(async () => { order.push('put'); }); + (f.files.createFile as ReturnType).mockImplementation(() => { order.push('insert'); return { id: 130 }; }); + const res = await f.host('db:write:files').dispatch( + req('files.create', { tripId: 1, input: { name: 'a.pdf', content_base64: b64('x'), mimetype: 'application/pdf' } }), 42, + ); + expect(res.ok).toBe(true); + expect(order).toEqual(['put', 'insert']); + const [category, name, source, opts] = (f.storage.put as ReturnType).mock.calls[0]; + expect(category).toBe('files'); + expect(name).toMatch(/^[0-9a-f-]{36}\.pdf$/); + expect(source).toBeInstanceOf(Readable); + expect(opts).toEqual({ contentType: 'application/pdf' }); + }); + + it('FILES-RPC-019 create defaults the stored contentType like the returned mimetype', async () => { + const f = build(); + await f.host('db:write:files').dispatch(req('files.create', { tripId: 1, input: { name: 'a.pdf', content_base64: b64('x') } }), 42); + const [, , , opts] = (f.storage.put as ReturnType).mock.calls[0]; + expect(opts).toEqual({ contentType: 'application/octet-stream' }); + }); + + it('FILES-RPC-020 a failed put creates no DB row and broadcasts nothing', async () => { + const f = build(); + (f.storage.put as ReturnType).mockRejectedValueOnce(new Error('backend down')); + const res = (await f.host('db:write:files').dispatch( + req('files.create', { tripId: 1, input: { name: 'a.pdf', content_base64: b64('x') } }), 42, + )) as RpcError; + expect(res.error).toBeDefined(); + expect(f.files.createFile).not.toHaveBeenCalled(); + expect(f.realtime.broadcast).not.toHaveBeenCalled(); + }); + + it('FILES-RPC-021 a refused create never reaches the storage layer', async () => { + const f = build(); + await f.host('db:write:files').dispatch(req('files.create', { tripId: 1, input: { name: 'evil.exe', content_base64: b64('MZ') } }), 42); + expect(f.storage.put).not.toHaveBeenCalled(); + }); + it('FILES-RPC-016e a file with no recorded size or mimetype still reads', async () => { const f = build({ file: { filename: 'visa.pdf', original_name: 'visa.pdf', mime_type: null, file_size: null, deleted_at: null } }); const res = await f.host('db:read:files:content').dispatch(req('files.getContent', { tripId: 1, fileId: 2 }), 42); diff --git a/server/tests/unit/middleware/globalMiddleware.redaction.test.ts b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts new file mode 100644 index 000000000..63f116b1a --- /dev/null +++ b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest'; +import { redact, SENSITIVE_KEYS } from '../../../src/middleware/globalMiddleware'; + +describe('globalMiddleware request-log redaction', () => { + it('redacts secretAccessKey (any casing) — storage admin PUT bodies land in the same debug log line', () => { + expect(redact({ secretAccessKey: 'sk-super-secret' })).toEqual({ secretAccessKey: '[REDACTED]' }); + expect(redact({ SecretAccessKey: 'sk-super-secret' })).toEqual({ SecretAccessKey: '[REDACTED]' }); + expect(redact({ secretaccesskey: 'sk-super-secret' })).toEqual({ secretaccesskey: '[REDACTED]' }); + }); + + it('lists secretaccesskey in the lowercase lookup set (lookup lowercases the key first)', () => { + expect(SENSITIVE_KEYS.has('secretaccesskey')).toBe(true); + }); + + it('still redacts every pre-existing sensitive key (no regression)', () => { + const input = { + password: 'p', + new_password: 'p', + current_password: 'p', + token: 't', + jwt: 'j', + authorization: 'a', + cookie: 'c', + client_secret: 's', + mfa_token: 'm', + code: '123456', + smtp_pass: 's', + }; + const out = redact(input) as Record; + for (const key of Object.keys(input)) { + expect(out[key]).toBe('[REDACTED]'); + } + }); + + it('redacts nested objects and arrays, leaving non-sensitive fields untouched', () => { + const out = redact({ + name: 'off-box', + options: { secretAccessKey: 'sk-1', accessKeyId: 'ak-1' }, + backends: [{ secretAccessKey: 'sk-2' }, { accessKeyId: 'ak-2' }], + }); + expect(out).toEqual({ + name: 'off-box', + options: { secretAccessKey: '[REDACTED]', accessKeyId: 'ak-1' }, + backends: [{ secretAccessKey: '[REDACTED]' }, { accessKeyId: 'ak-2' }], + }); + }); + + it('passes non-object values through untouched', () => { + expect(redact('plain string')).toBe('plain string'); + expect(redact(42)).toBe(42); + expect(redact(null)).toBe(null); + expect(redact(undefined)).toBe(undefined); + }); +}); diff --git a/server/tests/unit/nest/admin.service.test.ts b/server/tests/unit/nest/admin.service.test.ts index e0e2139cf..fe84dd6d6 100644 --- a/server/tests/unit/nest/admin.service.test.ts +++ b/server/tests/unit/nest/admin.service.test.ts @@ -80,6 +80,7 @@ import type { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env import { __clearVersionCacheForTests } from '../../../src/nest/admin/admin.helpers'; import { makeNotificationsService, makeNotificationPreferencesService } from '../../helpers/notifications'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; const dbs = new DatabaseService(testDb); const realtime = new RealtimeService(); @@ -90,7 +91,7 @@ const userCleanup = new UserCleanupService(dbs, new BudgetService(dbs, permissio // and the mailer was missing entirely, so `auth` was built with its last four // collaborators shifted by one. Nothing failed, because none of the cases below // reach a path that uses them. -const auth = new AuthService(dbs, permissions, new TripMembershipService(dbs), webauthn, userCleanup, new MailerService(dbs), new EphemeralTokenService()); +const auth = new AuthService(dbs, permissions, new TripMembershipService(dbs), webauthn, userCleanup, new MailerService(dbs), new EphemeralTokenService(), new AllowedFileTypesService(dbs)); const svc = new AdminService( dbs, new AddonsService(dbs), diff --git a/server/tests/unit/nest/admin.version-notification.test.ts b/server/tests/unit/nest/admin.version-notification.test.ts index 9710104ae..fc514e145 100644 --- a/server/tests/unit/nest/admin.version-notification.test.ts +++ b/server/tests/unit/nest/admin.version-notification.test.ts @@ -59,6 +59,7 @@ import { NotificationsService } from '../../../src/nest/notifications/notificati import { AdminService } from '../../../src/nest/admin/admin.service'; import { makeNotificationsService, makeNotificationPreferencesService } from '../../helpers/notifications'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; const dbs = new DatabaseService(testDb); const realtime = new RealtimeService(); @@ -69,7 +70,7 @@ const userCleanup = new UserCleanupService(dbs, new BudgetService(dbs, permissio // and the mailer was missing entirely, so `auth` was built with its last four // collaborators shifted by one. Nothing failed, because the version-check path // below never reaches them. -const auth = new AuthService(dbs, permissions, new TripMembershipService(dbs), webauthn, userCleanup, new MailerService(dbs), new EphemeralTokenService()); +const auth = new AuthService(dbs, permissions, new TripMembershipService(dbs), webauthn, userCleanup, new MailerService(dbs), new EphemeralTokenService(), new AllowedFileTypesService(dbs)); const svc = new AdminService( dbs, new AddonsService(dbs), diff --git a/server/tests/unit/nest/auth.controller.test.ts b/server/tests/unit/nest/auth.controller.test.ts index e3eb5a23c..b3f2afea2 100644 --- a/server/tests/unit/nest/auth.controller.test.ts +++ b/server/tests/unit/nest/auth.controller.test.ts @@ -35,8 +35,9 @@ const audit = { writeAudit } as unknown as AuditService; const apc = (a: AuthService, limiter: RateLimitService) => new AuthPublicController(a, limiter, audit); // Tokens moved to TokenService; the controller takes it second. Stubbed via a // third, optional argument so every non-token call site stays as it was. +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as import('../../../src/nest/storage/storage.service').StorageService; const ac = (a: AuthService, limiter: RateLimitService, t: Partial = {}, pr: Partial = {}) => - new AuthController(a, pr as UserProfileService, t as TokenService, limiter, audit, new RuntimeEnvService()); + new AuthController(a, pr as UserProfileService, t as TokenService, limiter, audit, new RuntimeEnvService(), storageStub); function thrown(fn: () => unknown): { status: number; body: unknown } { try { fn(); } catch (err) { diff --git a/server/tests/unit/nest/auth.service.test.ts b/server/tests/unit/nest/auth.service.test.ts index 8588ea1a8..0f0c4c5e5 100644 --- a/server/tests/unit/nest/auth.service.test.ts +++ b/server/tests/unit/nest/auth.service.test.ts @@ -82,6 +82,8 @@ import { WebauthnConfigService } from '../../../src/nest/auth/webauthn-config.se import { revokeUserSessions } from '../../../src/mcp/sessionManager'; import { MailerService } from '../../../src/nest/notifications/mailer/mailer.service'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; +import { DEFAULT_ALLOWED_EXTENSIONS } from '../../../src/nest/files/files.constants'; // MailerService is injected since the notifications fold — a stub instead of a // module mock. sendPasswordResetEmail is the only thing auth reaches for. @@ -103,6 +105,7 @@ const svc = new AuthService( new UserCleanupService(new DatabaseService(testDb), new BudgetService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new ExchangeRatesService(), new RealtimeService())), mailerStub, new EphemeralTokenService(), + new AllowedFileTypesService(new DatabaseService(testDb)), ); // --------------------------------------------------------------------------- @@ -559,6 +562,17 @@ describe('getAppConfig', () => { vi.unstubAllEnvs(); }); + it('AUTH-DB-050b: fresh-install fallback matches DEFAULT_ALLOWED_EXTENSIONS (pkpass/md included)', () => { + // No allowed_file_types row: the config payload must advertise the same + // default list the upload filters actually enforce — the historical + // hardcoded copy dropped pkpass, pkpasses, md and markdown, so the client + // greyed out types the server accepts. + createUser(testDb); + testDb.prepare("DELETE FROM app_settings WHERE key = 'allowed_file_types'").run(); + const cfg = svc.getAppConfig(null); + expect(cfg.allowed_file_types).toBe(DEFAULT_ALLOWED_EXTENSIONS); + }); + it('AUTH-DB-051: authenticated caller gets the permissions block; app_settings rows flow through', () => { const { user } = createUser(testDb); testDb.prepare("INSERT OR REPLACE INTO app_settings (key, value) VALUES ('require_mfa', 'true')").run(); diff --git a/server/tests/unit/nest/auto-backup-settings.test.ts b/server/tests/unit/nest/auto-backup-settings.test.ts index bbf18ad82..59730d6c7 100644 --- a/server/tests/unit/nest/auto-backup-settings.test.ts +++ b/server/tests/unit/nest/auto-backup-settings.test.ts @@ -38,24 +38,15 @@ import { saveSettings, type BackupSettings, } from '../../../src/nest/backup/auto-backup.settings'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; -// readdirSync and statSync are overloaded in node:fs, and vi.mocked() picks the last -// overload (Dirent[] / BigIntStats) rather than the one the cleanup calls. These handles -// point at the very same mock functions from the factory above, pinned to the plain -// string[] / Stats signatures that cleanupOldBackups actually uses. +// The settings half still does file I/O; these handles pin the mock functions +// from the factory above to the plain signatures loadSettings/saveSettings use. +// (Retention no longer touches fs — it goes through StorageService.) const existsSyncMock = fs.existsSync as unknown as Mock<(path: string) => boolean>; const readFileSyncMock = fs.readFileSync as unknown as Mock<(path: string, enc: string) => string>; const writeFileSyncMock = fs.writeFileSync as unknown as Mock<(path: string, data: string) => void>; const mkdirSyncMock = fs.mkdirSync as unknown as Mock<(path: string, opts?: unknown) => void>; -const readdirSyncMock = fs.readdirSync as unknown as Mock<(path: string) => string[]>; -const statSyncMock = fs.statSync as unknown as Mock<(path: string) => fs.Stats>; -const unlinkSyncMock = fs.unlinkSync as unknown as Mock<(path: string) => void>; - -// cleanupOldBackups reads nothing but mtimeMs off a stat result, so the stubs below stay -// deliberately partial instead of faking a whole fs.Stats. -function statStub(partial: Partial): fs.Stats { - return partial as fs.Stats; -} function settings(overrides: Partial = {}): BackupSettings { return { @@ -196,76 +187,91 @@ describe('cleanupOldBackups', () => { return `${prefix}-${stamp}.zip`; } - beforeEach(() => { - readdirSyncMock.mockReset(); - statSyncMock.mockReset(); - unlinkSyncMock.mockReset(); - statSyncMock.mockReturnValue(statStub({ mtime: new Date(), mtimeMs: NOW, birthtimeMs: NOW, size: 0 })); - }); + /** Retention runs against the backups category through storage now. */ + function storageWith(entries: Array<{ key: string; mtimeMs?: number }>): StorageService { + return { + list: vi.fn(async function* () { + for (const e of entries) yield { size: 0, mtimeMs: NOW, ...e }; + }), + delete: vi.fn(async () => {}), + } as unknown as StorageService; + } + const deletedKeys = (s: StorageService) => (s.delete as Mock).mock.calls.map(([, key]) => key as string); - it('never deletes manual backup-*.zip files regardless of age', () => { + it('never deletes manual backup-*.zip files regardless of age', async () => { const manual = isoFilename(365 * 5, 'backup'); - const auto = isoFilename(0); - readdirSyncMock.mockReturnValue([manual, auto]); - cleanupOldBackups(7, NOW); - const deleted = unlinkSyncMock.mock.calls.map(([p]) => p); - expect(deleted.some(p => p.includes(manual))).toBe(false); + const storage = storageWith([{ key: manual }, { key: isoFilename(0) }]); + await cleanupOldBackups(storage, 7, NOW); + expect(deletedKeys(storage)).not.toContain(manual); }); - it('keeps auto-backups newer than retention', () => { - const recent = isoFilename(3); - readdirSyncMock.mockReturnValue([recent]); - cleanupOldBackups(7, NOW); - expect(unlinkSyncMock).not.toHaveBeenCalled(); + it('keeps auto-backups newer than retention', async () => { + const storage = storageWith([{ key: isoFilename(3) }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).not.toHaveBeenCalled(); }); - it('deletes auto-backups older than retention', () => { + it('deletes auto-backups older than retention', async () => { const old = isoFilename(30); - readdirSyncMock.mockReturnValue([old]); - cleanupOldBackups(7, NOW); - expect(unlinkSyncMock).toHaveBeenCalledOnce(); - const [calledPath] = unlinkSyncMock.mock.calls[0]; - expect(calledPath).toContain(old); + const storage = storageWith([{ key: old }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).toHaveBeenCalledOnce(); + expect(storage.delete).toHaveBeenCalledWith('backups', old); }); - it('overlayfs regression: birthtimeMs=0 does not delete a same-day backup', () => { - const fresh = isoFilename(0); - readdirSyncMock.mockReturnValue([fresh]); - statSyncMock.mockReturnValue(statStub({ birthtimeMs: 0, mtimeMs: NOW, mtime: new Date(NOW), size: 100 })); - cleanupOldBackups(7, NOW); - expect(unlinkSyncMock).not.toHaveBeenCalled(); + it('ages by filename timestamp first: a fresh name survives a bogus epoch mtime (overlayfs regression)', async () => { + // On overlayfs the fs timestamps lie (birthtime 0); the filename stamp is + // authoritative. ObjectStat only carries mtimeMs — set it to epoch and the + // same-day filename must still win. + const storage = storageWith([{ key: isoFilename(0), mtimeMs: 0 }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).not.toHaveBeenCalled(); }); - it('malformed filename falls back to mtimeMs: keeps recent file', () => { - readdirSyncMock.mockReturnValue(['auto-backup-garbage.zip']); - statSyncMock.mockReturnValue(statStub({ birthtimeMs: 0, mtimeMs: NOW - 1 * DAY, mtime: new Date(NOW - 1 * DAY), size: 0 })); - cleanupOldBackups(7, NOW); - expect(unlinkSyncMock).not.toHaveBeenCalled(); + it('malformed filename falls back to mtimeMs: keeps recent file', async () => { + const storage = storageWith([{ key: 'auto-backup-garbage.zip', mtimeMs: NOW - 1 * DAY }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).not.toHaveBeenCalled(); }); - it('malformed filename falls back to mtimeMs: deletes stale file', () => { - readdirSyncMock.mockReturnValue(['auto-backup-garbage.zip']); - statSyncMock.mockReturnValue(statStub({ birthtimeMs: 0, mtimeMs: NOW - 30 * DAY, mtime: new Date(NOW - 30 * DAY), size: 0 })); - cleanupOldBackups(7, NOW); - expect(unlinkSyncMock).toHaveBeenCalledOnce(); + it('malformed filename falls back to mtimeMs: deletes stale file', async () => { + const storage = storageWith([{ key: 'auto-backup-garbage.zip', mtimeMs: NOW - 30 * DAY }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).toHaveBeenCalledOnce(); }); - it('ignores non-zip files and does not crash', () => { + it('ignores non-zip files and does not crash', async () => { const old = isoFilename(30); - readdirSyncMock.mockReturnValue([old, 'notes.txt']); - cleanupOldBackups(7, NOW); - const calls = unlinkSyncMock.mock.calls; - expect(calls.every(([p]) => !p.includes('notes.txt'))).toBe(true); - expect(calls.length).toBe(1); + const storage = storageWith([{ key: old }, { key: 'notes.txt', mtimeMs: 0 }]); + await cleanupOldBackups(storage, 7, NOW); + expect(deletedKeys(storage)).toEqual([old]); }); - it('swallows readdirSync errors without throwing', () => { - readdirSyncMock.mockImplementation(() => { throw new Error('ENOENT'); }); - expect(() => cleanupOldBackups(7, NOW)).not.toThrow(); + it('ignores nested keys (storage.list recurses; the legacy readdir was single-level)', async () => { + const storage = storageWith([{ key: `restore-123/${isoFilename(30)}`, mtimeMs: 0 }]); + await cleanupOldBackups(storage, 7, NOW); + expect(storage.delete).not.toHaveBeenCalled(); }); - it('swallows non-Error throws without throwing (string rejection path)', () => { - readdirSyncMock.mockImplementation(() => { throw 'nope'; }); - expect(() => cleanupOldBackups(7, NOW)).not.toThrow(); + it('swallows storage list errors without throwing', async () => { + const storage = { + list: vi.fn(() => { throw new Error('ENOENT'); }), + delete: vi.fn(async () => {}), + } as unknown as StorageService; + await expect(cleanupOldBackups(storage, 7, NOW)).resolves.toBeUndefined(); + }); + + it('swallows delete failures without throwing (mirror replica or fs error)', async () => { + const storage = storageWith([{ key: isoFilename(30) }]); + (storage.delete as Mock).mockRejectedValue(new Error('EACCES')); + await expect(cleanupOldBackups(storage, 7, NOW)).resolves.toBeUndefined(); + }); + + it('swallows non-Error throws without throwing (string rejection path)', async () => { + const storage = { + list: vi.fn(() => { throw 'nope'; }), + delete: vi.fn(async () => {}), + } as unknown as StorageService; + await expect(cleanupOldBackups(storage, 7, NOW)).resolves.toBeUndefined(); }); }); diff --git a/server/tests/unit/nest/backup.controller.test.ts b/server/tests/unit/nest/backup.controller.test.ts index 4bd65f136..be8c874a5 100644 --- a/server/tests/unit/nest/backup.controller.test.ts +++ b/server/tests/unit/nest/backup.controller.test.ts @@ -4,24 +4,25 @@ import type { Request, Response } from 'express'; vi.mock('../../../src/nest/audit/client-ip', () => ({ getClientIp: vi.fn(() => '1.2.3.4') })); vi.mock('../../../src/nest/audit/audit-log.logger', () => ({ LOG_LEVEL: 'error', logInfo: vi.fn(), logDebug: vi.fn(), logError: vi.fn(), logWarn: vi.fn() })); -// The controller imports the tmp-dir + size cap at module load. The thin -// BackupService wrapper forwards every call straight into this module, so the -// mock also stubs the delegated functions for the wrapper tests below. +// The thin BackupService wrapper forwards every call straight into this +// module, so the mock stubs the delegated functions for the wrapper tests +// below (the multer size cap is consumed by backup.module's upload factory). vi.mock('../../../src/nest/backup/backup.impl', () => ({ - getUploadTmpDir: () => '/tmp', MAX_BACKUP_UPLOAD_SIZE: 1024, BACKUP_RATE_WINDOW: 3600000, listBackups: vi.fn().mockReturnValue([{ filename: 'svc.zip' }]), createBackup: vi.fn().mockResolvedValue({ filename: 'svc.zip', size: 5 }), restoreFromZip: vi.fn().mockResolvedValue({ success: true }), + restoreBackup: vi.fn().mockResolvedValue({ success: true }), deleteBackup: vi.fn(), isValidBackupFilename: vi.fn().mockReturnValue(true), - backupFilePath: vi.fn().mockReturnValue('/data/backups/svc.zip'), backupFileExists: vi.fn().mockReturnValue(true), + sendBackupToResponse: vi.fn().mockResolvedValue(undefined), checkRateLimit: vi.fn().mockReturnValue(true), })); import { BackupController } from '../../../src/nest/backup/backup.controller'; +import { StorageNotFoundError } from '../../../src/nest/storage/storage.types'; import { BackupService as RealBackupService } from '../../../src/nest/backup/backup.service'; import { AdminGuard } from '../../../src/nest/auth/admin.guard'; import type { BackupService } from '../../../src/nest/backup/backup.service'; @@ -50,12 +51,13 @@ function svc(o: Partial = {}): BackupService { listBackups: vi.fn().mockReturnValue([]), createBackup: vi.fn(), restoreFromZip: vi.fn(), + restoreBackup: vi.fn(), getAutoSettings: vi.fn(), updateAutoSettings: vi.fn(), deleteBackup: vi.fn(), isValidBackupFilename: vi.fn().mockReturnValue(true), - backupFilePath: vi.fn().mockReturnValue('/b/x.zip'), backupFileExists: vi.fn().mockReturnValue(true), + sendBackupToResponse: vi.fn().mockResolvedValue(undefined), checkRateLimit: vi.fn().mockReturnValue(true), rateWindow: 3600000, ...o, @@ -93,9 +95,9 @@ describe('AdminGuard (used by BackupController)', () => { }); describe('BackupController', () => { - it('GET /list returns backups, 500 on error', () => { - expect(bc(svc({ listBackups: vi.fn().mockReturnValue([{ filename: 'a.zip' }]) } as Partial)).list()).toEqual({ backups: [{ filename: 'a.zip' }] }); - expect(thrown(() => bc(svc({ listBackups: vi.fn(() => { throw new Error('io'); }) } as Partial)).list())).toEqual({ status: 500, body: { error: 'Error loading backups' } }); + it('GET /list returns backups, 500 on error', async () => { + await expect(bc(svc({ listBackups: vi.fn().mockResolvedValue([{ filename: 'a.zip' }]) } as Partial)).list()).resolves.toEqual({ backups: [{ filename: 'a.zip' }] }); + expect(await thrownAsync(() => bc(svc({ listBackups: vi.fn(() => { throw new Error('io'); }) } as Partial)).list())).toEqual({ status: 500, body: { error: 'Error loading backups' } }); }); it('POST /create 429 when rate-limited, else creates + audits', async () => { @@ -106,25 +108,37 @@ describe('BackupController', () => { expect(writeAudit).toHaveBeenCalledWith(expect.objectContaining({ action: 'backup.create', resource: 'b.zip' })); }); - it('GET /download 400 invalid / 404 missing, else res.download', () => { - const res = { download: vi.fn() } as unknown as Response; - expect(thrown(() => bc(svc({ isValidBackupFilename: vi.fn().mockReturnValue(false) })).download('x', res))).toEqual({ status: 400, body: { error: 'Invalid filename' } }); - expect(thrown(() => bc(svc({ backupFileExists: vi.fn().mockReturnValue(false) })).download('x.zip', res))).toEqual({ status: 404, body: { error: 'Backup not found' } }); - bc(svc()).download('x.zip', res); - expect(res.download).toHaveBeenCalledWith('/b/x.zip', 'x.zip'); + it('GET /download 400 invalid / 404 missing, else serves through storage', async () => { + const res = {} as unknown as Response; + expect(await thrownAsync(() => bc(svc({ isValidBackupFilename: vi.fn().mockReturnValue(false) })).download('x', res))).toEqual({ status: 400, body: { error: 'Invalid filename' } }); + expect(await thrownAsync(() => bc(svc({ backupFileExists: vi.fn().mockResolvedValue(false) })).download('x.zip', res))).toEqual({ status: 404, body: { error: 'Backup not found' } }); + const sendBackupToResponse = vi.fn().mockResolvedValue(undefined); + await bc(svc({ sendBackupToResponse } as Partial)).download('x.zip', res); + expect(sendBackupToResponse).toHaveBeenCalledWith('x.zip', res); + }); + + it('GET /download maps a StorageNotFoundError raced past the pre-check to the same 404', async () => { + const res = {} as unknown as Response; + const sendBackupToResponse = vi.fn().mockRejectedValue(new StorageNotFoundError('x.zip')); + expect(await thrownAsync(() => bc(svc({ sendBackupToResponse } as Partial)).download('x.zip', res))).toEqual({ status: 404, body: { error: 'Backup not found' } }); + // A non-miss failure is NOT swallowed into the 404 envelope. + const boom = vi.fn().mockRejectedValue(new Error('io')); + await expect(bc(svc({ sendBackupToResponse: boom } as Partial)).download('x.zip', res)).rejects.toThrow('io'); }); it('POST /restore maps the service status, else audits', async () => { expect(await thrownAsync(() => bc(svc({ isValidBackupFilename: vi.fn().mockReturnValue(false) })).restore(user, 'x', req))).toEqual({ status: 400, body: { error: 'Invalid filename' } }); expect(await thrownAsync(() => bc(svc({ backupFileExists: vi.fn().mockReturnValue(false) })).restore(user, 'x.zip', req))).toEqual({ status: 404, body: { error: 'Backup not found' } }); - expect(await thrownAsync(() => bc(svc({ restoreFromZip: vi.fn().mockResolvedValue({ success: false, status: 422, error: 'bad zip' }) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 422, body: { error: 'bad zip' } }); - const res = await bc(svc({ restoreFromZip: vi.fn().mockResolvedValue({ success: true }) } as Partial)).restore(user, 'x.zip', req); + expect(await thrownAsync(() => bc(svc({ restoreBackup: vi.fn().mockResolvedValue({ success: false, status: 422, error: 'bad zip' }) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 422, body: { error: 'bad zip' } }); + const restoreBackup = vi.fn().mockResolvedValue({ success: true }); + const res = await bc(svc({ restoreBackup } as Partial)).restore(user, 'x.zip', req); expect(res).toEqual({ success: true }); + expect(restoreBackup).toHaveBeenCalledWith('x.zip'); expect(writeAudit).toHaveBeenCalledWith(expect.objectContaining({ action: 'backup.restore', resource: 'x.zip' })); }); it('POST /restore falls back to status 400 when the service omits one', async () => { - expect(await thrownAsync(() => bc(svc({ restoreFromZip: vi.fn().mockResolvedValue({ success: false, error: 'nope' }) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 400, body: { error: 'nope' } }); + expect(await thrownAsync(() => bc(svc({ restoreBackup: vi.fn().mockResolvedValue({ success: false, error: 'nope' }) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 400, body: { error: 'nope' } }); }); it('POST /upload-restore 400 without a file, cleans up the tmp file', async () => { @@ -154,7 +168,7 @@ describe('BackupController', () => { it('maps unexpected service errors to 500 (create, restore, auto-settings)', async () => { vi.spyOn(console, 'error').mockImplementation(() => {}); expect(await thrownAsync(() => bc(svc({ createBackup: vi.fn().mockRejectedValue(new Error('disk')) } as Partial)).create(user, req))).toEqual({ status: 500, body: { error: 'Error creating backup' } }); - expect(await thrownAsync(() => bc(svc({ restoreFromZip: vi.fn().mockRejectedValue(new Error('boom')) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 500, body: { error: 'Error restoring backup' } }); + expect(await thrownAsync(() => bc(svc({ restoreBackup: vi.fn().mockRejectedValue(new Error('boom')) } as Partial)).restore(user, 'x.zip', req))).toEqual({ status: 500, body: { error: 'Error restoring backup' } }); expect(thrown(() => bc(svc(), job({ getAutoSettings: vi.fn(() => { throw new Error('io'); }) })).autoSettings())).toEqual({ status: 500, body: { error: 'Could not load backup settings' } }); }); @@ -187,39 +201,46 @@ describe('BackupController', () => { expect(writeAudit).toHaveBeenCalledWith(expect.objectContaining({ action: 'backup.auto_settings' })); }); - it('DELETE /:filename 400/404, else deletes + audits', () => { - expect(thrown(() => bc(svc({ isValidBackupFilename: vi.fn().mockReturnValue(false) })).remove(user, 'x', req))).toEqual({ status: 400, body: { error: 'Invalid filename' } }); - expect(thrown(() => bc(svc({ backupFileExists: vi.fn().mockReturnValue(false) })).remove(user, 'x.zip', req))).toEqual({ status: 404, body: { error: 'Backup not found' } }); + it('DELETE /:filename 400/404, else deletes + audits', async () => { + expect(await thrownAsync(() => bc(svc({ isValidBackupFilename: vi.fn().mockReturnValue(false) })).remove(user, 'x', req))).toEqual({ status: 400, body: { error: 'Invalid filename' } }); + expect(await thrownAsync(() => bc(svc({ backupFileExists: vi.fn().mockResolvedValue(false) })).remove(user, 'x.zip', req))).toEqual({ status: 404, body: { error: 'Backup not found' } }); const deleteBackup = vi.fn(); - expect(bc(svc({ deleteBackup } as Partial)).remove(user, 'x.zip', req)).toEqual({ success: true }); + await expect(bc(svc({ deleteBackup } as Partial)).remove(user, 'x.zip', req)).resolves.toEqual({ success: true }); expect(deleteBackup).toHaveBeenCalledWith('x.zip'); }); }); describe('BackupService (wrapper)', () => { - const wrapper = new RealBackupService(); + // The impl module is fully mocked above, so an empty sentinel stands in for + // the injected StorageService — the wrapper's job is passing it through. + const storage = { __sentinel: 'storage' } as unknown as import('../../../src/nest/storage/storage.service').StorageService; + const wrapper = new RealBackupService(storage); it('forwards every call straight to the legacy backup service', async () => { expect(wrapper.listBackups()).toEqual([{ filename: 'svc.zip' }]); - expect(backupSvc.listBackups).toHaveBeenCalled(); + expect(backupSvc.listBackups).toHaveBeenCalledWith(storage); await expect(wrapper.createBackup()).resolves.toEqual({ filename: 'svc.zip', size: 5 }); expect(backupSvc.createBackup).toHaveBeenCalled(); await expect(wrapper.restoreFromZip('/tmp/a.zip')).resolves.toEqual({ success: true }); - expect(backupSvc.restoreFromZip).toHaveBeenCalledWith('/tmp/a.zip'); + expect(backupSvc.restoreFromZip).toHaveBeenCalledWith(storage, '/tmp/a.zip'); + + await expect(wrapper.restoreBackup('svc.zip')).resolves.toEqual({ success: true }); + expect(backupSvc.restoreBackup).toHaveBeenCalledWith(storage, 'svc.zip'); wrapper.deleteBackup('svc.zip'); - expect(backupSvc.deleteBackup).toHaveBeenCalledWith('svc.zip'); + expect(backupSvc.deleteBackup).toHaveBeenCalledWith(storage, 'svc.zip'); expect(wrapper.isValidBackupFilename('svc.zip')).toBe(true); expect(backupSvc.isValidBackupFilename).toHaveBeenCalledWith('svc.zip'); - expect(wrapper.backupFilePath('svc.zip')).toBe('/data/backups/svc.zip'); - expect(backupSvc.backupFilePath).toHaveBeenCalledWith('svc.zip'); - expect(wrapper.backupFileExists('svc.zip')).toBe(true); - expect(backupSvc.backupFileExists).toHaveBeenCalledWith('svc.zip'); + expect(backupSvc.backupFileExists).toHaveBeenCalledWith(storage, 'svc.zip'); + + const fakeRes = {} as Response; + await wrapper.sendBackupToResponse('svc.zip', fakeRes); + expect(backupSvc.sendBackupToResponse).toHaveBeenCalledWith(storage, 'svc.zip', fakeRes); expect(wrapper.checkRateLimit('ip', 3, 1000)).toBe(true); expect(backupSvc.checkRateLimit).toHaveBeenCalledWith('ip', 3, 1000); @@ -235,4 +256,22 @@ describe('BackupModule', () => { const { BackupModule } = await import('../../../src/nest/backup/backup.module'); expect(new BackupModule()).toBeInstanceOf(BackupModule); }); + + it('spools restore uploads to the storage tempDir with the legacy filter and cap', async () => { + // The restore upload is a restore INPUT, not a stored object — it gets no + // category and spools to the driver-agnostic scratch dir (spec Uploads #9). + const { buildBackupUploadOptions } = await import('../../../src/nest/backup/backup.module'); + const storage = { tempDir: vi.fn(() => '/data/tmp') } as unknown as import('../../../src/nest/storage/storage.service').StorageService; + + const opts = buildBackupUploadOptions(storage); + + expect(opts.dest).toBe('/data/tmp'); + expect(opts.limits).toEqual({ fileSize: 1024 }); // MAX_BACKUP_UPLOAD_SIZE from the impl mock + const accept = vi.fn(); + opts.fileFilter?.(undefined as never, { originalname: 'a.zip' } as Express.Multer.File, accept); + expect(accept).toHaveBeenCalledWith(null, true); + const reject = vi.fn(); + opts.fileFilter?.(undefined as never, { originalname: 'a.tar.gz' } as Express.Multer.File, reject); + expect(reject).toHaveBeenCalledWith(expect.objectContaining({ message: 'Only ZIP files allowed' }), false); + }); }); diff --git a/server/tests/unit/nest/backup.impl.test.ts b/server/tests/unit/nest/backup.impl.test.ts index 62a8a143d..ee9e182c0 100644 --- a/server/tests/unit/nest/backup.impl.test.ts +++ b/server/tests/unit/nest/backup.impl.test.ts @@ -73,11 +73,45 @@ import { createBackup, deleteBackup, restoreFromZip, + restoreBackup, BACKUP_RATE_WINDOW, - backupFilePath, backupFileExists, listBackups, + sendBackupToResponse, } from '../../../src/nest/backup/backup.impl'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; + +// --------------------------------------------------------------------------- +// Storage stub — backup.impl functions receive StorageService as a parameter +// (BackupService injects it and forwards). This file mocks fs wholesale, so a +// real LocalDriver would see the mocked fs: use plain stub objects instead. +// --------------------------------------------------------------------------- + +function stubStorage(overrides: Record = {}): StorageService { + return { + // async generator; tests override with listOf(...) entries + list: vi.fn(async function* () {}), + stat: vi.fn(async () => null), + exists: vi.fn(async () => false), + delete: vi.fn(async () => {}), + put: vi.fn(async () => {}), + getStream: vi.fn(), + sendToResponse: vi.fn(async () => {}), + withLocalFile: vi.fn(async (_c: string, _k: string, fn: (p: string) => Promise) => fn('/stub/local/path')), + // Default: every object has a local path (the zero-copy default-install branch). + getLocalPathOrNull: vi.fn(async () => '/stub/local/path'), + spoolDirFor: vi.fn(() => '/stub/spool'), + tempDir: vi.fn(() => '/stub/tmp'), + health: vi.fn(() => ({ replicaFailures: [] })), + reloadConfig: vi.fn(), + ...overrides, + } as unknown as StorageService; +} + +const listOf = (entries: Array<{ key: string; size?: number; mtimeMs?: number }>) => + vi.fn(async function* () { + for (const e of entries) yield { size: 0, mtimeMs: 0, ...e }; + }); // --------------------------------------------------------------------------- // formatSize @@ -290,244 +324,264 @@ describe('BACKUP-036 createBackup', () => { vi.clearAllMocks(); }); - it('BACKUP-036a — happy path: creates zip and returns BackupInfo', async () => { - // Set up fs mocks - fsMock.existsSync.mockImplementation((p: string) => { - // backupsDir exists, dbPath does not (skip DB file), uploadsDir does not exist - return false; - }); - fsMock.mkdirSync.mockReturnValue(undefined); - - // Mock WriteStream with event emitter behaviour + /** Wires the write-stream + archiver mocks so finalize() resolves the build. */ + function setupArchiveSuccess() { const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - // Mock archiver instance - archiverInstanceMock.on.mockImplementation((event: string, cb: Function) => { - // noop — no error - }); + fsMock.createWriteStream.mockReturnValue({ + on: vi.fn((event: string, cb: Function) => { writableEvents[event] = cb; }), + } as never); + archiverInstanceMock.on.mockImplementation(() => {}); archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - // Trigger 'close' on the output stream to resolve the Promise - if (writableEvents['close']) writableEvents['close'](); - }); + archiverInstanceMock.finalize.mockImplementation(() => { writableEvents['close']?.(); }); archiverMock.mockReturnValue(archiverInstanceMock); + return writableEvents; + } - fsMock.statSync.mockReturnValue({ size: 2048, birthtime: new Date('2026-04-06T12:00:00Z') }); + /** storage.stat stub for the post-put BackupInfo read. */ + const statOf = (size: number, mtimeMs = Date.parse('2026-04-06T12:00:00Z')) => + vi.fn(async (_c: string, key: string) => ({ key, size, mtimeMs })); - const result = await createBackup(); + /** storage.list stub keyed by category. */ + const listByCategory = (map: Record>) => + vi.fn((category: string) => + (async function* () { + for (const e of map[category] ?? []) yield { size: 1, mtimeMs: 0, ...e }; + })(), + ); + + it('BACKUP-036a — happy path: builds in the backups spool, commits via put, returns BackupInfo', async () => { + // No travel.db, no enc key, no plugin roots. + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(2048) }); + + const result = await createBackup(storage); - expect(result).toHaveProperty('filename'); expect(result.filename).toMatch(/^backup-.*\.zip$/); expect(result.size).toBe(2048); expect(result.sizeText).toBe('2.0 KB'); - expect(result).toHaveProperty('created_at'); + expect(result.created_at).toBe('2026-04-06T12:00:00.000Z'); expect(archiverMock).toHaveBeenCalledWith('zip', { zlib: { level: 9 } }); expect(archiverInstanceMock.pipe).toHaveBeenCalled(); expect(archiverInstanceMock.finalize).toHaveBeenCalled(); + // The zip is built in the backups backend's spool, then committed via put. + expect(fsMock.createWriteStream).toHaveBeenCalledWith(expect.stringContaining('/stub/spool/zip-build-backup-')); + expect(storage.put).toHaveBeenCalledWith('backups', result.filename, { + tmpPath: expect.stringContaining('/stub/spool/zip-build-backup-'), + }); + expect(storage.stat).toHaveBeenCalledWith('backups', result.filename); + }); + + it('BACKUP-036j — archives category objects under the legacy uploads/ entry names via the local fast path', async () => { + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ + stat: statOf(1024), + list: listByCategory({ + files: [{ key: 'a.pdf' }], + journey: [{ key: 'thumbs/x.jpg' }], + }), + }); + + await createBackup(storage); + + expect(storage.getLocalPathOrNull).toHaveBeenCalledWith('files', 'a.pdf'); + expect(storage.getLocalPathOrNull).toHaveBeenCalledWith('journey', 'thumbs/x.jpg'); + expect(archiverInstanceMock.file).toHaveBeenCalledWith('/stub/local/path', { name: 'uploads/files/a.pdf' }); + expect(archiverInstanceMock.file).toHaveBeenCalledWith('/stub/local/path', { name: 'uploads/journey/thumbs/x.jpg' }); + // The local fast path never touches getStream — this is the zero-copy default-install branch. + expect(storage.getStream).not.toHaveBeenCalled(); + }); + + it('BACKUP-036k — a remote-driver object (no local path) is streamed into per-backup staging and archived from there', async () => { + fsMock.existsSync.mockReturnValue(false); + const writableEvents = setupArchiveSuccess(); + const { PassThrough } = await import('node:stream'); + const remoteStream = new PassThrough(); + remoteStream.end(Buffer.from('remote upload bytes')); + fsMock.createWriteStream.mockImplementation((p: string) => { + // The zip destination stream still needs the writableEvents wiring + // setupArchiveSuccess set up (finalize() resolves the build via 'close'); + // staged files use a minimal fake that just resolves the pipeline. + if (String(p).includes('zip-build-')) { + return { on: vi.fn((event: string, cb: Function) => { writableEvents[event] = cb; }) }; + } + // pipeline() (real, from node:stream/promises) needs a real Writable — + // a PassThrough gives it one without touching the real filesystem. + return new PassThrough(); + }); + const storage = stubStorage({ + stat: statOf(2048), + list: listByCategory({ files: [{ key: 'remote.pdf' }] }), + getLocalPathOrNull: vi.fn(async () => null), + getStream: vi.fn(async (category: string, key: string) => { + expect(category).toBe('files'); + expect(key).toBe('remote.pdf'); + return { stream: remoteStream, stat: { key, size: 20, mtimeMs: 0 } }; + }), + }); + + await createBackup(storage); + + expect(storage.getLocalPathOrNull).toHaveBeenCalledWith('files', 'remote.pdf'); + expect(storage.getStream).toHaveBeenCalledWith('files', 'remote.pdf'); + const fileCall = archiverInstanceMock.file.mock.calls.find( + (c: unknown[]) => (c[1] as { name?: string })?.name === 'uploads/files/remote.pdf', + ); + expect(fileCall).toBeDefined(); + const stagedPath = fileCall![0] as string; + expect(stagedPath).toContain('/stub/spool/staging-backup-'); + expect(stagedPath).toContain('files/remote.pdf'); + // The staging dir is removed alongside zipSpool/dbSnap in the existing finally. + expect(fsMock.rmSync).toHaveBeenCalledWith( + expect.stringContaining('/stub/spool/staging-backup-'), + { recursive: true, force: true }, + ); + }); + + it('BACKUP-036l — a vanished remote object (archiver "warning") fails the backup instead of silently dropping it', async () => { + fsMock.existsSync.mockReturnValue(false); + const archiveEvents: Record = {}; + fsMock.createWriteStream.mockReturnValue({ on: vi.fn() } as never); + archiverInstanceMock.on.mockImplementation((event: string, cb: Function) => { archiveEvents[event] = cb; }); + archiverInstanceMock.pipe.mockReturnValue(undefined); + archiverInstanceMock.finalize.mockImplementation(() => { + // archiver emits 'warning' (not 'error') for entries it couldn't stat/read + // (e.g. ENOENT) — without archive.on('warning', reject) this resolves clean. + archiveEvents['warning']?.(new Error('ENOENT: no such file, stat entry')); + }); + archiverMock.mockReturnValue(archiverInstanceMock); + const storage = stubStorage(); + + await expect(createBackup(storage)).rejects.toThrow('ENOENT'); + expect(storage.put).not.toHaveBeenCalled(); + }); + + it('BACKUP-036m — a mirror replica failure surfaced via health never fails the request', async () => { + // Mirror semantics: put succeeds against the primary; replica failures are + // recorded on health(), not thrown. createBackup must still return its info. + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ + stat: statOf(1024), + health: vi.fn(() => ({ replicaFailures: [{ backend: 'nas-backups', key: 'backup-x.zip', error: 'EIO' }] })), + }); + + const result = await createBackup(storage); + + expect(result.filename).toMatch(/^backup-.*\.zip$/); + expect(storage.put).toHaveBeenCalledOnce(); }); it('BACKUP-036p — archives the plugin data + code trees when present, skipping dev-links', async () => { // Only the plugin roots exist (db/uploads absent → skipped). fsMock.existsSync.mockImplementation((p: string) => String(p).includes('plugins')); - fsMock.mkdirSync.mockReturnValue(undefined); // Two plugin code dirs: 'notes' is real, 'devlink' resolves outside the root. // The plugin-data snapshot reads with { withFileTypes: true }; hand it Dirent-likes there. const dirent = (name: string) => ({ name, isDirectory: () => true }); fsMock.readdirSync.mockImplementation((_p: string, opts?: { withFileTypes?: boolean }) => (opts?.withFileTypes ? [dirent('notes'), dirent('devlink')] : ['notes', 'devlink']) as never); fsMock.realpathSync.mockImplementation((p: string) => (String(p).endsWith('devlink') ? '/somewhere/else/devlink' : p)); - fsMock.statSync.mockReturnValue({ size: 2048, birthtime: new Date('2026-04-06T12:00:00Z'), isDirectory: () => true } as never); + fsMock.statSync.mockReturnValue({ isDirectory: () => true } as never); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(2048) }); - const writableEvents: Record = {}; - fsMock.createWriteStream.mockReturnValue({ on: vi.fn((e: string, cb: Function) => { writableEvents[e] = cb; }) } as never); - archiverInstanceMock.on.mockImplementation(() => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { writableEvents['close']?.(); }); - archiverMock.mockReturnValue(archiverInstanceMock); - - await createBackup(); + await createBackup(storage); // the consistent snapshot of the data tree is archived under plugins-data/ expect(archiverInstanceMock.directory).toHaveBeenCalledWith(expect.stringContaining('plugins-snap'), 'plugins-data'); // the real code dir is archived, the dev-link is skipped expect(archiverInstanceMock.directory).toHaveBeenCalledWith(expect.stringContaining('notes'), 'plugins-code/notes'); expect(archiverInstanceMock.directory).not.toHaveBeenCalledWith(expect.anything(), 'plugins-code/devlink'); + // the snapshot staging lives in the backups spool + expect(archiverInstanceMock.directory).toHaveBeenCalledWith(expect.stringContaining('/stub/spool/plugins-snap-backup-'), 'plugins-data'); }); it('BACKUP-036b — WAL checkpoint error is swallowed (non-critical)', async () => { // db.exec throws on WAL checkpoint dbMock.db.exec.mockImplementationOnce(() => { throw new Error('WAL checkpoint failed'); }); - - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); fsMock.existsSync.mockReturnValue(false); - fsMock.mkdirSync.mockReturnValue(undefined); - - archiverInstanceMock.on.mockImplementation((_event: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 512, birthtime: new Date('2026-04-06T12:00:00Z') }); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(512) }); // Should not throw even though WAL checkpoint failed - const result = await createBackup(); + const result = await createBackup(storage); expect(result).toHaveProperty('filename'); expect(result.size).toBe(512); }); - it('BACKUP-036c — archiver error cleans up partial file and re-throws', async () => { + it('BACKUP-036c — archiver error cleans up the spool staging, skips put and re-throws', async () => { fsMock.existsSync.mockReturnValue(false); - fsMock.mkdirSync.mockReturnValue(undefined); - - const writableEvents: Record = {}; const archiveEvents: Record = {}; - - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - archiverInstanceMock.on.mockImplementation((event: string, cb: Function) => { - archiveEvents[event] = cb; - }); + fsMock.createWriteStream.mockReturnValue({ on: vi.fn() } as never); + archiverInstanceMock.on.mockImplementation((event: string, cb: Function) => { archiveEvents[event] = cb; }); archiverInstanceMock.pipe.mockReturnValue(undefined); archiverInstanceMock.finalize.mockImplementation(() => { // Simulate archive error instead of success - if (archiveEvents['error']) archiveEvents['error'](new Error('disk full')); + archiveEvents['error']?.(new Error('disk full')); }); archiverMock.mockReturnValue(archiverInstanceMock); + const storage = stubStorage(); - // The output file "exists" after partial write so cleanup runs - fsMock.existsSync.mockImplementation((p: string) => { - // Return true only when checking the output path (ends with .zip) - return String(p).endsWith('.zip'); - }); - fsMock.unlinkSync.mockReturnValue(undefined); + await expect(createBackup(storage)).rejects.toThrow('disk full'); - await expect(createBackup()).rejects.toThrow('disk full'); - // Partial file should have been removed - expect(fsMock.unlinkSync).toHaveBeenCalled(); + // Nothing is committed; the half-built spool file is removed in the finally. + expect(storage.put).not.toHaveBeenCalled(); + expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('zip-build-backup-'), { force: true }); }); - it('BACKUP-036d — includes travel.db when it exists', async () => { - fsMock.existsSync.mockImplementation((p: string) => { - // backupsDir does not need to be created (exists), dbPath exists, no uploads - if (String(p).endsWith('travel.db')) return true; - return false; - }); - fsMock.mkdirSync.mockReturnValue(undefined); + it('BACKUP-036d — includes travel.db when it exists, snapshotted into the spool', async () => { + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('travel.db')); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(1024) }); - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); - - await createBackup(); + await createBackup(storage); // the core DB is snapshotted (VACUUM INTO) and archived under the name travel.db expect(dbMock.db.exec).toHaveBeenCalledWith(expect.stringContaining('VACUUM INTO')); expect(archiverInstanceMock.file).toHaveBeenCalledWith( - expect.any(String), + expect.stringContaining('/stub/spool/travel-snap-backup-'), { name: 'travel.db' } ); }); - it('BACKUP-036e — includes uploads but excludes the re-derivable photo caches', async () => { - fsMock.existsSync.mockImplementation((p: string) => { - if (String(p).endsWith('uploads')) return true; - return false; - }); - fsMock.mkdirSync.mockReturnValue(undefined); - - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; + it('BACKUP-036e — excludes the re-derivable photo caches nested under photos/', async () => { + // In mode A the google/trek caches live inside the photos/ prefix, so the + // category walk must skip them. (In mode B photos-google is a separate + // category that is simply never enumerated — no extra branch needed.) + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ + stat: statOf(1024), + list: listByCategory({ + photos: [{ key: 'flat.jpg' }, { key: 'google/g.jpg' }, { key: 'trek/t.bin' }], }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); }); - archiverMock.mockReturnValue(archiverInstanceMock); - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); + await createBackup(storage); - await createBackup(); - - expect(archiverInstanceMock.glob).toHaveBeenCalledWith( - '**/*', - expect.objectContaining({ - cwd: expect.stringContaining('uploads'), - ignore: ['photos/google/**', 'photos/trek/**', 'backups/**', 'restore-*/**'], - }), - { prefix: 'uploads' }, - ); - // The re-derivable caches must not be archived verbatim. - expect(archiverInstanceMock.directory).not.toHaveBeenCalled(); + expect(archiverInstanceMock.file).toHaveBeenCalledWith('/stub/local/path', { name: 'uploads/photos/flat.jpg' }); + const names = archiverInstanceMock.file.mock.calls.map(c => c[1]?.name as string); + expect(names.some(n => n?.includes('google/'))).toBe(false); + expect(names.some(n => n?.includes('trek/'))).toBe(false); + // Only the archived categories are ever listed — the excludes are structural. + expect(storage.getLocalPathOrNull).toHaveBeenCalledTimes(1); }); - it('BACKUP-036h — never sweeps backups/ or restore-* into the archive (issue #1358)', async () => { - // Regression guard: when data and uploads map to the same directory, the - // uploads glob would otherwise pick up the backups/ dir and recursively - // embed every prior backup zip, compounding size without bound. - fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('uploads')); - fsMock.mkdirSync.mockReturnValue(undefined); + it('BACKUP-036h — only category prefixes are ever archived; backups/ and restore-* are structurally out (issue #1358)', async () => { + // The uploads/** glob is gone: even when data and uploads map to the same + // directory, enumeration only ever walks the six archived categories, so + // prior backup zips and restore-* staging can never be swept into the zip. + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(1024) }); - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); + await createBackup(storage); - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); - - await createBackup(); - - const globCall = archiverInstanceMock.glob.mock.calls.at(-1); - const ignore: string[] = globCall?.[1]?.ignore ?? []; - expect(ignore).toContain('backups/**'); - expect(ignore).toContain('restore-*/**'); + expect(archiverInstanceMock.glob).not.toHaveBeenCalled(); + const listed = (storage.list as ReturnType).mock.calls.map(c => c[0] as string); + expect(listed.sort()).toEqual(['avatars', 'covers', 'files', 'journey', 'photos', 'places']); + expect(listed).not.toContain('backups'); }); it('BACKUP-036f — bundles .encryption_key when present and ENCRYPTION_KEY env is unset', async () => { @@ -535,26 +589,10 @@ describe('BACKUP-036 createBackup', () => { delete process.env.ENCRYPTION_KEY; try { fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('.encryption_key')); - fsMock.mkdirSync.mockReturnValue(undefined); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(1024) }); - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); - - await createBackup(); + await createBackup(storage); expect(archiverInstanceMock.file).toHaveBeenCalledWith( expect.stringContaining('.encryption_key'), @@ -568,26 +606,10 @@ describe('BACKUP-036 createBackup', () => { it('BACKUP-036g — does NOT bundle .encryption_key when ENCRYPTION_KEY env is set', async () => { // setup.ts sets process.env.ENCRYPTION_KEY, so the env is the source of truth. fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('.encryption_key')); - fsMock.mkdirSync.mockReturnValue(undefined); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(1024) }); - const writableEvents: Record = {}; - const fakeWriteStream = { - on: vi.fn((event: string, cb: Function) => { - writableEvents[event] = cb; - }), - }; - fsMock.createWriteStream.mockReturnValue(fakeWriteStream); - - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); - - await createBackup(); + await createBackup(storage); expect(archiverInstanceMock.file).not.toHaveBeenCalledWith( expect.stringContaining('.encryption_key'), @@ -599,28 +621,19 @@ describe('BACKUP-036 createBackup', () => { // The scheduler passes 'auto-backup' so retention and the admin panel can // still tell scheduled archives apart by filename. fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('travel.db')); - fsMock.mkdirSync.mockReturnValue(undefined); + setupArchiveSuccess(); + const storage = stubStorage({ stat: statOf(1024) }); - const writableEvents: Record = {}; - fsMock.createWriteStream.mockReturnValue({ - on: vi.fn((event: string, cb: Function) => { writableEvents[event] = cb; }), - } as never); - archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); - archiverInstanceMock.pipe.mockReturnValue(undefined); - archiverInstanceMock.finalize.mockImplementation(() => { - if (writableEvents['close']) writableEvents['close'](); - }); - archiverMock.mockReturnValue(archiverInstanceMock); - - fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); - - const result = await createBackup('auto-backup'); + const result = await createBackup(storage, 'auto-backup'); expect(result.filename).toMatch(/^auto-backup-.*\.zip$/); expect(archiverInstanceMock.file).toHaveBeenCalledWith( - expect.stringContaining('.travel-snap-auto-backup-'), + expect.stringContaining('travel-snap-auto-backup-'), { name: 'travel.db' }, ); + expect(storage.put).toHaveBeenCalledWith('backups', result.filename, { + tmpPath: expect.stringContaining('zip-build-auto-backup-'), + }); }); }); @@ -633,25 +646,22 @@ describe('BACKUP-037 deleteBackup', () => { vi.clearAllMocks(); }); - it('BACKUP-037a — happy path: calls unlinkSync with correct path', () => { - fsMock.unlinkSync.mockReturnValue(undefined); + it('BACKUP-037a — happy path: deletes through storage under the backups category', async () => { + const storage = stubStorage(); - deleteBackup('backup-2026-04-06T12-00-00.zip'); + await deleteBackup(storage, 'backup-2026-04-06T12-00-00.zip'); - expect(fsMock.unlinkSync).toHaveBeenCalledOnce(); - expect(fsMock.unlinkSync).toHaveBeenCalledWith( - expect.stringContaining('backup-2026-04-06T12-00-00.zip') - ); + expect(storage.delete).toHaveBeenCalledOnce(); + expect(storage.delete).toHaveBeenCalledWith('backups', 'backup-2026-04-06T12-00-00.zip'); }); - it('BACKUP-037b — throws when unlinkSync throws (file not found)', () => { - fsMock.unlinkSync.mockImplementation(() => { - const err: NodeJS.ErrnoException = new Error('ENOENT: no such file or directory'); - err.code = 'ENOENT'; - throw err; - }); + it('BACKUP-037b — propagates a storage delete failure', async () => { + // Note: storage.delete is idempotent on a MISSING object (route parity holds + // because the controller pre-checks existence and 404s); this pins that a + // real backend failure still surfaces. + const storage = stubStorage({ delete: vi.fn(async () => { throw new Error('EIO: i/o error'); }) }); - expect(() => deleteBackup('backup-missing.zip')).toThrow('ENOENT'); + await expect(deleteBackup(storage, 'backup-missing.zip')).rejects.toThrow('EIO'); }); }); @@ -679,7 +689,7 @@ describe('BACKUP-038 restoreFromZip', () => { }); fsMock.rmSync.mockReturnValue(undefined); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result.success).toBe(false); expect(result.error).toMatch(/travel\.db not found/i); @@ -691,7 +701,7 @@ describe('BACKUP-038 restoreFromZip', () => { files: [{ uncompressedSize: 6 * 1024 * 1024 * 1024 }], // 6 GB > 5 GB cap }); - const result = await restoreFromZip('/data/tmp/bomb.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/bomb.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -761,7 +771,7 @@ describe('BACKUP-061 restoreFromZip extraction', () => { it('BACKUP-061a — refuses an entry whose path escapes the archive root (zip-slip)', async () => { unzipperMock.Open.file.mockResolvedValueOnce({ files: [zipEntry('../../etc/passwd')] }); - const result = await restoreFromZip('/data/tmp/slip.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/slip.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -779,7 +789,7 @@ describe('BACKUP-061 restoreFromZip extraction', () => { it.runIf(process.platform === 'win32')('BACKUP-061b — a drive-letter entry path is refused the same way', async () => { unzipperMock.Open.file.mockResolvedValueOnce({ files: [zipEntry('C:/Windows/system32/evil.dll')] }); - const result = await restoreFromZip('/data/tmp/abs.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/abs.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -793,7 +803,7 @@ describe('BACKUP-061 restoreFromZip extraction', () => { // What happens after extraction is BACKUP-042..045's business; this case only // cares that the directory entry never reached the writer. - await restoreFromZip('/data/tmp/dirs.zip').catch(() => undefined); + await restoreFromZip(stubStorage(), '/data/tmp/dirs.zip').catch(() => undefined); expect(fsMock.createWriteStream).toHaveBeenCalledTimes(1); }); @@ -805,7 +815,7 @@ describe('BACKUP-061 restoreFromZip extraction', () => { lying.uncompressedSize = 8; unzipperMock.Open.file.mockResolvedValueOnce({ files: [lying] }); - const result = await restoreFromZip('/data/tmp/liar.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/liar.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -833,7 +843,7 @@ describe('BACKUP-061 restoreFromZip extraction', () => { // A corrupt stream is NOT dressed up as a 400 "too large": it leaves the function // as a throw, which is what makes the controller answer 500 rather than telling the // admin their perfectly-sized backup is over the cap. - await expect(restoreFromZip('/data/tmp/corrupt.zip')).rejects.toThrow('corrupt deflate stream'); + await expect(restoreFromZip(stubStorage(), '/data/tmp/corrupt.zip')).rejects.toThrow('corrupt deflate stream'); expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('restore-'), { recursive: true, force: true }); }); @@ -855,14 +865,26 @@ describe('BACKUP-061 restoreFromZip extraction', () => { dbMock.reinitialize.mockImplementationOnce(() => { throw new Error('database is locked'); }); + const storage = stubStorage(); - const result = await restoreFromZip('/data/tmp/ok.zip'); + const result = await restoreFromZip(storage, '/data/tmp/ok.zip'); // The files already landed, so this is neither a success nor a plain failure: the // admin has to restart, and the message has to say so. expect(result.success).toBe(false); expect(result.status).toBe(500); expect(result.error).toMatch(/restart the server/i); + // A failed reopen leaves no live DB handle for the registry to read: reload + // (and with it any rehydration) is skipped rather than reading through a + // torn/unavailable connection — already reported as "restart required". + expect(storage.reloadConfig).not.toHaveBeenCalled(); + expect(storage.put).not.toHaveBeenCalled(); + // Plugin-tree staging is pure filesystem (plugin-backup.ts), has no DB + // dependency, and is the archive's ONLY copy of that data — it must still + // run even when the DB reopen failed, or extractDir's cleanup right after + // would delete it with no recovery path. fs.cpSync only fires from inside + // stageExtractedPluginTrees in this flow, so its call is the proof. + expect(fsMock.cpSync).toHaveBeenCalled(); }); }); @@ -874,16 +896,8 @@ const DatabaseMock = vi.hoisted(() => vi.fn()); vi.mock('better-sqlite3', () => ({ default: DatabaseMock })); -// --------------------------------------------------------------------------- -// backupFilePath -// --------------------------------------------------------------------------- - -describe('BACKUP-039 backupFilePath', () => { - it('BACKUP-039a — returns a path ending with the given filename', () => { - const result = backupFilePath('backup-test.zip'); - expect(result).toMatch(/backup-test\.zip$/); - }); -}); +// BACKUP-039 (backupFilePath) retired: every consumer addresses backups as +// (category, name) through StorageService now — no absolute path leaves the impl. // --------------------------------------------------------------------------- // backupFileExists @@ -894,17 +908,47 @@ describe('BACKUP-040 backupFileExists', () => { vi.clearAllMocks(); }); - it('BACKUP-040a — returns true when existsSync returns true', () => { - fsMock.existsSync.mockReturnValue(true); - expect(backupFileExists('backup-2026-01-01T00-00-00.zip')).toBe(true); - expect(fsMock.existsSync).toHaveBeenCalledWith( - expect.stringContaining('backup-2026-01-01T00-00-00.zip') + it('BACKUP-040a — returns true when storage.exists resolves true', async () => { + const storage = stubStorage({ exists: vi.fn(async () => true) }); + await expect(backupFileExists(storage, 'backup-2026-01-01T00-00-00.zip')).resolves.toBe(true); + expect(storage.exists).toHaveBeenCalledWith('backups', 'backup-2026-01-01T00-00-00.zip'); + }); + + it('BACKUP-040b — returns false when storage.exists resolves false', async () => { + const storage = stubStorage({ exists: vi.fn(async () => false) }); + await expect(backupFileExists(storage, 'backup-missing.zip')).resolves.toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// sendBackupToResponse +// --------------------------------------------------------------------------- + +describe('BACKUP-062 sendBackupToResponse', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('BACKUP-062a — serves through storage with the res.download attachment header', async () => { + const storage = stubStorage(); + const res = { setHeader: vi.fn() } as unknown as import('express').Response; + + await sendBackupToResponse(storage, 'backup-2026-01-01T00-00-00.zip', res); + + expect(storage.sendToResponse).toHaveBeenCalledOnce(); + expect(storage.sendToResponse).toHaveBeenCalledWith( + 'backups', + 'backup-2026-01-01T00-00-00.zip', + res, + { disposition: 'attachment; filename="backup-2026-01-01T00-00-00.zip"' }, ); }); - it('BACKUP-040b — returns false when existsSync returns false', () => { - fsMock.existsSync.mockReturnValue(false); - expect(backupFileExists('backup-missing.zip')).toBe(false); + it('BACKUP-062b — propagates a storage failure (the controller owns the miss contract)', async () => { + const storage = stubStorage({ sendToResponse: vi.fn(async () => { throw new Error('missing'); }) }); + const res = {} as import('express').Response; + + await expect(sendBackupToResponse(storage, 'backup-x.zip', res)).rejects.toThrow('missing'); }); }); @@ -915,23 +959,20 @@ describe('BACKUP-040 backupFileExists', () => { describe('BACKUP-041 listBackups', () => { beforeEach(() => { vi.clearAllMocks(); - // ensureBackupsDir: backupsDir already exists so mkdirSync is not called - fsMock.existsSync.mockReturnValue(true); }); - it('BACKUP-041a — returns empty array when no .zip files in directory', () => { - fsMock.readdirSync.mockReturnValue([]); - expect(listBackups()).toEqual([]); + it('BACKUP-041a — returns empty array when the category has no objects', async () => { + const storage = stubStorage(); + await expect(listBackups(storage)).resolves.toEqual([]); + expect(storage.list).toHaveBeenCalledWith('backups'); }); - it('BACKUP-041b — returns BackupInfo array for each .zip file', () => { - fsMock.readdirSync.mockReturnValue(['backup-2026-01-01T00-00-00.zip']); - fsMock.statSync.mockReturnValue({ - size: 1024, - mtime: new Date('2026-01-01T00:00:00Z'), + it('BACKUP-041b — returns BackupInfo for each .zip object', async () => { + const storage = stubStorage({ + list: listOf([{ key: 'backup-2026-01-01T00-00-00.zip', size: 1024, mtimeMs: Date.parse('2026-01-01T00:00:00Z') }]), }); - const result = listBackups(); + const result = await listBackups(storage); expect(result).toHaveLength(1); expect(result[0].filename).toBe('backup-2026-01-01T00-00-00.zip'); @@ -940,37 +981,47 @@ describe('BACKUP-041 listBackups', () => { expect(result[0].created_at).toBe('2026-01-01T00:00:00.000Z'); }); - it('BACKUP-041c — sorts results newest-first', () => { - fsMock.readdirSync.mockReturnValue([ - 'backup-2026-01-01T00-00-00.zip', - 'backup-2026-06-01T00-00-00.zip', - ]); - fsMock.statSync.mockImplementation((p: string) => { - if (String(p).includes('2026-01-01')) { - return { size: 512, mtime: new Date('2026-01-01T00:00:00Z') }; - } - return { size: 2048, mtime: new Date('2026-06-01T00:00:00Z') }; + it('BACKUP-041c — sorts results newest-first by mtime', async () => { + const storage = stubStorage({ + list: listOf([ + { key: 'backup-2026-01-01T00-00-00.zip', size: 512, mtimeMs: Date.parse('2026-01-01T00:00:00Z') }, + { key: 'backup-2026-06-01T00-00-00.zip', size: 2048, mtimeMs: Date.parse('2026-06-01T00:00:00Z') }, + ]), }); - const result = listBackups(); + const result = await listBackups(storage); expect(result).toHaveLength(2); expect(result[0].filename).toBe('backup-2026-06-01T00-00-00.zip'); expect(result[1].filename).toBe('backup-2026-01-01T00-00-00.zip'); }); - it('BACKUP-041d — filters out non-.zip files', () => { - fsMock.readdirSync.mockReturnValue([ - 'backup-2026-01-01T00-00-00.zip', - 'README.txt', - 'backup-partial.tar.gz', - ]); - fsMock.statSync.mockReturnValue({ - size: 1024, - mtime: new Date('2026-01-01T00:00:00Z'), + it('BACKUP-041d — filters out non-.zip objects', async () => { + const storage = stubStorage({ + list: listOf([ + { key: 'backup-2026-01-01T00-00-00.zip', size: 1024, mtimeMs: Date.parse('2026-01-01T00:00:00Z') }, + { key: 'README.txt' }, + { key: 'backup-partial.tar.gz' }, + ]), }); - const result = listBackups(); + const result = await listBackups(storage); + + expect(result).toHaveLength(1); + expect(result[0].filename).toBe('backup-2026-01-01T00-00-00.zip'); + }); + + it('BACKUP-041e — skips nested keys (storage.list recurses; the legacy readdir was single-level)', async () => { + // A restore-* staging tree only sits under the backups root when an install + // maps data and uploads to the same directory — it must not surface. + const storage = stubStorage({ + list: listOf([ + { key: 'restore-123/uploads/x.zip' }, + { key: 'backup-2026-01-01T00-00-00.zip', size: 1024, mtimeMs: Date.parse('2026-01-01T00:00:00Z') }, + ]), + }); + + const result = await listBackups(storage); expect(result).toHaveLength(1); expect(result[0].filename).toBe('backup-2026-01-01T00-00-00.zip'); @@ -1014,7 +1065,7 @@ describe('BACKUP-042 restoreFromZip — integrity check fails', () => { return fakeDbInstance; }); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -1050,7 +1101,7 @@ describe('BACKUP-043 restoreFromZip — missing required table', () => { return fakeDbInstance; }); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -1076,7 +1127,7 @@ describe('BACKUP-044 restoreFromZip — Database constructor throws (invalid SQL throw new Error('file is not a database'); }); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result.success).toBe(false); expect(result.status).toBe(400); @@ -1126,7 +1177,7 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { fsMock.copyFileSync.mockReturnValue(undefined); fsMock.rmSync.mockReturnValue(undefined); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result).toEqual({ success: true }); }); @@ -1147,7 +1198,7 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { return true; }); - await restoreFromZip('/data/tmp/upload.zip'); + await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(callOrder.indexOf('closeDb')).toBeLessThan(callOrder.indexOf('copyFileSync')); }); @@ -1167,7 +1218,7 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { }); fsMock.rmSync.mockReturnValue(undefined); - await expect(restoreFromZip('/data/tmp/upload.zip')).rejects.toThrow('disk full'); + await expect(restoreFromZip(stubStorage(), '/data/tmp/upload.zip')).rejects.toThrow('disk full'); expect(dbMock.reinitialize).toHaveBeenCalled(); }); @@ -1186,7 +1237,7 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { fsMock.copyFileSync.mockReturnValue(undefined); fsMock.rmSync.mockReturnValue(undefined); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result).toEqual({ success: true }); // Key copied from the extract dir into the live data dir. @@ -1210,7 +1261,7 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { fsMock.copyFileSync.mockReturnValue(undefined); fsMock.rmSync.mockReturnValue(undefined); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(stubStorage(), '/data/tmp/upload.zip'); expect(result).toEqual({ success: true }); expect(fsMock.copyFileSync).not.toHaveBeenCalledWith( @@ -1218,16 +1269,48 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { expect.stringContaining('.encryption_key'), ); }); + + it('BACKUP-045f — reloadConfig runs after reinitialize and before uploads rehydration (audit #4)', async () => { + // The registry reads storage.* app_settings through the DB handle — which + // is closed and reopened around this restore. reloadConfig() must run + // AFTER that reopen (else it reads a torn/unavailable connection) and + // BEFORE any rehydrated byte is put, so rehydration lands per the + // RESTORED config rather than the stale pre-restore one. + setupSuccessfulExtraction(); + setupAllTablesPresent(); + + const callOrder: string[] = []; + dbMock.reinitialize.mockImplementation(() => { callOrder.push('reinitialize'); }); + + const dirent = (name: string, dir = false) => ({ name, isDirectory: () => dir, isFile: () => !dir }); + fsMock.existsSync.mockImplementation((p: string) => !String(p).endsWith('.encryption_key')); + fsMock.readdirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => { + const s = String(p); + const entries = s.endsWith('uploads') ? [dirent('files', true)] : s.endsWith('files') ? [dirent('a.pdf')] : []; + return (opts?.withFileTypes ? entries : entries.map(e => e.name)) as never; + }); + fsMock.unlinkSync.mockReturnValue(undefined); + fsMock.copyFileSync.mockReturnValue(undefined); + fsMock.rmSync.mockReturnValue(undefined); + + const storage = stubStorage({ + reloadConfig: vi.fn(() => { callOrder.push('reloadConfig'); }), + put: vi.fn(async () => { callOrder.push('put:rehydrate'); }), + }); + + const result = await restoreFromZip(storage, '/data/tmp/upload.zip'); + + expect(result).toEqual({ success: true }); + expect(callOrder).toEqual(['reinitialize', 'reloadConfig', 'put:rehydrate']); + }); }); -describe('BACKUP-046 restoreFromZip — with uploads directory', () => { +describe('BACKUP-046 restoreFromZip — uploads rehydration through storage', () => { beforeEach(() => { vi.clearAllMocks(); }); - it('BACKUP-046a — cpSync is called to copy uploads when they exist in the archive', async () => { - setupSuccessfulExtraction(); - + function setupAllTablesPresent() { const fakeDbInstance = { prepare: vi.fn() .mockReturnValueOnce({ @@ -1247,94 +1330,144 @@ describe('BACKUP-046 restoreFromZip — with uploads directory', () => { DatabaseMock.mockImplementation(function () { return fakeDbInstance; }); + return fakeDbInstance; + } + const dirent = (name: string, dir = false) => ({ name, isDirectory: () => dir, isFile: () => !dir }); + + /** Common fs wiring: travel.db + extracted uploads exist; the extracted tree + * is described per-path via `tree` (walked with { withFileTypes: true }). */ + function setupExtractedUploads(tree: Record[]>) { fsMock.existsSync.mockImplementation((p: string) => { - // travel.db present, extractedUploads present - if (String(p).endsWith('travel.db')) return true; - if (String(p).includes('uploads')) return true; + if (String(p).endsWith('.encryption_key')) return false; return true; }); - fsMock.readdirSync.mockImplementation((p: string) => { - // uploadsDir has one subdirectory 'photos'; 'photos' has one file - if (String(p).includes('uploads') && !String(p).includes('restore-')) { - return ['photos'] as any; - } - if (String(p).includes('photos')) return ['img1.jpg'] as any; - return [] as any; + fsMock.readdirSync.mockImplementation((p: string, opts?: { withFileTypes?: boolean }) => { + const s = String(p); + const key = Object.keys(tree).find(k => s.endsWith(k)); + const entries = key ? tree[key] : []; + return (opts?.withFileTypes ? entries : entries.map(e => e.name)) as never; }); - fsMock.statSync.mockReturnValue({ isDirectory: () => true } as any); fsMock.unlinkSync.mockReturnValue(undefined); fsMock.copyFileSync.mockReturnValue(undefined); - fsMock.cpSync.mockReturnValue(undefined); fsMock.rmSync.mockReturnValue(undefined); + } - await restoreFromZip('/data/tmp/upload.zip'); - - expect(fsMock.cpSync).toHaveBeenCalledWith( - expect.stringContaining('uploads'), - expect.stringContaining('uploads'), - { recursive: true, force: true } - ); - }); - - it('BACKUP-046b — copies into the symlink target, not the symlink itself (#1193)', async () => { - // In Docker, uploadsDir (/app/server/uploads) is a symlink to the mounted - // /app/uploads volume. cpSync(dereference:false) would throw - // ERR_FS_CP_DIR_TO_NON_DIR overwriting the symlink node with a directory. - // The fix resolves the symlink with realpathSync first, so the copy targets - // the real directory behind it. + it('BACKUP-046a — wipes only top-level category objects, then puts each extracted entry (legacy wipe parity)', async () => { setupSuccessfulExtraction(); - - const fakeDbInstance = { - prepare: vi.fn() - .mockReturnValueOnce({ - get: vi.fn().mockReturnValue({ integrity_check: 'ok' }), - }) - .mockReturnValueOnce({ - all: vi.fn().mockReturnValue([ - { name: 'users' }, - { name: 'trips' }, - { name: 'trip_members' }, - { name: 'places' }, - { name: 'days' }, - ]), - }), - close: vi.fn(), - }; - DatabaseMock.mockImplementation(function () { - return fakeDbInstance; + setupAllTablesPresent(); + setupExtractedUploads({ + '/uploads': [dirent('files', true), dirent('journey', true)], + '/uploads/files': [dirent('a.pdf')], + '/uploads/journey': [dirent('thumbs', true)], + '/uploads/journey/thumbs': [dirent('t.jpg')], }); - fsMock.existsSync.mockImplementation((p: string) => { - if (String(p).endsWith('travel.db')) return true; - if (String(p).includes('uploads')) return true; - return true; + // Pre-existing objects: one bare per category plus a nested journey thumb — + // the legacy wipe unlinked one level deep only, so nested keys must survive. + // One delete rejects to pin the swallowed-per-file-error behavior. + const storage = stubStorage({ + list: vi.fn((category: string) => + (async function* () { + yield { key: 'old.bin', size: 1, mtimeMs: 0 }; + if (category === 'journey') yield { key: 'thumbs/old.jpg', size: 1, mtimeMs: 0 }; + })(), + ), + delete: vi.fn(async (category: string) => { + // one category's wipe fails — swallowed per-file, exactly like the old + // unlink loop (exercises the best-effort .catch) + if (category === 'covers') throw new Error('EACCES'); + }), }); - fsMock.readdirSync.mockImplementation((p: string) => { - if (String(p).includes('uploads') && !String(p).includes('restore-')) { - return ['photos'] as any; - } - if (String(p).includes('photos')) return ['img1.jpg'] as any; - return [] as any; - }); - fsMock.statSync.mockReturnValue({ isDirectory: () => true } as any); - fsMock.unlinkSync.mockReturnValue(undefined); - fsMock.copyFileSync.mockReturnValue(undefined); - fsMock.cpSync.mockReturnValue(undefined); - fsMock.rmSync.mockReturnValue(undefined); - // Resolve the uploads symlink to a distinct real target directory. - const REAL_TARGET = '/app/uploads'; - fsMock.realpathSync.mockReturnValueOnce(REAL_TARGET); - const result = await restoreFromZip('/data/tmp/upload.zip'); + const result = await restoreFromZip(storage, '/data/tmp/upload.zip'); expect(result).toEqual({ success: true }); - // The copy destination must be the resolved real path, never the symlink. - expect(fsMock.cpSync).toHaveBeenCalledWith( - expect.stringContaining('uploads'), - REAL_TARGET, - { recursive: true, force: true } - ); + // wipe: bare keys deleted in every archived category, nested keys never + const deleted = (storage.delete as ReturnType).mock.calls; + expect(deleted.every(([, key]) => !(key as string).includes('/'))).toBe(true); + expect(deleted.map(([c]) => c).sort()).toEqual(['avatars', 'covers', 'files', 'journey', 'photos', 'places']); + // rehydration: every extracted file becomes a category put, nested keys intact + expect(storage.put).toHaveBeenCalledWith('files', 'a.pdf', { tmpPath: expect.stringContaining('/uploads/files/a.pdf') }); + expect(storage.put).toHaveBeenCalledWith('journey', 'thumbs/t.jpg', { tmpPath: expect.stringContaining('/uploads/journey/thumbs/t.jpg') }); + // No uploads bulk copy remains (plugin-tree staging still uses cpSync — out of scope). + const cpTargets = fsMock.cpSync.mock.calls.map(c => String(c[1])); + expect(cpTargets.some(t => t.includes('uploads'))).toBe(false); + }); + + it('BACKUP-046b — skips entries that cannot map to a storage key, with a warning (2026-08-17 decision)', async () => { + setupSuccessfulExtraction(); + setupAllTablesPresent(); + setupExtractedUploads({ + '/uploads': [dirent('files', true), dirent('mystery', true), dirent('stray.txt')], + '/uploads/files': [dirent('a.pdf')], + '/uploads/mystery': [dirent('b.bin')], + }); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const { StorageInvalidKeyError } = await import('../../../src/nest/storage/storage.types'); + const storage = stubStorage({ + put: vi.fn(async (_c: string, key: string) => { + // dot-segment keys from old `dot: true` archives are rejected by + // central key validation — the restore must skip, not fail + if (key === 'a.pdf') throw new StorageInvalidKeyError(key); + }), + }); + + const result = await restoreFromZip(storage, '/data/tmp/upload.zip'); + + expect(result).toEqual({ success: true }); + // unknown top-level dir + top-level file + invalid key: all skipped, warned + expect(warn).toHaveBeenCalledWith(expect.stringContaining('mystery/b.bin')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('stray.txt')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('a.pdf')); + warn.mockRestore(); + }); + + it('BACKUP-046c — a genuine put failure still fails the restore', async () => { + setupSuccessfulExtraction(); + setupAllTablesPresent(); + setupExtractedUploads({ + '/uploads': [dirent('files', true)], + '/uploads/files': [dirent('a.pdf')], + }); + const storage = stubStorage({ + put: vi.fn(async () => { throw new Error('ENOSPC: no space left'); }), + }); + + await expect(restoreFromZip(storage, '/data/tmp/upload.zip')).rejects.toThrow('ENOSPC'); + // the DB reopen still ran (finally) — the process is never left closed + expect(dbMock.reinitialize).toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// restoreBackup (stored-zip restore via withLocalFile) +// --------------------------------------------------------------------------- + +describe('BACKUP-063 restoreBackup', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('BACKUP-063a — reads the stored zip through withLocalFile("backups") and returns the restore result', async () => { + const storage = stubStorage({ + withLocalFile: vi.fn(async () => ({ success: true })), + }); + + await expect(restoreBackup(storage, 'backup-2026-01-01T00-00-00.zip')).resolves.toEqual({ success: true }); + expect(storage.withLocalFile).toHaveBeenCalledWith('backups', 'backup-2026-01-01T00-00-00.zip', expect.any(Function)); + }); + + it('BACKUP-063b — the local path handed back by storage feeds the restore core', async () => { + // The default stub invokes the callback with a local path; the restore core + // runs against it for real (empty archive → travel.db missing → 400 result). + fsMock.existsSync.mockReturnValue(false); + unzipperMock.Open.file.mockResolvedValue({ files: [] }); + const storage = stubStorage(); + + const result = await restoreBackup(storage, 'backup-2026-01-01T00-00-00.zip'); + + expect(result).toEqual({ success: false, error: 'Invalid backup: travel.db not found', status: 400 }); }); }); diff --git a/server/tests/unit/nest/cache-sweep-jobs.test.ts b/server/tests/unit/nest/cache-sweep-jobs.test.ts index 00aacfb32..de41f7d46 100644 --- a/server/tests/unit/nest/cache-sweep-jobs.test.ts +++ b/server/tests/unit/nest/cache-sweep-jobs.test.ts @@ -13,8 +13,10 @@ vi.mock('../../../src/nest/audit/audit-log.logger', () => logMock); import { TrekPhotoCacheJob } from '../../../src/nest/memories/trek-photo-cache.job'; import { PlacePhotoCacheJob } from '../../../src/nest/place-photos/place-photo-cache.job'; import { AirtrailSyncJob } from '../../../src/nest/integrations/airtrail-sync.job'; +import { JourneyThumbsJob } from '../../../src/nest/memories/journey-thumbs.job'; import type { TrekPhotoCacheService } from '../../../src/nest/memories/trek-photo-cache.service'; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; +import type { ThumbnailService } from '../../../src/nest/memories/thumbnail.service'; import type { AirtrailSyncService } from '../../../src/nest/integrations/airtrail-sync.service'; import type { DatabaseService } from '../../../src/nest/database/database.service'; import type { CronRegistrarService } from '../../../src/nest/scheduling/cron-registrar.service'; @@ -80,10 +82,10 @@ describe('PlacePhotoCacheJob', () => { expect(registrar.register).toHaveBeenCalledWith('place-photo-cache', '30 3 * * *', expect.any(Function)); }); - it('CSJOB-006 — logs only when something was removed', () => { - make(0).job.sweep(); + it('CSJOB-006 — logs only when something was removed', async () => { + await make(0).job.sweep(); expect(logMock.logInfo).not.toHaveBeenCalled(); - make(3).job.sweep(); + await make(3).job.sweep(); expect(logMock.logInfo).toHaveBeenCalledWith('Place-photo cache cleanup: removed 3 orphaned file(s)/row(s)'); }); @@ -99,6 +101,40 @@ describe('PlacePhotoCacheJob', () => { }); }); +describe('JourneyThumbsJob', () => { + function make(removed: number | (() => number) = 0, enabled = true) { + const registrar = registrarStub(enabled); + const thumbnails = { sweepOrphanThumbs: vi.fn(typeof removed === 'function' ? removed : () => Promise.resolve(removed)) }; + const job = new JourneyThumbsJob(thumbnails as unknown as ThumbnailService, registrar as unknown as CronRegistrarService); + return { job, registrar, thumbnails }; + } + + it('CSJOB-010 — boot sweep runs immediately, then the daily app-tz cron registers', () => { + const { job, registrar, thumbnails } = make(); + job.onApplicationBootstrap(); + expect(thumbnails.sweepOrphanThumbs).toHaveBeenCalledTimes(1); + expect(registrar.register).toHaveBeenCalledWith('journey-thumbs', '0 4 * * *', expect.any(Function)); + }); + + it('CSJOB-011 — logs only when something was removed', async () => { + await make(0).job.sweep(); + expect(logMock.logInfo).not.toHaveBeenCalled(); + await make(2).job.sweep(); + expect(logMock.logInfo).toHaveBeenCalledWith('Journey thumbnail cleanup: removed 2 orphaned thumbnail(s)'); + }); + + it('CSJOB-012 — a throwing sweep is contained to the cleanup log line, and the gate skips the boot sweep', async () => { + const { job } = make(() => { throw new Error('storage down'); }); + await job.sweep(); + expect(logMock.logError).toHaveBeenCalledWith('Journey thumbnail cleanup: storage down'); + + const gated = make(0, false); + gated.job.onApplicationBootstrap(); + expect(gated.thumbnails.sweepOrphanThumbs).not.toHaveBeenCalled(); + expect(gated.registrar.register).not.toHaveBeenCalled(); + }); +}); + describe('AirtrailSyncJob', () => { function make(intervalSetting: string | undefined, enabled = true) { const registrar = registrarStub(enabled); diff --git a/server/tests/unit/nest/collab.controller.test.ts b/server/tests/unit/nest/collab.controller.test.ts index 76426a06f..29292f8ac 100644 --- a/server/tests/unit/nest/collab.controller.test.ts +++ b/server/tests/unit/nest/collab.controller.test.ts @@ -3,6 +3,7 @@ import { HttpException } from '@nestjs/common'; import { CollabController } from '../../../src/nest/collab/collab.controller'; import type { CollabService } from '../../../src/nest/collab/collab.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; const user = { id: 1, username: 'u', role: 'user', email: 'u@example.test' } as User; @@ -18,6 +19,8 @@ function svc(o: Partial = {}): CollabService { } as unknown as CollabService; } +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as StorageService; + function thrown(fn: () => unknown): { status: number; body: unknown } { try { fn(); } catch (err) { expect(err).toBeInstanceOf(HttpException); @@ -42,7 +45,7 @@ describe('CollabController (parity with the legacy /api/trips/:tripId/collab rou describe('notes', () => { it('GET lists', () => { const s = svc({ listNotes: vi.fn().mockReturnValue([{ id: 1 }]) } as Partial); - expect(new CollabController(s).listNotes(user, '5')).toEqual({ notes: [{ id: 1 }] }); + expect(new CollabController(s, storageStub).listNotes(user, '5')).toEqual({ notes: [{ id: 1 }] }); }); it('POST creates + broadcasts + notifies (empty title now 400s in the Zod pipe)', () => { @@ -50,118 +53,119 @@ describe('CollabController (parity with the legacy /api/trips/:tripId/collab rou const broadcast = vi.fn(); const notifyCollab = vi.fn(); const s = svc({ createNote, broadcast, notifyCollab } as Partial); - expect(new CollabController(s).createNote(user, '5', { title: 'T', content: 'c' }, 'sock')).toEqual({ note: { id: 9 } }); + expect(new CollabController(s, storageStub).createNote(user, '5', { title: 'T', content: 'c' }, 'sock')).toEqual({ note: { id: 9 } }); expect(createNote).toHaveBeenCalledWith('5', 1, { title: 'T', content: 'c', category: undefined, color: undefined, website: undefined }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:note:created', { note: { id: 9 } }, 'sock'); expect(notifyCollab).toHaveBeenCalledWith('5', user); }); it('PUT 404 when missing, else updates + broadcasts', () => { - expect(thrown(() => new CollabController(svc({ updateNote: vi.fn().mockReturnValue(null) } as Partial)).updateNote(user, '5', '9', {}))).toEqual({ status: 404, body: { error: 'Note not found' } }); + expect(thrown(() => new CollabController(svc({ updateNote: vi.fn().mockReturnValue(null) } as Partial), storageStub).updateNote(user, '5', '9', {}))).toEqual({ status: 404, body: { error: 'Note not found' } }); const broadcast = vi.fn(); const s = svc({ updateNote: vi.fn().mockReturnValue({ id: 9 }), broadcast } as Partial); - expect(new CollabController(s).updateNote(user, '5', '9', { title: 'b' }, 'sock')).toEqual({ note: { id: 9 } }); + expect(new CollabController(s, storageStub).updateNote(user, '5', '9', { title: 'b' }, 'sock')).toEqual({ note: { id: 9 } }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:note:updated', { note: { id: 9 } }, 'sock'); }); - it('DELETE 404 when missing, else success + broadcasts', () => { - expect(thrown(() => new CollabController(svc({ deleteNote: vi.fn().mockReturnValue(false) } as Partial)).deleteNote(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Note not found' } }); + it('DELETE 404 when missing, else success + broadcasts', async () => { + expect(await thrownAsync(() => new CollabController(svc({ deleteNote: vi.fn().mockResolvedValue(false) } as Partial), storageStub).deleteNote(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Note not found' } }); const broadcast = vi.fn(); - const s = svc({ deleteNote: vi.fn().mockReturnValue(true), broadcast } as Partial); - expect(new CollabController(s).deleteNote(user, '5', '9', 'sock')).toEqual({ success: true }); + const s = svc({ deleteNote: vi.fn().mockResolvedValue(true), broadcast } as Partial); + expect(await new CollabController(s, storageStub).deleteNote(user, '5', '9', 'sock')).toEqual({ success: true }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:note:deleted', { noteId: 9 }, 'sock'); }); }); describe('note files', () => { const file = { filename: 'a.pdf' } as Express.Multer.File; - it('403 without file_upload, 400 without file, 404 unknown note, else returns result', () => { - expect(thrown(() => new CollabController(svc({ canUploadFiles: vi.fn().mockReturnValue(false) })).addNoteFile(user, '5', '9', file))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); - expect(thrown(() => new CollabController(svc()).addNoteFile(user, '5', '9', undefined))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); - expect(thrown(() => new CollabController(svc({ addNoteFile: vi.fn().mockReturnValue(null) } as Partial)).addNoteFile(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Note not found' } }); + it('403 without file_upload, 400 without file, 404 unknown note, else commits + returns result', async () => { + expect(await thrownAsync(() => new CollabController(svc({ canUploadFiles: vi.fn().mockReturnValue(false) }), storageStub).addNoteFile(user, '5', '9', file))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); + expect(await thrownAsync(() => new CollabController(svc(), storageStub).addNoteFile(user, '5', '9', undefined))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); + expect(await thrownAsync(() => new CollabController(svc({ addNoteFile: vi.fn().mockReturnValue(null) } as Partial), storageStub).addNoteFile(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Note not found' } }); const broadcast = vi.fn(); const s = svc({ addNoteFile: vi.fn().mockReturnValue({ file: { id: 3 } }), getFormattedNoteById: vi.fn().mockReturnValue({ id: 9 }), broadcast } as Partial); - expect(new CollabController(s).addNoteFile(user, '5', '9', file, 'sock')).toEqual({ file: { id: 3 } }); + expect(await new CollabController(s, storageStub).addNoteFile(user, '5', '9', file, 'sock')).toEqual({ file: { id: 3 } }); + expect(storageStub.put).toHaveBeenCalledWith('files', 'a.pdf', { tmpPath: undefined }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:note:updated', { note: { id: 9 } }, 'sock'); }); - it('DELETE file 404 when missing, else success', () => { - expect(thrown(() => new CollabController(svc({ deleteNoteFile: vi.fn().mockReturnValue(false) } as Partial)).deleteNoteFile(user, '5', '9', '3'))).toEqual({ status: 404, body: { error: 'File not found' } }); - const s = svc({ deleteNoteFile: vi.fn().mockReturnValue(true), getFormattedNoteById: vi.fn().mockReturnValue({ id: 9 }), broadcast: vi.fn() } as Partial); - expect(new CollabController(s).deleteNoteFile(user, '5', '9', '3')).toEqual({ success: true }); + it('DELETE file 404 when missing, else success', async () => { + expect(await thrownAsync(() => new CollabController(svc({ deleteNoteFile: vi.fn().mockResolvedValue(false) } as Partial), storageStub).deleteNoteFile(user, '5', '9', '3'))).toEqual({ status: 404, body: { error: 'File not found' } }); + const s = svc({ deleteNoteFile: vi.fn().mockResolvedValue(true), getFormattedNoteById: vi.fn().mockReturnValue({ id: 9 }), broadcast: vi.fn() } as Partial); + expect(await new CollabController(s, storageStub).deleteNoteFile(user, '5', '9', '3')).toEqual({ success: true }); }); }); describe('polls', () => { it('POST creates (missing question / <2 options now 400 in the Zod pipe)', () => { const s = svc({ createPoll: vi.fn().mockReturnValue({ id: 7 }), broadcast: vi.fn() } as Partial); - expect(new CollabController(s).createPoll(user, '5', { question: 'q', options: ['a', 'b'] })).toEqual({ poll: { id: 7 } }); + expect(new CollabController(s, storageStub).createPoll(user, '5', { question: 'q', options: ['a', 'b'] })).toEqual({ poll: { id: 7 } }); }); it('vote maps not_found/closed/invalid_index, else broadcasts the poll', () => { - expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'not_found' }) } as Partial)).votePoll(user, '5', '7', { option_index: 0 }))).toEqual({ status: 404, body: { error: 'Poll not found' } }); - expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'closed' }) } as Partial)).votePoll(user, '5', '7', { option_index: 0 }))).toEqual({ status: 400, body: { error: 'Poll is closed' } }); - expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'invalid_index' }) } as Partial)).votePoll(user, '5', '7', { option_index: 9 }))).toEqual({ status: 400, body: { error: 'Invalid option index' } }); + expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'not_found' }) } as Partial), storageStub).votePoll(user, '5', '7', { option_index: 0 }))).toEqual({ status: 404, body: { error: 'Poll not found' } }); + expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'closed' }) } as Partial), storageStub).votePoll(user, '5', '7', { option_index: 0 }))).toEqual({ status: 400, body: { error: 'Poll is closed' } }); + expect(thrown(() => new CollabController(svc({ votePoll: vi.fn().mockReturnValue({ error: 'invalid_index' }) } as Partial), storageStub).votePoll(user, '5', '7', { option_index: 9 }))).toEqual({ status: 400, body: { error: 'Invalid option index' } }); const broadcast = vi.fn(); const s = svc({ votePoll: vi.fn().mockReturnValue({ poll: { id: 7 } }), broadcast } as Partial); - expect(new CollabController(s).votePoll(user, '5', '7', { option_index: 0 }, 'sock')).toEqual({ poll: { id: 7 } }); + expect(new CollabController(s, storageStub).votePoll(user, '5', '7', { option_index: 0 }, 'sock')).toEqual({ poll: { id: 7 } }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:poll:voted', { poll: { id: 7 } }, 'sock'); }); it('close 404 when missing, else broadcasts', () => { - expect(thrown(() => new CollabController(svc({ closePoll: vi.fn().mockReturnValue(null) } as Partial)).closePoll(user, '5', '7'))).toEqual({ status: 404, body: { error: 'Poll not found' } }); + expect(thrown(() => new CollabController(svc({ closePoll: vi.fn().mockReturnValue(null) } as Partial), storageStub).closePoll(user, '5', '7'))).toEqual({ status: 404, body: { error: 'Poll not found' } }); const s = svc({ closePoll: vi.fn().mockReturnValue({ id: 7 }), broadcast: vi.fn() } as Partial); - expect(new CollabController(s).closePoll(user, '5', '7')).toEqual({ poll: { id: 7 } }); + expect(new CollabController(s, storageStub).closePoll(user, '5', '7')).toEqual({ poll: { id: 7 } }); }); it('delete 404 when missing, else success', () => { - expect(thrown(() => new CollabController(svc({ deletePoll: vi.fn().mockReturnValue(false) } as Partial)).deletePoll(user, '5', '7'))).toEqual({ status: 404, body: { error: 'Poll not found' } }); + expect(thrown(() => new CollabController(svc({ deletePoll: vi.fn().mockReturnValue(false) } as Partial), storageStub).deletePoll(user, '5', '7'))).toEqual({ status: 404, body: { error: 'Poll not found' } }); const s = svc({ deletePoll: vi.fn().mockReturnValue(true), broadcast: vi.fn() } as Partial); - expect(new CollabController(s).deletePoll(user, '5', '7')).toEqual({ success: true }); + expect(new CollabController(s, storageStub).deletePoll(user, '5', '7')).toEqual({ success: true }); }); }); describe('messages', () => { it('POST 400 whitespace-only, 400 reply_not_found, else creates + notifies (length checks now in the Zod pipe)', () => { - expect(thrown(() => new CollabController(svc()).createMessage(user, '5', { text: ' ' }))).toEqual({ status: 400, body: { error: 'Message text is required' } }); - expect(thrown(() => new CollabController(svc({ createMessage: vi.fn().mockReturnValue({ error: 'reply_not_found' }) } as Partial)).createMessage(user, '5', { text: 'hi', reply_to: 99 }))).toEqual({ status: 400, body: { error: 'Reply target message not found' } }); + expect(thrown(() => new CollabController(svc(), storageStub).createMessage(user, '5', { text: ' ' }))).toEqual({ status: 400, body: { error: 'Message text is required' } }); + expect(thrown(() => new CollabController(svc({ createMessage: vi.fn().mockReturnValue({ error: 'reply_not_found' }) } as Partial), storageStub).createMessage(user, '5', { text: 'hi', reply_to: 99 }))).toEqual({ status: 400, body: { error: 'Reply target message not found' } }); const broadcast = vi.fn(); const notifyCollab = vi.fn(); const s = svc({ createMessage: vi.fn().mockReturnValue({ message: { id: 3 } }), broadcast, notifyCollab } as Partial); - expect(new CollabController(s).createMessage(user, '5', { text: 'hello' }, 'sock')).toEqual({ message: { id: 3 } }); + expect(new CollabController(s, storageStub).createMessage(user, '5', { text: 'hello' }, 'sock')).toEqual({ message: { id: 3 } }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:message:created', { message: { id: 3 } }, 'sock'); expect(notifyCollab).toHaveBeenCalledWith('5', user, 'hello'); }); it('react 404 unknown, else broadcasts reactions (empty emoji now 400s in the Zod pipe)', () => { - expect(thrown(() => new CollabController(svc({ reactMessage: vi.fn().mockReturnValue({ found: false, reactions: [] }) } as Partial)).react(user, '5', '3', { emoji: '👍' }))).toEqual({ status: 404, body: { error: 'Message not found' } }); + expect(thrown(() => new CollabController(svc({ reactMessage: vi.fn().mockReturnValue({ found: false, reactions: [] }) } as Partial), storageStub).react(user, '5', '3', { emoji: '👍' }))).toEqual({ status: 404, body: { error: 'Message not found' } }); const broadcast = vi.fn(); const s = svc({ reactMessage: vi.fn().mockReturnValue({ found: true, reactions: [{ emoji: '👍', count: 1 }] }), broadcast } as Partial); - expect(new CollabController(s).react(user, '5', '3', { emoji: '👍' }, 'sock')).toEqual({ reactions: [{ emoji: '👍', count: 1 }] }); + expect(new CollabController(s, storageStub).react(user, '5', '3', { emoji: '👍' }, 'sock')).toEqual({ reactions: [{ emoji: '👍', count: 1 }] }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:message:reacted', { messageId: 3, reactions: [{ emoji: '👍', count: 1 }] }, 'sock'); }); it('delete maps not_found (404) / not_owner (403), else success with username', () => { - expect(thrown(() => new CollabController(svc({ deleteMessage: vi.fn().mockReturnValue({ error: 'not_found' }) } as Partial)).deleteMessage(user, '5', '3'))).toEqual({ status: 404, body: { error: 'Message not found' } }); - expect(thrown(() => new CollabController(svc({ deleteMessage: vi.fn().mockReturnValue({ error: 'not_owner' }) } as Partial)).deleteMessage(user, '5', '3'))).toEqual({ status: 403, body: { error: 'You can only delete your own messages' } }); + expect(thrown(() => new CollabController(svc({ deleteMessage: vi.fn().mockReturnValue({ error: 'not_found' }) } as Partial), storageStub).deleteMessage(user, '5', '3'))).toEqual({ status: 404, body: { error: 'Message not found' } }); + expect(thrown(() => new CollabController(svc({ deleteMessage: vi.fn().mockReturnValue({ error: 'not_owner' }) } as Partial), storageStub).deleteMessage(user, '5', '3'))).toEqual({ status: 403, body: { error: 'You can only delete your own messages' } }); const broadcast = vi.fn(); const s = svc({ deleteMessage: vi.fn().mockReturnValue({ username: 'bob' }), broadcast } as Partial); - expect(new CollabController(s).deleteMessage(user, '5', '3', 'sock')).toEqual({ success: true }); + expect(new CollabController(s, storageStub).deleteMessage(user, '5', '3', 'sock')).toEqual({ success: true }); expect(broadcast).toHaveBeenCalledWith('5', 'collab:message:deleted', { messageId: 3, username: 'bob' }, 'sock'); }); }); describe('link preview', () => { it('400 without url, maps an error result to 400, else returns the preview', async () => { - expect(await thrownAsync(() => new CollabController(svc()).linkPreview(user, '5', undefined))).toEqual({ status: 400, body: { error: 'URL is required' } }); - expect(await thrownAsync(() => new CollabController(svc({ linkPreview: vi.fn().mockResolvedValue({ error: 'bad url' }) } as Partial)).linkPreview(user, '5', 'http://x'))).toEqual({ status: 400, body: { error: 'bad url' } }); + expect(await thrownAsync(() => new CollabController(svc(), storageStub).linkPreview(user, '5', undefined))).toEqual({ status: 400, body: { error: 'URL is required' } }); + expect(await thrownAsync(() => new CollabController(svc({ linkPreview: vi.fn().mockResolvedValue({ error: 'bad url' }) } as Partial), storageStub).linkPreview(user, '5', 'http://x'))).toEqual({ status: 400, body: { error: 'bad url' } }); const s = svc({ linkPreview: vi.fn().mockResolvedValue({ title: 'T', description: null, image: null, url: 'http://x' }) } as Partial); - expect(await new CollabController(s).linkPreview(user, '5', 'http://x')).toEqual({ title: 'T', description: null, image: null, url: 'http://x' }); + expect(await new CollabController(s, storageStub).linkPreview(user, '5', 'http://x')).toEqual({ title: 'T', description: null, image: null, url: 'http://x' }); }); it('falls back to a null preview when the service throws', async () => { const s = svc({ linkPreview: vi.fn().mockRejectedValue(new Error('network')) } as Partial); - expect(await new CollabController(s).linkPreview(user, '5', 'http://x')).toEqual({ title: null, description: null, image: null, url: 'http://x' }); + expect(await new CollabController(s, storageStub).linkPreview(user, '5', 'http://x')).toEqual({ title: null, description: null, image: null, url: 'http://x' }); }); }); }); diff --git a/server/tests/unit/nest/collab.service.test.ts b/server/tests/unit/nest/collab.service.test.ts index a64990f66..d85d84854 100644 --- a/server/tests/unit/nest/collab.service.test.ts +++ b/server/tests/unit/nest/collab.service.test.ts @@ -67,8 +67,10 @@ import { PermissionsService } from '../../../src/nest/permissions/permissions.se import { CollabService } from '../../../src/nest/collab/collab.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { notificationsStub } from '../../helpers/notifications'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; -const svc = new CollabService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new RealtimeService(), notificationsStub()); +const collabFx = makeStorageFixture('files/'); +const svc = new CollabService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new RealtimeService(), notificationsStub(), collabFx.storage); beforeAll(() => { createTables(testDb); @@ -421,7 +423,7 @@ describe('hardening', () => { it('COLLAB-SVC-034: votePoll switch is atomic — prior vote survives a failed INSERT', () => { const { user1, trip } = setup(); const dbs = new DatabaseService(testDb); - const failing = new CollabService(dbs, new PermissionsService(dbs), new RealtimeService(), notificationsStub()); + const failing = new CollabService(dbs, new PermissionsService(dbs), new RealtimeService(), notificationsStub(), collabFx.storage); const poll = failing.createPoll(trip.id, user1.id, { question: 'Q?', options: ['A', 'B'] }); failing.votePoll(trip.id, poll!.id, user1.id, 0); @@ -439,10 +441,10 @@ describe('hardening', () => { expect(votes).toEqual([{ option_index: 0 }]); }); - it('COLLAB-SVC-035: deleteNote is atomic — trip_files rows survive a failed note DELETE', () => { + it('COLLAB-SVC-035: deleteNote is atomic — trip_files rows survive a failed note DELETE', async () => { const { user1, trip } = setup(); const dbs = new DatabaseService(testDb); - const failing = new CollabService(dbs, new PermissionsService(dbs), new RealtimeService(), notificationsStub()); + const failing = new CollabService(dbs, new PermissionsService(dbs), new RealtimeService(), notificationsStub(), collabFx.storage); const note = failing.createNote(trip.id, user1.id, { title: 'With file' }); testDb.prepare('INSERT INTO trip_files (trip_id, note_id, filename, original_name) VALUES (?, ?, ?, ?)') .run(trip.id, note.id, 'files/a.pdf', 'a.pdf'); @@ -452,13 +454,32 @@ describe('hardening', () => { if (sql.includes('DELETE FROM collab_notes')) throw new Error('boom'); return realRun(sql, ...params); }); - expect(() => failing.deleteNote(trip.id, note.id)).toThrow('boom'); + await expect(failing.deleteNote(trip.id, note.id)).rejects.toThrow('boom'); spy.mockRestore(); expect(testDb.prepare('SELECT COUNT(*) as c FROM trip_files WHERE note_id = ?').get(note.id)).toEqual({ c: 1 }); expect(testDb.prepare('SELECT COUNT(*) as c FROM collab_notes WHERE id = ?').get(note.id)).toEqual({ c: 1 }); }); + it('COLLAB-SVC-036: a failing storage delete is swallowed — note + file deletes still succeed', async () => { + const { user1, trip } = setup(); + const dbs = new DatabaseService(testDb); + const failingStorage = { delete: vi.fn().mockRejectedValue(new Error('EACCES')) }; + const failing = new CollabService(dbs, new PermissionsService(dbs), new RealtimeService(), notificationsStub(), failingStorage as unknown as import('../../../src/nest/storage/storage.service').StorageService); + const note = failing.createNote(trip.id, user1.id, { title: 'Sticky file' }); + testDb.prepare('INSERT INTO trip_files (trip_id, note_id, filename, original_name) VALUES (?, ?, ?, ?)') + .run(trip.id, note.id, 'stuck.pdf', 'stuck.pdf'); + const fileId = (testDb.prepare('SELECT id FROM trip_files WHERE note_id = ?').get(note.id) as { id: number }).id; + + expect(await failing.deleteNoteFile(trip.id, note.id, fileId)).toBe(true); + expect(testDb.prepare('SELECT COUNT(*) as c FROM trip_files WHERE id = ?').get(fileId)).toEqual({ c: 0 }); + + testDb.prepare('INSERT INTO trip_files (trip_id, note_id, filename, original_name) VALUES (?, ?, ?, ?)') + .run(trip.id, note.id, 'stuck2.pdf', 'stuck2.pdf'); + expect(await failing.deleteNote(trip.id, note.id)).toBe(true); + expect(testDb.prepare('SELECT COUNT(*) as c FROM collab_notes WHERE id = ?').get(note.id)).toEqual({ c: 0 }); + }); + it('COLLAB-SVC-036: getFormattedNoteById is trip-scoped and null-safe', () => { const { user1, trip } = setup(); const otherTrip = createTrip(testDb, user1.id); diff --git a/server/tests/unit/nest/collections.controller.test.ts b/server/tests/unit/nest/collections.controller.test.ts index d264d051e..427e2e95a 100644 --- a/server/tests/unit/nest/collections.controller.test.ts +++ b/server/tests/unit/nest/collections.controller.test.ts @@ -3,8 +3,11 @@ import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.serv import { HttpException } from '@nestjs/common'; import { CollectionsController } from '../../../src/nest/collections/collections.controller'; import type { CollectionsService } from '../../../src/nest/collections/collections.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as StorageService; + const user = { id: 1, username: 'owner', role: 'user', email: 'u@example.test' } as User; /** Service mock — access granted + owner by default; each method a spy. */ @@ -53,12 +56,21 @@ function thrown(fn: () => unknown): { status: number; body: unknown } { throw new Error('expected the handler to throw'); } +async function thrownAsync(fn: () => Promise): Promise<{ status: number; body: unknown }> { + try { await fn(); } catch (err) { + expect(err).toBeInstanceOf(HttpException); + const e = err as HttpException; + return { status: e.getStatus(), body: e.getResponse() }; + } + throw new Error('expected the handler to throw'); +} + describe('CollectionsController', () => { afterEach(() => { delete process.env.DEMO_MODE; }); it('lists / creates / reads / deletes lists (pass-throughs)', () => { const svc = makeService(); - const c = new CollectionsController(svc, new RuntimeEnvService()); + const c = new CollectionsController(svc, new RuntimeEnvService(), storageStub); expect(c.list(user)).toEqual({ collections: [], incomingInvites: [] }); expect(c.create(user, { name: 'Italy' } as never)).toEqual({ id: 7 }); expect(c.get(user, '3')).toEqual({ collection: { id: 3 }, places: [] }); @@ -74,22 +86,22 @@ describe('CollectionsController', () => { // bad payload before the handler runs. it('reorders a valid array', () => { const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).reorder(user, { orderedIds: [3, 1, 2] } as never)).toEqual({ success: true }); + expect(new CollectionsController(svc, new RuntimeEnvService(), storageStub).reorder(user, { orderedIds: [3, 1, 2] } as never)).toEqual({ success: true }); expect(svc.reorderCollections).toHaveBeenCalledWith(1, [3, 1, 2]); }); }); describe('places', () => { - it('savePlace / updatePlace / setStatus / deletePlace forward the socket id', () => { + it('savePlace / updatePlace / setStatus / deletePlace forward the socket id', async () => { const svc = makeService(); - const c = new CollectionsController(svc, new RuntimeEnvService()); + const c = new CollectionsController(svc, new RuntimeEnvService(), storageStub); c.savePlace(user, { collection_id: 3, name: 'A' } as never, 'sid'); expect(svc.savePlace).toHaveBeenCalledWith(1, { collection_id: 3, name: 'A' }, 'sid'); c.updatePlace(user, '9', { name: 'B' } as never, 'sid'); expect(svc.updatePlace).toHaveBeenCalledWith(1, 9, { name: 'B' }, 'sid'); c.setStatus(user, '9', { status: 'want' } as never, 'sid'); expect(svc.setStatus).toHaveBeenCalledWith(1, 9, 'want', 'sid'); - expect(c.deletePlace(user, '9', 'sid')).toEqual({ success: true }); + expect(await c.deletePlace(user, '9', 'sid')).toEqual({ success: true }); expect(svc.deletePlace).toHaveBeenCalledWith(1, 9, 'sid'); c.saveFromTrip(user, { collection_id: 3, source_trip_id: 5, source_place_id: 8 } as never, 'sid'); expect(svc.saveFromTripPlace).toHaveBeenCalledWith(1, 3, 5, 8, undefined, 'sid'); @@ -99,9 +111,9 @@ describe('CollectionsController', () => { // The legacy 'ids must be an array of numbers' 400 is gone: // collectionDeleteManyRequestSchema types the array, so the pipe rejects a // bad payload before the handler runs. - it('deleteMany deletes a valid list', () => { + it('deleteMany deletes a valid list', async () => { const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).deleteMany(user, { ids: [1, 2] } as never, 'sid')).toEqual({ deleted: [1, 2] }); + expect(await new CollectionsController(svc, new RuntimeEnvService(), storageStub).deleteMany(user, { ids: [1, 2] } as never, 'sid')).toEqual({ deleted: [1, 2] }); expect(svc.deletePlacesMany).toHaveBeenCalledWith(1, [1, 2], 'sid'); }); }); @@ -109,12 +121,12 @@ describe('CollectionsController', () => { describe('membership lookup', () => { it('parses lat/lng when present', () => { const svc = makeService(); - new CollectionsController(svc, new RuntimeEnvService()).membership(user, 'gpid', undefined, 'Rome', '41.9', '12.5'); + new CollectionsController(svc, new RuntimeEnvService(), storageStub).membership(user, 'gpid', undefined, 'Rome', '41.9', '12.5'); expect(svc.findMembership).toHaveBeenCalledWith(1, { google_place_id: 'gpid', google_ftid: undefined, name: 'Rome', lat: 41.9, lng: 12.5 }); }); it('leaves lat/lng undefined when absent', () => { const svc = makeService(); - new CollectionsController(svc, new RuntimeEnvService()).membership(user); + new CollectionsController(svc, new RuntimeEnvService(), storageStub).membership(user); expect(svc.findMembership).toHaveBeenCalledWith(1, { google_place_id: undefined, google_ftid: undefined, name: undefined, lat: undefined, lng: undefined }); }); }); @@ -122,29 +134,29 @@ describe('CollectionsController', () => { describe('invites (owner-gated)', () => { it('403 when a non-owner invites', () => { const svc = makeService({ isOwner: vi.fn().mockReturnValue(false) }); - expect(thrown(() => new CollectionsController(svc, new RuntimeEnvService()).invite(user, { collection_id: 3, user_id: 2 } as never))) + expect(thrown(() => new CollectionsController(svc, new RuntimeEnvService(), storageStub).invite(user, { collection_id: 3, user_id: 2 } as never))) .toEqual({ status: 403, body: { error: 'Only the owner can invite' } }); }); it('surfaces a sendInvite error with its status', () => { const svc = makeService({ sendInvite: vi.fn().mockReturnValue({ error: 'Already a member', status: 409 }) }); - expect(thrown(() => new CollectionsController(svc, new RuntimeEnvService()).invite(user, { collection_id: 3, user_id: 2 } as never))) + expect(thrown(() => new CollectionsController(svc, new RuntimeEnvService(), storageStub).invite(user, { collection_id: 3, user_id: 2 } as never))) .toEqual({ status: 409, body: { error: 'Already a member' } }); }); it('invites successfully as owner', () => { const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).invite(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); + expect(new CollectionsController(svc, new RuntimeEnvService(), storageStub).invite(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); }); it('accept surfaces an error, else succeeds', () => { const bad = makeService({ acceptInvite: vi.fn().mockReturnValue({ error: 'Gone', status: 404 }) }); - expect(thrown(() => new CollectionsController(bad, new RuntimeEnvService()).acceptInvite(user, { collection_id: 3 } as never, 'sid'))) + expect(thrown(() => new CollectionsController(bad, new RuntimeEnvService(), storageStub).acceptInvite(user, { collection_id: 3 } as never, 'sid'))) .toEqual({ status: 404, body: { error: 'Gone' } }); const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).acceptInvite(user, { collection_id: 3 } as never, 'sid')).toEqual({ success: true }); + expect(new CollectionsController(svc, new RuntimeEnvService(), storageStub).acceptInvite(user, { collection_id: 3 } as never, 'sid')).toEqual({ success: true }); expect(svc.acceptInvite).toHaveBeenCalledWith(1, 3, 'sid'); }); it('decline + leave forward the socket id', () => { const svc = makeService(); - const c = new CollectionsController(svc, new RuntimeEnvService()); + const c = new CollectionsController(svc, new RuntimeEnvService(), storageStub); expect(c.declineInvite(user, { collection_id: 3 } as never, 'sid')).toEqual({ success: true }); expect(svc.declineInvite).toHaveBeenCalledWith(1, 3, 'sid'); expect(c.leave(user, { collection_id: 3 } as never, 'sid')).toEqual({ success: true }); @@ -152,64 +164,66 @@ describe('CollectionsController', () => { }); it('cancel is owner-gated', () => { const notOwner = makeService({ isOwner: vi.fn().mockReturnValue(false) }); - expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService()).cancelInvite(user, { collection_id: 3, user_id: 2 } as never))) + expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService(), storageStub).cancelInvite(user, { collection_id: 3, user_id: 2 } as never))) .toEqual({ status: 403, body: { error: 'Only the owner can cancel invites' } }); const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).cancelInvite(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); + expect(new CollectionsController(svc, new RuntimeEnvService(), storageStub).cancelInvite(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); }); }); describe('members', () => { it('removeMember is owner-gated', () => { const notOwner = makeService({ isOwner: vi.fn().mockReturnValue(false) }); - expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService()).removeMember(user, { collection_id: 3, user_id: 2 } as never))) + expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService(), storageStub).removeMember(user, { collection_id: 3, user_id: 2 } as never))) .toEqual({ status: 403, body: { error: 'Only the owner can remove members' } }); const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).removeMember(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); + expect(new CollectionsController(svc, new RuntimeEnvService(), storageStub).removeMember(user, { collection_id: 3, user_id: 2 } as never)).toEqual({ success: true }); expect(svc.removeMember).toHaveBeenCalledWith(1, 3, 2); }); it('availableUsers is owner-gated', () => { const notOwner = makeService({ isOwner: vi.fn().mockReturnValue(false) }); - expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService()).availableUsers(user, '3'))) + expect(thrown(() => new CollectionsController(notOwner, new RuntimeEnvService(), storageStub).availableUsers(user, '3'))) .toEqual({ status: 403, body: { error: 'Only the owner can manage members' } }); - expect(new CollectionsController(makeService(), new RuntimeEnvService()).availableUsers(user, '3')).toEqual({ users: [{ id: 2 }] }); + expect(new CollectionsController(makeService(), new RuntimeEnvService(), storageStub).availableUsers(user, '3')).toEqual({ users: [{ id: 2 }] }); }); }); describe('uploadCover', () => { const file = { filename: 'x.jpg' } as Express.Multer.File; - it('403 in demo mode for a demo user', () => { + it('403 in demo mode for a demo user', async () => { process.env.DEMO_MODE = 'true'; const demo = { ...user, email: 'demo@trek.app' } as User; - expect(thrown(() => new CollectionsController(makeService(), new RuntimeEnvService()).uploadCover(demo, '3', file))) + expect(await thrownAsync(() => new CollectionsController(makeService(), new RuntimeEnvService(), storageStub).uploadCover(demo, '3', file))) .toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' } }); }); - it('400 when no file was uploaded', () => { - expect(thrown(() => new CollectionsController(makeService(), new RuntimeEnvService()).uploadCover(user, '3', undefined))) + it('400 when no file was uploaded', async () => { + expect(await thrownAsync(() => new CollectionsController(makeService(), new RuntimeEnvService(), storageStub).uploadCover(user, '3', undefined))) .toEqual({ status: 400, body: { error: 'No image uploaded' } }); }); - it('stores the cover and forwards the socket id', () => { + it('commits + stores the cover and forwards the socket id', async () => { const svc = makeService(); - expect(new CollectionsController(svc, new RuntimeEnvService()).uploadCover(user, '3', file, 'sid')).toEqual({ id: 3 }); + expect(await new CollectionsController(svc, new RuntimeEnvService(), storageStub).uploadCover(user, '3', file, 'sid')).toEqual({ id: 3 }); + expect(storageStub.put).toHaveBeenCalledWith('covers', 'x.jpg', { tmpPath: undefined }); expect(svc.setCollectionCover).toHaveBeenCalledWith(1, 3, '/uploads/covers/x.jpg', 'sid'); }); }); describe('uploadPlaceImage', () => { const file = { filename: 'x.jpg' } as Express.Multer.File; - it('403 in demo mode for a demo user', () => { + it('403 in demo mode for a demo user', async () => { process.env.DEMO_MODE = 'true'; const demo = { ...user, email: 'demo@trek.app' } as User; - expect(thrown(() => new CollectionsController(makeService(), new RuntimeEnvService()).uploadPlaceImage(demo, '9', file))) + expect(await thrownAsync(() => new CollectionsController(makeService(), new RuntimeEnvService(), storageStub).uploadPlaceImage(demo, '9', file))) .toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' } }); }); - it('400 when no file was uploaded', () => { - expect(thrown(() => new CollectionsController(makeService(), new RuntimeEnvService()).uploadPlaceImage(user, '9', undefined))) + it('400 when no file was uploaded', async () => { + expect(await thrownAsync(() => new CollectionsController(makeService(), new RuntimeEnvService(), storageStub).uploadPlaceImage(user, '9', undefined))) .toEqual({ status: 400, body: { error: 'No image uploaded' } }); }); - it('stores the place image via /uploads/places and forwards the socket id', () => { + it('commits + stores the place image via /uploads/places and forwards the socket id', async () => { const svc = makeService({ setPlaceImage: vi.fn().mockReturnValue({ id: 9 }) }); - expect(new CollectionsController(svc, new RuntimeEnvService()).uploadPlaceImage(user, '9', file, 'sid')).toEqual({ id: 9 }); + expect(await new CollectionsController(svc, new RuntimeEnvService(), storageStub).uploadPlaceImage(user, '9', file, 'sid')).toEqual({ id: 9 }); + expect(storageStub.put).toHaveBeenCalledWith('places', 'x.jpg', { tmpPath: undefined }); expect(svc.setPlaceImage).toHaveBeenCalledWith(1, 9, '/uploads/places/x.jpg', 'sid'); }); }); @@ -217,7 +231,7 @@ describe('CollectionsController', () => { describe('labels', () => { it('create / update / delete / assign / unassign forward user + socket id', () => { const svc = makeService(); - const c = new CollectionsController(svc, new RuntimeEnvService()); + const c = new CollectionsController(svc, new RuntimeEnvService(), storageStub); c.createLabel(user, { collection_id: 3, name: 'Berlin', color: '#ff0000' } as never, 'sock'); expect(svc.createLabel).toHaveBeenCalledWith(1, 3, 'Berlin', '#ff0000', 'sock'); diff --git a/server/tests/unit/nest/collections.service.test.ts b/server/tests/unit/nest/collections.service.test.ts index 06d5b17f6..ba0abe91c 100644 --- a/server/tests/unit/nest/collections.service.test.ts +++ b/server/tests/unit/nest/collections.service.test.ts @@ -53,14 +53,14 @@ import { PermissionsService } from '../../../src/nest/permissions/permissions.se import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { CollectionsService } from '../../../src/nest/collections/collections.service'; import { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; -import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; -import { PLACE_IMAGES_DIR } from '../../../src/nest/places/place-image'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; import { notificationsStub } from '../../helpers/notifications'; -const svc = new CollectionsService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new RealtimeService(), notificationsStub(notifSend)); +const storageFx = makeStorageFixture(''); +const svc = new CollectionsService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new RealtimeService(), notificationsStub(notifSend), storageFx.storage); // The real cache: these cases assert what removeIfUnreferenced actually does // about collection_places (#1081), so a stub would assert nothing. -const photoCache = new PlacePhotoCacheService(new DatabaseService(testDb), new RuntimeEnvService()); +const photoCache = new PlacePhotoCacheService(new DatabaseService(testDb), makeStorageFixture('photos/google/').storage); const removeIfUnreferenced = (id: string) => photoCache.removeIfUnreferenced(id); function clearCollections() { @@ -222,7 +222,7 @@ describe('status + updatePlace move', () => { expect(svc.setStatus(u.id, p.id, 'visited').status).toBe('visited'); }); - it('COLLECTIONS-SVC-017: updatePlace moves to another list (asserts access on target, resets owner_id)', () => { + it('COLLECTIONS-SVC-017: updatePlace moves to another list (asserts access on target, resets owner_id)', async () => { const owner = createUser(testDb).user; const a = svc.createCollection(owner.id, { name: 'A' }); const targetOwner = createUser(testDb).user; @@ -231,38 +231,38 @@ describe('status + updatePlace move', () => { testDb.prepare("INSERT INTO collection_members (collection_id, user_id, status) VALUES (?, ?, 'accepted')").run(b.id, owner.id); const p = svc.savePlace(owner.id, { collection_id: a.id, name: 'Movable' }).place!; - const moved = svc.updatePlace(owner.id, p.id, { collection_id: b.id }); + const moved = await svc.updatePlace(owner.id, p.id, { collection_id: b.id }); expect(moved.collection_id).toBe(b.id); const row = testDb.prepare('SELECT owner_id FROM collection_places WHERE id = ?').get(p.id) as { owner_id: number }; expect(row.owner_id).toBe(targetOwner.id); // reset to the target collection's owner }); - it('COLLECTIONS-SVC-018: updatePlace move to an inaccessible target is rejected', () => { + it('COLLECTIONS-SVC-018: updatePlace move to an inaccessible target is rejected', async () => { const owner = createUser(testDb).user; const a = svc.createCollection(owner.id, { name: 'A' }); const stranger = createUser(testDb).user; const b = svc.createCollection(stranger.id, { name: 'B' }); const p = svc.savePlace(owner.id, { collection_id: a.id, name: 'X' }).place!; - expect(() => svc.updatePlace(owner.id, p.id, { collection_id: b.id })).toThrow(); + await expect(svc.updatePlace(owner.id, p.id, { collection_id: b.id })).rejects.toThrow(); }); // #1870: the address column was missing from the UPDATE set, so a typo or a // moved restaurant could only be fixed by deleting and re-adding the place. - it('COLLECTIONS-SVC-019: updatePlace corrects the address and clears it with null', () => { + it('COLLECTIONS-SVC-019: updatePlace corrects the address and clears it with null', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Rome' }); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Trattoria', address: 'Via Vechia 1' }).place!; - expect(svc.updatePlace(u.id, p.id, { address: 'Via Nuova 1' }).address).toBe('Via Nuova 1'); - expect(svc.updatePlace(u.id, p.id, { address: null }).address).toBeNull(); + expect((await svc.updatePlace(u.id, p.id, { address: 'Via Nuova 1' })).address).toBe('Via Nuova 1'); + expect((await svc.updatePlace(u.id, p.id, { address: null })).address).toBeNull(); }); - it('COLLECTIONS-SVC-019b: an update without an address leaves the stored one alone', () => { + it('COLLECTIONS-SVC-019b: an update without an address leaves the stored one alone', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Rome' }); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Trattoria', address: 'Via Vechia 1' }).place!; - expect(svc.updatePlace(u.id, p.id, { name: 'Trattoria da Enzo' }).address).toBe('Via Vechia 1'); + expect((await svc.updatePlace(u.id, p.id, { name: 'Trattoria da Enzo' })).address).toBe('Via Vechia 1'); }); }); @@ -329,14 +329,14 @@ describe('copyToTrip', () => { // ── delete + delete-many ───────────────────────────────────────────────────── describe('delete places', () => { - it('COLLECTIONS-SVC-024: deletePlace + deletePlacesMany assert access', () => { + it('COLLECTIONS-SVC-024: deletePlace + deletePlacesMany assert access', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'D' }); const p1 = svc.savePlace(u.id, { collection_id: col.id, name: 'A' }).place!; const p2 = svc.savePlace(u.id, { collection_id: col.id, name: 'B' }).place!; - svc.deletePlace(u.id, p1.id); + await svc.deletePlace(u.id, p1.id); expect(testDb.prepare('SELECT COUNT(*) n FROM collection_places WHERE collection_id = ?').get(col.id)).toEqual({ n: 1 }); - expect(svc.deletePlacesMany(u.id, [p2.id])).toEqual([p2.id]); + expect(await svc.deletePlacesMany(u.id, [p2.id])).toEqual([p2.id]); expect(testDb.prepare('SELECT COUNT(*) n FROM collection_places WHERE collection_id = ?').get(col.id)).toEqual({ n: 0 }); }); }); @@ -501,22 +501,22 @@ describe('owner_id semantics', () => { // ── Photo-cache guard ──────────────────────────────────────────────────────── describe('photo-cache widening', () => { - it('COLLECTIONS-SVC-042: a collection_places row keeps a photo no places row references', () => { + it('COLLECTIONS-SVC-042: a collection_places row keeps a photo no places row references', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Photos' }); // cache meta for place_id 'gp-x', referenced ONLY by a collection_places row. testDb.prepare('INSERT INTO google_place_photo_meta (place_id, attribution, fetched_at) VALUES (?, ?, ?)').run('gp-x', null, Date.now()); svc.savePlace(u.id, { collection_id: col.id, name: 'Cached', google_place_id: 'gp-x' }); - removeIfUnreferenced('gp-x'); // would evict if isReferenced ignored collection_places + await removeIfUnreferenced('gp-x'); // would evict if isReferenced ignored collection_places const meta = testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('gp-x'); expect(meta).toBeDefined(); }); - it('COLLECTIONS-SVC-043: an unreferenced photo is still reclaimable', () => { + it('COLLECTIONS-SVC-043: an unreferenced photo is still reclaimable', async () => { testDb.prepare('INSERT INTO google_place_photo_meta (place_id, attribution, fetched_at) VALUES (?, ?, ?)').run('gp-orphan', null, Date.now()); - removeIfUnreferenced('gp-orphan'); + await removeIfUnreferenced('gp-orphan'); const meta = testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('gp-orphan'); expect(meta).toBeUndefined(); }); @@ -553,14 +553,14 @@ describe('collection labels', () => { expect(svc.createLabel(editor.id, col.id, 'Museums').id).toBeGreaterThan(0); }); - it('COLLECTIONS-SVC-052: updatePlace label_ids sets labels; a label from another list is ignored', () => { + it('COLLECTIONS-SVC-052: updatePlace label_ids sets labels; a label from another list is ignored', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'DE' }); const other = svc.createCollection(u.id, { name: 'Other' }); const l1 = svc.createLabel(u.id, col.id, 'Berlin'); const foreign = svc.createLabel(u.id, other.id, 'Paris'); const place = svc.savePlace(u.id, { collection_id: col.id, name: 'Gate' }).place!; - svc.updatePlace(u.id, place.id, { label_ids: [l1.id, foreign.id] }); + await svc.updatePlace(u.id, place.id, { label_ids: [l1.id, foreign.id] }); const stored = svc.getCollection(u.id, col.id).places.find(p => p.id === place.id)!; expect(stored.label_ids).toEqual([l1.id]); }); @@ -581,27 +581,27 @@ describe('collection labels', () => { expect(after.find(p => p.id === p2.id)!.label_ids).toEqual([l.id]); }); - it('COLLECTIONS-SVC-054: deleteLabel removes it and cascades its place assignments', () => { + it('COLLECTIONS-SVC-054: deleteLabel removes it and cascades its place assignments', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'DE' }); const l = svc.createLabel(u.id, col.id, 'Berlin'); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Gate' }).place!; - svc.updatePlace(u.id, p.id, { label_ids: [l.id] }); + await svc.updatePlace(u.id, p.id, { label_ids: [l.id] }); svc.deleteLabel(u.id, l.id); expect(svc.getCollection(u.id, col.id).collection.labels).toHaveLength(0); expect(svc.getCollection(u.id, col.id).places.find(x => x.id === p.id)!.label_ids).toEqual([]); }); - it('COLLECTIONS-SVC-055: moving a place to another list drops its labels', () => { + it('COLLECTIONS-SVC-055: moving a place to another list drops its labels', async () => { const u = createUser(testDb).user; const a = svc.createCollection(u.id, { name: 'A' }); const b = svc.createCollection(u.id, { name: 'B' }); const l = svc.createLabel(u.id, a.id, 'Berlin'); const p = svc.savePlace(u.id, { collection_id: a.id, name: 'Gate' }).place!; - svc.updatePlace(u.id, p.id, { label_ids: [l.id] }); + await svc.updatePlace(u.id, p.id, { label_ids: [l.id] }); - svc.updatePlace(u.id, p.id, { collection_id: b.id }); + await svc.updatePlace(u.id, p.id, { collection_id: b.id }); expect(svc.getCollection(u.id, b.id).places.find(x => x.id === p.id)!.label_ids).toEqual([]); }); }); @@ -610,41 +610,40 @@ describe('collection labels', () => { describe('custom saved-place image', () => { function writePlaceImage(name: string): string { - fs.mkdirSync(PLACE_IMAGES_DIR, { recursive: true }); - const filePath = path.join(PLACE_IMAGES_DIR, name); + const filePath = path.join(storageFx.root, name); fs.writeFileSync(filePath, 'jpeg-bytes'); return filePath; } - it('COLLECTIONS-SVC-060: updatePlace sets image_url', () => { + it('COLLECTIONS-SVC-060: updatePlace sets image_url', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Photos' }); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Pic' }).place!; - const updated = svc.updatePlace(u.id, p.id, { image_url: '/uploads/places/col-set.jpg' }); + const updated = await svc.updatePlace(u.id, p.id, { image_url: '/uploads/places/col-set.jpg' }); expect(updated.image_url).toBe('/uploads/places/col-set.jpg'); }); - it('COLLECTIONS-SVC-061: setPlaceImage stores the url and reclaims a replaced upload', () => { + it('COLLECTIONS-SVC-061: setPlaceImage stores the url and reclaims a replaced upload', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Photos' }); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Pic' }).place!; const fileA = writePlaceImage('col-replace-a.jpg'); - svc.setPlaceImage(u.id, p.id, '/uploads/places/col-replace-a.jpg'); + await svc.setPlaceImage(u.id, p.id, '/uploads/places/col-replace-a.jpg'); expect(fs.existsSync(fileA)).toBe(true); - const res = svc.setPlaceImage(u.id, p.id, '/uploads/places/col-replace-b.jpg'); + const res = await svc.setPlaceImage(u.id, p.id, '/uploads/places/col-replace-b.jpg'); expect(res.image_url).toBe('/uploads/places/col-replace-b.jpg'); expect(fs.existsSync(fileA)).toBe(false); }); - it('COLLECTIONS-SVC-062: deletePlace reclaims the uploaded image when unreferenced', () => { + it('COLLECTIONS-SVC-062: deletePlace reclaims the uploaded image when unreferenced', async () => { const u = createUser(testDb).user; const col = svc.createCollection(u.id, { name: 'Photos' }); const p = svc.savePlace(u.id, { collection_id: col.id, name: 'Pic', image_url: '/uploads/places/col-delete.jpg' }).place!; const fileA = writePlaceImage('col-delete.jpg'); expect(fs.existsSync(fileA)).toBe(true); - svc.deletePlace(u.id, p.id); + await svc.deletePlace(u.id, p.id); expect(fs.existsSync(fileA)).toBe(false); }); }); @@ -820,7 +819,7 @@ describe('membership lookups', () => { // ── Post-fold quirk fixes (the trailing fix(server) commit) ───────────────── describe('atomic bulk writes (post-fold quirk fixes)', () => { - it('COLLECTIONS-SVC-090: deletePlacesMany is all-or-nothing — a mid-list 403 deletes nothing', () => { + it('COLLECTIONS-SVC-090: deletePlacesMany is all-or-nothing — a mid-list 403 deletes nothing', async () => { const u = createUser(testDb).user; const otherOwner = createUser(testDb).user; const mine = svc.createCollection(u.id, { name: 'Mine' }); @@ -832,7 +831,7 @@ describe('atomic bulk writes (post-fold quirk fixes)', () => { // The relocated legacy interleaved checks with deletes, so p1 was gone by // the time p2's 403 fired. Now every id is checked first: nothing deleted. - expect(() => svc.deletePlacesMany(u.id, [p1.id, p2.id])).toThrow('Only an admin can delete places from this list'); + await expect(svc.deletePlacesMany(u.id, [p1.id, p2.id])).rejects.toThrow('Only an admin can delete places from this list'); expect(testDb.prepare('SELECT COUNT(*) n FROM collection_places WHERE id = ?').get(p1.id)).toEqual({ n: 1 }); expect(testDb.prepare('SELECT COUNT(*) n FROM collection_places WHERE id = ?').get(p2.id)).toEqual({ n: 1 }); }); @@ -925,7 +924,7 @@ describe('bulk status', () => { expect(statuses.map(s => s.status)).toEqual(['visited', 'visited', 'idea']); }); - it('COLLECTIONS-SVC-096: a place renamed in the list is still found by its source link', () => { + it('COLLECTIONS-SVC-096: a place renamed in the list is still found by its source link', async () => { const u = createUser(testDb).user; createCategory(testDb); const trip = createTrip(testDb, u.id); @@ -933,7 +932,7 @@ describe('bulk status', () => { const col = svc.createCollection(u.id, { name: 'Rome' }); const saved = svc.saveFromTripPlace(u.id, col.id, trip.id, place.id).place!; // Renamed on both sides, and moved far enough that coordinates cannot match. - svc.updatePlace(u.id, saved.id, { name: 'Dinner spot', lat: 45, lng: 9 }); + await svc.updatePlace(u.id, saved.id, { name: 'Dinner spot', lat: 45, lng: 9 }); testDb.prepare('UPDATE places SET name = ? WHERE id = ?').run('Enzo', place.id); expect(svc.setStatusFromTrip(u.id, trip.id, [place.id], 'visited')).toEqual({ updated: 1, places: 1 }); diff --git a/server/tests/unit/nest/exception-filter.test.ts b/server/tests/unit/nest/exception-filter.test.ts index 75c0fe9b5..7e8c503a6 100644 --- a/server/tests/unit/nest/exception-filter.test.ts +++ b/server/tests/unit/nest/exception-filter.test.ts @@ -3,8 +3,8 @@ import { HttpException } from '@nestjs/common'; import { MulterError } from 'multer'; import { TrekExceptionFilter } from '../../../src/nest/common/trek-exception.filter'; -function mockHost() { - const res = { status: vi.fn().mockReturnThis(), json: vi.fn().mockReturnThis() }; +function mockHost(extra: Record = {}) { + const res = { status: vi.fn().mockReturnThis(), json: vi.fn().mockReturnThis(), destroy: vi.fn(), headersSent: false, ...extra }; const host = { switchToHttp: () => ({ getResponse: () => res }) } as never; return { res, host }; } @@ -12,6 +12,30 @@ function mockHost() { describe('TrekExceptionFilter', () => { const filter = new TrekExceptionFilter(); + // A storage stream (or any other in-flight write) can reject after + // sendToResponse()/pipeline() already flushed headers — e.g. a client + // abort mid-download. Writing a JSON envelope there would throw + // ERR_HTTP_HEADERS_SENT or corrupt a response the client is mid-read on. + it('destroys the socket and writes nothing when headers are already sent — but still logs (the only place a post-header failure is recorded)', () => { + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); + const { res, host } = mockHost({ headersSent: true }); + const exception = new Error('mid-stream failure'); + filter.catch(exception, host); + expect(res.destroy).toHaveBeenCalledTimes(1); + expect(res.status).not.toHaveBeenCalled(); + expect(res.json).not.toHaveBeenCalled(); + expect(consoleError).toHaveBeenCalledWith('Unhandled error after headers sent:', exception); + consoleError.mockRestore(); + }); + + it('still writes the normal error envelope when headers were not sent', () => { + const { res, host } = mockHost({ headersSent: false }); + filter.catch(new HttpException('Bad thing', 400), host); + expect(res.destroy).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith({ error: 'Bad thing' }); + }); + it('passes through { error, code } bodies (auth guards) unchanged', () => { const { res, host } = mockHost(); filter.catch(new HttpException({ error: 'Access token required', code: 'AUTH_REQUIRED' }, 401), host); diff --git a/server/tests/unit/nest/files.controller.test.ts b/server/tests/unit/nest/files.controller.test.ts index da0a1abb4..426d8418c 100644 --- a/server/tests/unit/nest/files.controller.test.ts +++ b/server/tests/unit/nest/files.controller.test.ts @@ -2,9 +2,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; import { HttpException } from '@nestjs/common'; import type { Request, Response } from 'express'; -import os from 'os'; -import path from 'path'; -import fs from 'fs'; vi.mock('../../../src/nest/common/demo', () => ({ isDemoEmail: vi.fn(() => false) })); @@ -13,6 +10,7 @@ import { FilesDownloadController } from '../../../src/nest/files/files-download. import { PhotosController } from '../../../src/nest/photos/photos.controller'; import type { FilesService } from '../../../src/nest/files/files.service'; import type { PhotosService } from '../../../src/nest/photos/photos.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import { isDemoEmail } from '../../../src/nest/common/demo'; import type { User } from '../../../src/types'; @@ -28,6 +26,13 @@ function fsvc(o: Partial = {}): FilesService { } as unknown as FilesService; } +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as StorageService; + +/** Construct the controller with the standard env + storage stubs. */ +function fc(svc: FilesService): FilesController { + return new FilesController(svc, new RuntimeEnvService(), storageStub); +} + function thrown(fn: () => unknown): { status: number; body: unknown } { try { fn(); } catch (err) { expect(err).toBeInstanceOf(HttpException); @@ -37,6 +42,15 @@ function thrown(fn: () => unknown): { status: number; body: unknown } { throw new Error('expected throw'); } +async function rejected(p: Promise): Promise<{ status: number; body: unknown }> { + try { await p; } catch (err) { + expect(err).toBeInstanceOf(HttpException); + const e = err as HttpException; + return { status: e.getStatus(), body: e.getResponse() }; + } + throw new Error('expected reject'); +} + beforeEach(() => vi.clearAllMocks()); afterEach(() => { delete process.env.DEMO_MODE; }); @@ -47,121 +61,128 @@ const trip = { id: 5, user_id: 42 } as never; describe('FilesController (parity with the legacy /api/trips/:tripId/files route)', () => { it('GET / lists with the trash flag', () => { const listFiles = vi.fn().mockReturnValue([{ id: 1 }]); - expect(new FilesController(fsvc({ listFiles } as Partial), new RuntimeEnvService()).list(user, trip, '5', 'true')).toEqual({ files: [{ id: 1 }] }); + expect(fc(fsvc({ listFiles } as Partial)).list(user, trip, '5', 'true')).toEqual({ files: [{ id: 1 }] }); expect(listFiles).toHaveBeenCalledWith('5', true); }); describe('POST / (upload)', () => { - it('404 without trip access — upload keeps its own check so the body is read first', () => { + it('404 without trip access — upload keeps its own check so the body is read first', async () => { // The guard would answer before multer, and a response sent mid-upload destroys // the socket: the caller would see ECONNRESET rather than this 404. const svc = fsvc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }); - expect(thrown(() => new FilesController(svc, new RuntimeEnvService()).upload(user, '5', undefined, {}))).toEqual({ + expect(await rejected(fc(svc).upload(user, '5', undefined, {}))).toEqual({ status: 404, body: { error: 'Trip not found' }, }); }); const file = { filename: 'a.pdf' } as Express.Multer.File; - it('403 in demo mode for a demo email', () => { + it('403 in demo mode for a demo email', async () => { process.env.DEMO_MODE = 'true'; vi.mocked(isDemoEmail).mockReturnValue(true); - expect(thrown(() => new FilesController(fsvc(), new RuntimeEnvService()).upload(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' } }); + expect(await rejected(fc(fsvc()).upload(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' } }); }); - it('403 without file_upload, 400 without a file, else creates + broadcasts', () => { - expect(thrown(() => new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).upload(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); - expect(thrown(() => new FilesController(fsvc(), new RuntimeEnvService()).upload(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); + it('403 without file_upload, 400 without a file, else commits + creates + broadcasts', async () => { + expect(await rejected(fc(fsvc({ can: vi.fn().mockReturnValue(false) })).upload(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); + expect(await rejected(fc(fsvc()).upload(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); const createFile = vi.fn().mockReturnValue({ id: 9 }); const broadcast = vi.fn(); const s = fsvc({ createFile, broadcast } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).upload(user, '5', file, { description: 'd' }, 'sock')).toEqual({ file: { id: 9 } }); + expect(await fc(s).upload(user, '5', file, { description: 'd' }, 'sock')).toEqual({ file: { id: 9 } }); + expect(storageStub.put).toHaveBeenCalledWith('files', 'a.pdf', { tmpPath: undefined }); expect(createFile).toHaveBeenCalledWith('5', file, 1, { place_id: undefined, description: 'd', reservation_id: undefined }); expect(broadcast).toHaveBeenCalledWith('5', 'file:created', { file: { id: 9 } }, 'sock'); }); - it('caps non-video by extension and accepts large video; cleans up on rejection (#823)', () => { + it('caps non-video by extension and accepts large video; cleans up on rejection (#823)', async () => { const big = { filename: 'big.pdf', originalname: 'big.pdf', size: 60 * 1024 * 1024, path: '' } as Express.Multer.File; - expect(thrown(() => new FilesController(fsvc(), new RuntimeEnvService()).upload(user, '5', big, {}))).toEqual({ status: 400, body: { error: 'File is too large' } }); + expect(await rejected(fc(fsvc()).upload(user, '5', big, {}))).toEqual({ status: 400, body: { error: 'File is too large' } }); const createFile = vi.fn().mockReturnValue({ id: 9 }); const vid = { filename: 'v.mp4', originalname: 'clip.mp4', size: 200 * 1024 * 1024, path: '' } as Express.Multer.File; - expect(new FilesController(fsvc({ createFile, broadcast: vi.fn() } as Partial), new RuntimeEnvService()).upload(user, '5', vid, {})).toEqual({ file: { id: 9 } }); + expect(await fc(fsvc({ createFile, broadcast: vi.fn() } as Partial)).upload(user, '5', vid, {})).toEqual({ file: { id: 9 } }); // A rejected upload with a real path triggers the unlink cleanup branch // (the file doesn't exist, so the inner best-effort catch swallows it). const withPath = { filename: 'a.pdf', path: '/nonexistent/zzz.pdf' } as Express.Multer.File; - expect(thrown(() => new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).upload(user, '5', withPath, {}))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); + expect(await rejected(fc(fsvc({ can: vi.fn().mockReturnValue(false) })).upload(user, '5', withPath, {}))).toEqual({ status: 403, body: { error: 'No permission to upload files' } }); + }); + + it('500 with cleanup when the storage commit fails', async () => { + vi.mocked(storageStub.put).mockRejectedValueOnce(new Error('disk full')); + const withPath = { filename: 'a.pdf', path: '/nonexistent/zzz.pdf' } as Express.Multer.File; + await expect(fc(fsvc()).upload(user, '5', withPath, {})).rejects.toThrow('disk full'); }); }); it('PUT /:id 403 without file_edit, 404 unknown, else updates + broadcasts', () => { - expect(thrown(() => new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).update(user, trip, '5', '9', {}))).toEqual({ status: 403, body: { error: 'No permission to edit files' } }); - expect(thrown(() => new FilesController(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).update(user, trip, '5', '9', {}))).toEqual({ status: 404, body: { error: 'File not found' } }); + expect(thrown(() => fc(fsvc({ can: vi.fn().mockReturnValue(false) })).update(user, trip, '5', '9', {}))).toEqual({ status: 403, body: { error: 'No permission to edit files' } }); + expect(thrown(() => fc(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial)).update(user, trip, '5', '9', {}))).toEqual({ status: 404, body: { error: 'File not found' } }); const updateFile = vi.fn().mockReturnValue({ id: 9 }); const s = fsvc({ getFileById: vi.fn().mockReturnValue({ id: 9, description: 'x' }), updateFile, broadcast: vi.fn() } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).update(user, trip, '5', '9', { description: 'new' })).toEqual({ file: { id: 9 } }); + expect(fc(s).update(user, trip, '5', '9', { description: 'new' })).toEqual({ file: { id: 9 } }); }); it('PATCH /:id/star 403/404, else toggles', () => { - expect(thrown(() => new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).star(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission' } }); - expect(thrown(() => new FilesController(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).star(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); + expect(thrown(() => fc(fsvc({ can: vi.fn().mockReturnValue(false) })).star(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission' } }); + expect(thrown(() => fc(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial)).star(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); const toggleStarred = vi.fn().mockReturnValue({ id: 9, starred: 1 }); const s = fsvc({ getFileById: vi.fn().mockReturnValue({ id: 9, starred: 0 }), toggleStarred, broadcast: vi.fn() } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).star(user, trip, '5', '9')).toEqual({ file: { id: 9, starred: 1 } }); + expect(fc(s).star(user, trip, '5', '9')).toEqual({ file: { id: 9, starred: 1 } }); expect(toggleStarred).toHaveBeenCalledWith('9', 0); }); it('DELETE /:id soft-delete 403/404, else success', () => { - expect(thrown(() => new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).remove(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission to delete files' } }); - expect(thrown(() => new FilesController(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).remove(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); + expect(thrown(() => fc(fsvc({ can: vi.fn().mockReturnValue(false) })).remove(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission to delete files' } }); + expect(thrown(() => fc(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial)).remove(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); const softDeleteFile = vi.fn(); const broadcast = vi.fn(); const s = fsvc({ getFileById: vi.fn().mockReturnValue({ id: 9 }), softDeleteFile, broadcast } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).remove(user, trip, '5', '9', 'sock')).toEqual({ success: true }); + expect(fc(s).remove(user, trip, '5', '9', 'sock')).toEqual({ success: true }); expect(broadcast).toHaveBeenCalledWith('5', 'file:deleted', { fileId: 9 }, 'sock'); }); it('POST /:id/restore 404 not in trash, else restores', () => { - expect(thrown(() => new FilesController(fsvc({ getDeletedFile: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).restore(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found in trash' } }); + expect(thrown(() => fc(fsvc({ getDeletedFile: vi.fn().mockReturnValue(undefined) } as Partial)).restore(user, trip, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found in trash' } }); const restoreFile = vi.fn().mockReturnValue({ id: 9 }); const s = fsvc({ getDeletedFile: vi.fn().mockReturnValue({ id: 9 }), restoreFile, broadcast: vi.fn() } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).restore(user, trip, '5', '9')).toEqual({ file: { id: 9 } }); + expect(fc(s).restore(user, trip, '5', '9')).toEqual({ file: { id: 9 } }); }); it('DELETE /:id/permanent 404 not in trash, else deletes', async () => { - await expect(new FilesController(fsvc({ getDeletedFile: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).permanent(user, trip, '5', '9')).rejects.toBeInstanceOf(HttpException); + await expect(fc(fsvc({ getDeletedFile: vi.fn().mockReturnValue(undefined) } as Partial)).permanent(user, trip, '5', '9')).rejects.toBeInstanceOf(HttpException); const permanentDeleteFile = vi.fn().mockResolvedValue(undefined); const s = fsvc({ getDeletedFile: vi.fn().mockReturnValue({ id: 9 }), permanentDeleteFile, broadcast: vi.fn() } as Partial); - expect(await new FilesController(s, new RuntimeEnvService()).permanent(user, trip, '5', '9')).toEqual({ success: true }); + expect(await fc(s).permanent(user, trip, '5', '9')).toEqual({ success: true }); }); it('DELETE /trash/empty 403, else returns the count', async () => { - await expect(new FilesController(fsvc({ can: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).emptyTrash(user, trip, '5')).rejects.toBeInstanceOf(HttpException); + await expect(fc(fsvc({ can: vi.fn().mockReturnValue(false) })).emptyTrash(user, trip, '5')).rejects.toBeInstanceOf(HttpException); const s = fsvc({ emptyTrash: vi.fn().mockResolvedValue(3) } as Partial); - expect(await new FilesController(s, new RuntimeEnvService()).emptyTrash(user, trip, '5')).toEqual({ success: true, deleted: 3 }); + expect(await fc(s).emptyTrash(user, trip, '5')).toEqual({ success: true, deleted: 3 }); }); it('POST /:id/link 404 unknown file, else links', () => { - expect(thrown(() => new FilesController(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).link(user, trip, '5', '9', {}))).toEqual({ status: 404, body: { error: 'File not found' } }); + expect(thrown(() => fc(fsvc({ getFileById: vi.fn().mockReturnValue(undefined) } as Partial)).link(user, trip, '5', '9', {}))).toEqual({ status: 404, body: { error: 'File not found' } }); const createFileLink = vi.fn().mockReturnValue([{ id: 1 }]); const s = fsvc({ getFileById: vi.fn().mockReturnValue({ id: 9 }), createFileLink } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).link(user, trip, '5', '9', { reservation_id: 2 })).toEqual({ success: true, links: [{ id: 1 }] }); + expect(fc(s).link(user, trip, '5', '9', { reservation_id: 2 })).toEqual({ success: true, links: [{ id: 1 }] }); }); it('DELETE /:id/link/:linkId removes the link; GET /:id/links lists', () => { const deleteFileLink = vi.fn(); - expect(new FilesController(fsvc({ deleteFileLink } as Partial), new RuntimeEnvService()).unlink(user, trip, '5', '9', '3')).toEqual({ success: true }); + expect(fc(fsvc({ deleteFileLink } as Partial)).unlink(user, trip, '5', '9', '3')).toEqual({ success: true }); expect(deleteFileLink).toHaveBeenCalledWith('3', '9'); const s = fsvc({ getFileLinks: vi.fn().mockReturnValue([{ id: 1 }]) } as Partial); - expect(new FilesController(s, new RuntimeEnvService()).links(user, trip, '5', '9')).toEqual({ links: [{ id: 1 }] }); + expect(fc(s).links(user, trip, '5', '9')).toEqual({ links: [{ id: 1 }] }); }); it('the trash + link routes all reject without file_delete / file_edit', async () => { const denied = () => fsvc({ can: vi.fn().mockReturnValue(false) }); - await expect(new FilesController(denied(), new RuntimeEnvService()).permanent(user, trip, '5', '9')).rejects.toMatchObject({ status: 403 }); - expect(thrown(() => new FilesController(denied(), new RuntimeEnvService()).restore(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission' } }); - expect(thrown(() => new FilesController(denied(), new RuntimeEnvService()).link(user, trip, '5', '9', {}))).toEqual({ status: 403, body: { error: 'No permission' } }); - expect(thrown(() => new FilesController(denied(), new RuntimeEnvService()).unlink(user, trip, '5', '9', '3'))).toEqual({ status: 403, body: { error: 'No permission' } }); + await expect(fc(denied()).permanent(user, trip, '5', '9')).rejects.toMatchObject({ status: 403 }); + expect(thrown(() => fc(denied()).restore(user, trip, '5', '9'))).toEqual({ status: 403, body: { error: 'No permission' } }); + expect(thrown(() => fc(denied()).link(user, trip, '5', '9', {}))).toEqual({ status: 403, body: { error: 'No permission' } }); + expect(thrown(() => fc(denied()).unlink(user, trip, '5', '9', '3'))).toEqual({ status: 403, body: { error: 'No permission' } }); }); }); @@ -172,95 +193,81 @@ describe('FilesDownloadController', () => { authenticateDownload: vi.fn().mockReturnValue({ userId: 1 }), verifyTripAccess: vi.fn().mockReturnValue({ user_id: 1 }), getFileById: vi.fn().mockReturnValue({ filename: 'x.pdf', original_name: 'x.pdf' }), - resolveFilePath: vi.fn().mockReturnValue({ resolved: 'C:/nope/x.pdf', safe: true }), ...o, } as unknown as FilesService; } + function dstor(o: Record = {}): StorageService { + return { + exists: vi.fn().mockResolvedValue(true), + sendToResponse: vi.fn().mockResolvedValue(undefined), + ...o, + } as unknown as StorageService; + } const req = { headers: {}, query: {} } as Request; - const res = { setHeader: vi.fn(), sendFile: vi.fn() } as unknown as Response; + const res = {} as Response; - it('maps the auth error from authenticateDownload', () => { + it('maps the auth error from authenticateDownload', async () => { const s = dsvc({ authenticateDownload: vi.fn().mockReturnValue({ error: 'Authentication required', status: 401 }) }); - expect(thrown(() => new FilesDownloadController(s).download(req, res, '5', '9'))).toEqual({ status: 401, body: { error: 'Authentication required' } }); - }); - it('404 without trip access, 404 unknown file, 403 on an unsafe path', () => { - expect(thrown(() => new FilesDownloadController(dsvc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) })).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'Trip not found' } }); - expect(thrown(() => new FilesDownloadController(dsvc({ getFileById: vi.fn().mockReturnValue(undefined) })).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); - expect(thrown(() => new FilesDownloadController(dsvc({ resolveFilePath: vi.fn().mockReturnValue({ resolved: '/x', safe: false }) })).download(req, res, '5', '9'))).toEqual({ status: 403, body: { error: 'Forbidden' } }); + expect(await rejected(new FilesDownloadController(s, dstor()).download(req, res, '5', '9'))).toEqual({ status: 401, body: { error: 'Authentication required' } }); }); - it('404 when the safe path is gone from disk', () => { - const missing = path.join(os.tmpdir(), `trek-no-such-${Date.now()}.pdf`); - const s = dsvc({ resolveFilePath: vi.fn().mockReturnValue({ resolved: missing, safe: true }) }); - expect(thrown(() => new FilesDownloadController(s).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); + it('404 without trip access and 404 for an unknown file', async () => { + expect(await rejected(new FilesDownloadController(dsvc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }), dstor()).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'Trip not found' } }); + expect(await rejected(new FilesDownloadController(dsvc({ getFileById: vi.fn().mockReturnValue(undefined) }), dstor()).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); }); - it('streams a regular file via sendFile with an explicit root', () => { - const real = path.join(os.tmpdir(), `trek-dl-${Date.now()}.pdf`); - fs.writeFileSync(real, 'x'); - try { - const sendFile = vi.fn(); - const localRes = { setHeader: vi.fn(), sendFile } as unknown as Response; - const s = dsvc({ resolveFilePath: vi.fn().mockReturnValue({ resolved: real, safe: true }) }); - new FilesDownloadController(s).download(req, localRes, '5', '9'); - expect(sendFile).toHaveBeenCalledWith(path.basename(real), { root: path.dirname(real) }); - expect(localRes.setHeader).not.toHaveBeenCalled(); - } finally { - fs.unlinkSync(real); - } + it('404 when the object is gone from storage', async () => { + const storage = dstor({ exists: vi.fn().mockResolvedValue(false) }); + expect(await rejected(new FilesDownloadController(dsvc(), storage).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); }); - it('serves a .pkpass inline with the Wallet MIME type and the original name', () => { - const real = path.join(os.tmpdir(), `trek-pass-${Date.now()}.pkpass`); - fs.writeFileSync(real, 'x'); - try { - const setHeader = vi.fn(); - const localRes = { setHeader, sendFile: vi.fn() } as unknown as Response; - const s = dsvc({ - getFileById: vi.fn().mockReturnValue({ filename: 'pass.pkpass', original_name: 'BoardingPass.pkpass' }), - resolveFilePath: vi.fn().mockReturnValue({ resolved: real, safe: true }), - }); - new FilesDownloadController(s).download(req, localRes, '5', '9'); - expect(setHeader).toHaveBeenCalledWith('Content-Type', 'application/vnd.apple.pkpass'); - expect(setHeader).toHaveBeenCalledWith('Content-Disposition', 'inline; filename="BoardingPass.pkpass"'); - } finally { - fs.unlinkSync(real); - } + it('404 when the stored name is an invalid key (exists rejects)', async () => { + const storage = dstor({ exists: vi.fn().mockRejectedValue(new Error('invalid storage key')) }); + expect(await rejected(new FilesDownloadController(dsvc(), storage).download(req, res, '5', '9'))).toEqual({ status: 404, body: { error: 'File not found' } }); }); - it('serves a .pkpasses bundle inline with its own Wallet MIME type', () => { - const real = path.join(os.tmpdir(), `trek-pass-${Date.now()}.pkpasses`); - fs.writeFileSync(real, 'x'); - try { - const setHeader = vi.fn(); - const localRes = { setHeader, sendFile: vi.fn() } as unknown as Response; - const s = dsvc({ - getFileById: vi.fn().mockReturnValue({ filename: 'passes.pkpasses', original_name: 'BoardingPasses.pkpasses' }), - resolveFilePath: vi.fn().mockReturnValue({ resolved: real, safe: true }), - }); - new FilesDownloadController(s).download(req, localRes, '5', '9'); - expect(setHeader).toHaveBeenCalledWith('Content-Type', 'application/vnd.apple.pkpasses'); - expect(setHeader).toHaveBeenCalledWith('Content-Disposition', 'inline; filename="BoardingPasses.pkpasses"'); - } finally { - fs.unlinkSync(real); - } + it('streams a regular file through storage with no wallet headers', async () => { + const storage = dstor(); + await new FilesDownloadController(dsvc(), storage).download(req, res, '5', '9'); + expect(storage.exists).toHaveBeenCalledWith('files', 'x.pdf'); + expect(storage.sendToResponse).toHaveBeenCalledWith('files', 'x.pdf', res, undefined); }); - it('falls back to the resolved basename when a .pkpass has no original name', () => { - const real = path.join(os.tmpdir(), `trek-pass-${Date.now()}.pkpass`); - fs.writeFileSync(real, 'x'); - try { - const setHeader = vi.fn(); - const localRes = { setHeader, sendFile: vi.fn() } as unknown as Response; - const s = dsvc({ - getFileById: vi.fn().mockReturnValue({ filename: 'pass.pkpass', original_name: null }), - resolveFilePath: vi.fn().mockReturnValue({ resolved: real, safe: true }), - }); - new FilesDownloadController(s).download(req, localRes, '5', '9'); - expect(setHeader).toHaveBeenCalledWith('Content-Disposition', `inline; filename="${path.basename(real)}"`); - } finally { - fs.unlinkSync(real); - } + it('tolerates a stray prefixed row via basename(), same as the download controller', async () => { + const storage = dstor(); + const s = dsvc({ getFileById: vi.fn().mockReturnValue({ filename: 'files/x.pdf', original_name: 'x.pdf' }) }); + await new FilesDownloadController(s, storage).download(req, res, '5', '9'); + expect(storage.sendToResponse).toHaveBeenCalledWith('files', 'x.pdf', res, undefined); + }); + + it('serves a .pkpass inline with the Wallet MIME type and the original name', async () => { + const storage = dstor(); + const s = dsvc({ getFileById: vi.fn().mockReturnValue({ filename: 'pass.pkpass', original_name: 'BoardingPass.pkpass' }) }); + await new FilesDownloadController(s, storage).download(req, res, '5', '9'); + expect(storage.sendToResponse).toHaveBeenCalledWith('files', 'pass.pkpass', res, { + contentType: 'application/vnd.apple.pkpass', + disposition: 'inline; filename="BoardingPass.pkpass"', + }); + }); + + it('serves a .pkpasses bundle inline with its own Wallet MIME type', async () => { + const storage = dstor(); + const s = dsvc({ getFileById: vi.fn().mockReturnValue({ filename: 'passes.pkpasses', original_name: 'BoardingPasses.pkpasses' }) }); + await new FilesDownloadController(s, storage).download(req, res, '5', '9'); + expect(storage.sendToResponse).toHaveBeenCalledWith('files', 'passes.pkpasses', res, { + contentType: 'application/vnd.apple.pkpasses', + disposition: 'inline; filename="BoardingPasses.pkpasses"', + }); + }); + + it('falls back to the stored basename when a .pkpass has no original name', async () => { + const storage = dstor(); + const s = dsvc({ getFileById: vi.fn().mockReturnValue({ filename: 'pass.pkpass', original_name: null }) }); + await new FilesDownloadController(s, storage).download(req, res, '5', '9'); + expect(storage.sendToResponse).toHaveBeenCalledWith('files', 'pass.pkpass', res, { + contentType: 'application/vnd.apple.pkpass', + disposition: 'inline; filename="pass.pkpass"', + }); }); }); diff --git a/server/tests/unit/nest/files.service.test.ts b/server/tests/unit/nest/files.service.test.ts index 184d3dd7c..69fdeb9b1 100644 --- a/server/tests/unit/nest/files.service.test.ts +++ b/server/tests/unit/nest/files.service.test.ts @@ -8,7 +8,6 @@ */ import { describe, it, expect, vi, beforeAll, beforeEach, afterAll, afterEach } from 'vitest'; import path from 'path'; -import fs from 'fs'; // ── DB setup ────────────────────────────────────────────────────────────────── @@ -76,7 +75,9 @@ import type { TripFile, User } from '../../../src/types'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; -const svc = new FilesService(new DatabaseService(testDb), permissionsStub, new RealtimeService(), new EphemeralTokenService()); +const storageDelete = vi.fn(); +const storageStub = { delete: storageDelete } as unknown as import('../../../src/nest/storage/storage.service').StorageService; +const svc = new FilesService(new DatabaseService(testDb), permissionsStub, new RealtimeService(), new EphemeralTokenService(), storageStub); beforeAll(() => { createTables(testDb); @@ -153,39 +154,30 @@ describe('files.constants', () => { const serverRoot = path.resolve(__dirname, '../../..'); expect(path.resolve(filesDir)).toBe(path.join(serverRoot, 'uploads', 'files')); }); - - it('FILE-SVC-005: resolveFilePath neutralizes traversal via basename and stays inside filesDir', () => { - const plain = svc.resolveFilePath('photo.jpg'); - expect(plain.resolved).toBe(path.resolve(path.join(filesDir, 'photo.jpg'))); - expect(plain.safe).toBe(true); - - const sneaky = svc.resolveFilePath('../../../etc/passwd'); - expect(sneaky.resolved).toBe(path.resolve(path.join(filesDir, 'passwd'))); - expect(sneaky.safe).toBe(true); - expect(sneaky.resolved.startsWith(path.resolve(filesDir))).toBe(true); - }); }); -// ── getAllowedExtensions ────────────────────────────────────────────────────── +// ── allowed extensions (canonical owner: AllowedFileTypesService) ───────────── +// FilesService.getAllowedExtensions was deleted in the storage slice-2 +// consolidation (it had no callers); the same four behaviors are pinned +// against the single remaining query owner. -describe('getAllowedExtensions', () => { +describe('AllowedFileTypesService.get', () => { it('FILE-SVC-006: returns the app_settings value when set', () => { setAppSetting(testDb, 'allowed_file_types', 'pdf,txt'); - expect(svc.getAllowedExtensions()).toBe('pdf,txt'); + expect(new AllowedFileTypesService(new DatabaseService(testDb)).get()).toBe('pdf,txt'); }); it('FILE-SVC-007: returns the default when the row is absent', () => { - expect(svc.getAllowedExtensions()).toBe(DEFAULT_ALLOWED_EXTENSIONS); + expect(new AllowedFileTypesService(new DatabaseService(testDb)).get()).toBe(DEFAULT_ALLOWED_EXTENSIONS); }); it('FILE-SVC-008: returns the default for an empty value (|| coercion, not ??)', () => { setAppSetting(testDb, 'allowed_file_types', ''); - expect(svc.getAllowedExtensions()).toBe(DEFAULT_ALLOWED_EXTENSIONS); + expect(new AllowedFileTypesService(new DatabaseService(testDb)).get()).toBe(DEFAULT_ALLOWED_EXTENSIONS); }); it('FILE-SVC-009: returns the default when the query throws (no app_settings table)', () => { - const bareSvc = new FilesService(new DatabaseService(bareDb), permissionsStub, new RealtimeService(), new EphemeralTokenService()); - expect(bareSvc.getAllowedExtensions()).toBe(DEFAULT_ALLOWED_EXTENSIONS); + expect(new AllowedFileTypesService(new DatabaseService(bareDb)).get()).toBe(DEFAULT_ALLOWED_EXTENSIONS); }); }); @@ -345,20 +337,20 @@ describe('toggleStarred / softDeleteFile / restoreFile', () => { // ── permanentDeleteFile / emptyTrash (unlink-first semantics) ───────────────── describe('permanentDeleteFile', () => { - it('FILE-SVC-022: removes the resolved path with force:true, then the DB row', async () => { + it('FILE-SVC-022: deletes the storage object (idempotent on missing), then the DB row', async () => { const { user, trip } = seedTrip(); const file = makeFile(trip.id, user.id, { filename: 'on-disk.pdf' }); - const rm = vi.spyOn(fs.promises, 'rm').mockResolvedValue(undefined); + storageDelete.mockResolvedValue(undefined); await svc.permanentDeleteFile(svc.getFileById(file.id, trip.id) as TripFile); - expect(rm).toHaveBeenCalledWith(svc.resolveFilePath('on-disk.pdf').resolved, { force: true }); + expect(storageDelete).toHaveBeenCalledWith('files', 'on-disk.pdf'); expect(svc.getFileById(file.id, trip.id)).toBeUndefined(); }); - it('FILE-SVC-023: an unlink failure logs [files], rethrows and keeps the DB row', async () => { + it('FILE-SVC-023: a delete failure logs [files], rethrows and keeps the DB row', async () => { const { user, trip } = seedTrip(); const file = makeFile(trip.id, user.id, { filename: 'stuck.pdf' }); const boom = new Error('EACCES'); - vi.spyOn(fs.promises, 'rm').mockRejectedValue(boom); + storageDelete.mockRejectedValue(boom); const err = vi.spyOn(console, 'error').mockImplementation(() => {}); await expect(svc.permanentDeleteFile(svc.getFileById(file.id, trip.id) as TripFile)).rejects.toThrow('EACCES'); expect(err).toHaveBeenCalledWith('[files] unlink failed for stuck.pdf, keeping DB row:', boom); @@ -374,7 +366,7 @@ describe('emptyTrash', () => { const kept = makeFile(trip.id, user.id, { filename: 'kept.pdf' }); svc.softDeleteFile(a.id); svc.softDeleteFile(b.id); - vi.spyOn(fs.promises, 'rm').mockResolvedValue(undefined); + storageDelete.mockResolvedValue(undefined); await expect(svc.emptyTrash(trip.id)).resolves.toBe(2); expect(svc.listFiles(trip.id, true)).toEqual([]); @@ -388,8 +380,8 @@ describe('emptyTrash', () => { svc.softDeleteFile(good.id); svc.softDeleteFile(bad.id); const boom = new Error('EBUSY'); - vi.spyOn(fs.promises, 'rm').mockImplementation((p) => - String(p).includes('bad.pdf') ? Promise.reject(boom) : Promise.resolve() + storageDelete.mockImplementation((_category: string, name: string) => + name.includes('bad.pdf') ? Promise.reject(boom) : Promise.resolve() ); const err = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -399,11 +391,10 @@ describe('emptyTrash', () => { expect(svc.getDeletedFile(bad.id, trip.id)).toBeDefined(); }); - it('FILE-SVC-026: an empty trash resolves to 0 without touching the disk', async () => { + it('FILE-SVC-026: an empty trash resolves to 0 without touching storage', async () => { const { trip } = seedTrip(); - const rm = vi.spyOn(fs.promises, 'rm'); await expect(svc.emptyTrash(trip.id)).resolves.toBe(0); - expect(rm).not.toHaveBeenCalled(); + expect(storageDelete).not.toHaveBeenCalled(); }); }); diff --git a/server/tests/unit/nest/immich.service.test.ts b/server/tests/unit/nest/immich.service.test.ts index 4493c5f39..91d01f04d 100644 --- a/server/tests/unit/nest/immich.service.test.ts +++ b/server/tests/unit/nest/immich.service.test.ts @@ -46,11 +46,15 @@ import { DatabaseService } from '../../../src/nest/database/database.service'; import { ImmichService } from '../../../src/nest/memories/immich.service'; import type { AuditService } from '../../../src/nest/audit/audit.service'; import type { MemoriesAccessService } from '../../../src/nest/memories/memories-access.service'; +import fs from 'node:fs'; +import path from 'node:path'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; const audit = { writeAudit: vi.fn() }; const access = { getAlbumIdFromLink: vi.fn() }; const dbs = new DatabaseService(testDb); -const svc = new ImmichService(dbs, audit as unknown as AuditService, access as unknown as MemoriesAccessService); +const journeyFx = makeStorageFixture('journey/'); +const svc = new ImmichService(dbs, audit as unknown as AuditService, access as unknown as MemoriesAccessService, journeyFx.storage); const USER = 1; @@ -478,3 +482,49 @@ describe('collectAlbumSelection', () => { expect(await svc.collectAlbumSelection('1', 'l1', USER)).toEqual({ error: 'Could not reach Immich', status: 502 }); }); }); + +describe('uploadToImmich', () => { + function writeJourneyObject(name: string, bytes: string): void { + const dir = path.join(journeyFx.root, 'journey'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, name), bytes); + } + + it('IMMICH-UP-001: returns null without credentials and never fetches', async () => { + expect(await svc.uploadToImmich(999, 'journey/pic.jpg', 'pic.jpg')).toBeNull(); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it('IMMICH-UP-002: returns null for a path outside the journey category', async () => { + seedUser(USER, 'https://immich.test', 'key-1'); + expect(await svc.uploadToImmich(USER, 'files/doc.pdf', 'doc.pdf')).toBeNull(); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it('IMMICH-UP-003: returns null when the object is missing (old existsSync guard)', async () => { + seedUser(USER, 'https://immich.test', 'key-1'); + expect(await svc.uploadToImmich(USER, 'journey/gone.jpg', 'gone.jpg')).toBeNull(); + expect(safeFetch).not.toHaveBeenCalled(); + }); + + it('IMMICH-UP-004: posts the bytes as multipart and answers the created asset id', async () => { + seedUser(USER, 'https://immich.test', 'key-1'); + writeJourneyObject('up.jpg', 'jpeg-bytes-here'); + safeFetch.mockResolvedValueOnce({ ok: true, json: async () => ({ id: 'immich-42' }) }); + + expect(await svc.uploadToImmich(USER, 'journey/up.jpg', 'up.jpg')).toBe('immich-42'); + + const [url, init] = safeFetch.mock.calls[0] as [string, { method: string; body: Buffer; headers: Record }]; + expect(url).toBe('https://immich.test/api/assets'); + expect(init.method).toBe('POST'); + expect(init.headers['x-api-key']).toBe('key-1'); + expect(init.body.includes(Buffer.from('jpeg-bytes-here'))).toBe(true); + }); + + it('IMMICH-UP-005: a rejected upstream answers null, not a throw', async () => { + seedUser(USER, 'https://immich.test', 'key-1'); + writeJourneyObject('rej.jpg', 'bytes'); + safeFetch.mockResolvedValueOnce({ ok: false, status: 500 }); + expect(await svc.uploadToImmich(USER, 'journey/rej.jpg', 'rej.jpg')).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/journey.controller.test.ts b/server/tests/unit/nest/journey.controller.test.ts index ba8fa5445..47e120170 100644 --- a/server/tests/unit/nest/journey.controller.test.ts +++ b/server/tests/unit/nest/journey.controller.test.ts @@ -1,14 +1,14 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { HttpException } from '@nestjs/common'; import type { Response } from 'express'; -import path from 'node:path'; -import fs from 'node:fs'; import { JourneyController } from '../../../src/nest/journey/journey.controller'; +import { journeyThumbName } from '../../../src/nest/memories/thumbnail.service'; import type { PhotoCaptureBackfillService } from '../../../src/nest/memories/photo-capture-backfill.service'; import { JourneyPublicController } from '../../../src/nest/journey/journey-public.controller'; import type { JourneyService } from '../../../src/nest/journey/journey.service'; import type { JourneyBookService } from '../../../src/nest/journey/journey-book.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; const user = { id: 1, username: 'u', role: 'user', email: 'u@example.test' } as User; @@ -17,6 +17,14 @@ function svc(o: Partial = {}): JourneyService { return { journeyAddonEnabled: vi.fn().mockReturnValue(true), ...o } as unknown as JourneyService; } +const storageExists = vi.fn(); +const storageSendToResponse = vi.fn(); +const storageStub = { + put: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + exists: storageExists, + sendToResponse: storageSendToResponse, +} as unknown as StorageService; // The capture backfill is detached and irrelevant to every case here — it only has // to exist so the constructor can call it. const captureBackfillStub = { schedule: vi.fn(), run: vi.fn() } as unknown as PhotoCaptureBackfillService; @@ -24,12 +32,15 @@ const captureBackfillStub = { schedule: vi.fn(), run: vi.fn() } as unknown as Ph /** * Build the controller. * - * The Studio book (#1973) is a third injected dependency, and every one of - * these cases would otherwise have to name it. It is stubbed unless a test is - * actually about the book. + * Storage and the Studio book (#1973) are injected dependencies that almost no + * case here is about, and naming all four at every call site is what made the + * previous shape unmaintainable — a fifth dependency would have meant touching + * ninety lines. Both are stubbed: pass a partial book when the case IS about + * the book, and reach for `storageStub`'s own mocks when it is about an object + * being stored or served. */ function ctl(service: JourneyService, books: Partial = {}): JourneyController { - return new JourneyController(service, books as JourneyBookService, captureBackfillStub); + return new JourneyController(service, storageStub, books as JourneyBookService, captureBackfillStub); } function thrown(fn: () => unknown): { status: number; body: unknown } { @@ -89,33 +100,36 @@ describe('JourneyController', () => { expect(thrown(() => ctl(svc({ linkPhotoToEntry: vi.fn().mockReturnValue(null) } as Partial)).linkPhoto(user, '3', { journey_photo_id: 9 }))).toEqual({ status: 403, body: { error: 'Not allowed' } }); }); - it('unlink photo (204) maps 404; delete photo 404 then unlinks file', () => { + it('unlink photo (204) maps 404; delete photo 404 then removes the object', async () => { expect(thrown(() => ctl(svc({ unlinkPhotoFromEntry: vi.fn().mockReturnValue(false) } as Partial)).unlinkPhoto(user, '3', '7'))).toEqual({ status: 404, body: { error: 'Not found or not allowed' } }); expect(ctl(svc({ unlinkPhotoFromEntry: vi.fn().mockReturnValue(true) } as Partial)).unlinkPhoto(user, '3', '7')).toBeUndefined(); - expect(thrown(() => ctl(svc({ deletePhoto: vi.fn().mockReturnValue(null) } as Partial)).deletePhoto(user, '7'))).toEqual({ status: 404, body: { error: 'Photo not found' } }); - expect(ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 7, file_path: null }) } as Partial)).deletePhoto(user, '7')).toEqual({ success: true }); + expect(await thrownAsync(() => ctl(svc({ deletePhoto: vi.fn().mockReturnValue(null) } as Partial)).deletePhoto(user, '7'))).toEqual({ status: 404, body: { error: 'Photo not found' } }); + expect(await ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 7, file_path: null }) } as Partial)).deletePhoto(user, '7')).toEqual({ success: true }); }); - it('gallery upload 400 no files / 403 not allowed, else returns photos', () => { - expect(thrown(() => ctl(svc()).uploadGalleryPhotos(user, '3', undefined))).toEqual({ status: 400, body: { error: 'No files uploaded' } }); - expect(thrown(() => ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([]) } as Partial)).uploadGalleryPhotos(user, '3', [{ filename: 'a.jpg' } as Express.Multer.File]))).toEqual({ status: 403, body: { error: 'Not allowed' } }); - expect(ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([{ id: 1 }]) } as Partial)).uploadGalleryPhotos(user, '3', [{ filename: 'a.jpg' } as Express.Multer.File])).toEqual({ photos: [{ id: 1 }] }); + it('gallery upload 400 no files / 403 not allowed, else commits + returns photos', async () => { + expect(await thrownAsync(() => ctl(svc()).uploadGalleryPhotos(user, '3', undefined))).toEqual({ status: 400, body: { error: 'No files uploaded' } }); + expect(await thrownAsync(() => ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([]) } as Partial)).uploadGalleryPhotos(user, '3', [{ filename: 'a.jpg' } as Express.Multer.File]))).toEqual({ status: 403, body: { error: 'Not allowed' } }); + expect(await ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([{ id: 1 }]) } as Partial)).uploadGalleryPhotos(user, '3', [{ filename: 'a.jpg' } as Express.Multer.File])).toEqual({ photos: [{ id: 1 }] }); + expect(storageStub.put).toHaveBeenCalledWith('journey', 'a.jpg', { tmpPath: undefined }); }); - it('gallery video: 400 no video, 403 not allowed, else stores the clip + poster (#823)', () => { + it('gallery video: 400 no video, 403 not allowed, else stores the clip + poster (#823)', async () => { const files = { video: [{ filename: 'v.mp4' } as Express.Multer.File], poster: [{ filename: 'p.jpg' } as Express.Multer.File] }; - expect(thrown(() => ctl(svc()).uploadGalleryVideo(user, '3', {}, {}))).toEqual({ status: 400, body: { error: 'No video uploaded' } }); - // Rejected with real paths → the cleanup unlinks the orphaned bytes (the files - // don't exist, so the best-effort catch swallows it). + expect(await thrownAsync(() => ctl(svc()).uploadGalleryVideo(user, '3', {}, {}))).toEqual({ status: 400, body: { error: 'No video uploaded' } }); + // Not-allowed after commit → the final objects are deleted through storage + // (the pre-commit unlink would miss them: file.path is already consumed). const withPaths = { video: [{ filename: 'v.mp4', path: '/nonexistent/v.mp4' } as Express.Multer.File], poster: [{ filename: 'p.jpg', path: '/nonexistent/p.jpg' } as Express.Multer.File] }; - expect(thrown(() => ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([]) } as Partial)).uploadGalleryVideo(user, '3', withPaths, { duration_ms: 'abc' }))).toEqual({ status: 403, body: { error: 'Not allowed' } }); + expect(await thrownAsync(() => ctl(svc({ uploadGalleryPhotos: vi.fn().mockReturnValue([]) } as Partial)).uploadGalleryVideo(user, '3', withPaths, { duration_ms: 'abc' }))).toEqual({ status: 403, body: { error: 'Not allowed' } }); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'v.mp4'); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'p.jpg'); const up = vi.fn().mockReturnValue([{ id: 7 }]); - expect(ctl(svc({ uploadGalleryPhotos: up } as Partial)).uploadGalleryVideo(user, '3', files, { duration_ms: '4200' })).toEqual({ photos: [{ id: 7 }] }); + expect(await ctl(svc({ uploadGalleryPhotos: up } as Partial)).uploadGalleryVideo(user, '3', files, { duration_ms: '4200' })).toEqual({ photos: [{ id: 7 }] }); expect(up).toHaveBeenCalledWith(3, 1, [{ path: 'journey/v.mp4', thumbnail: 'journey/p.jpg', mediaType: 'video', durationMs: 4200 }]); // No poster + no duration → thumbnail undefined, durationMs null. const up2 = vi.fn().mockReturnValue([{ id: 8 }]); - ctl(svc({ uploadGalleryPhotos: up2 } as Partial)).uploadGalleryVideo(user, '3', { video: [{ filename: 'v2.mp4' } as Express.Multer.File] }, {}); + await ctl(svc({ uploadGalleryPhotos: up2 } as Partial)).uploadGalleryVideo(user, '3', { video: [{ filename: 'v2.mp4' } as Express.Multer.File] }, {}); expect(up2).toHaveBeenCalledWith(3, 1, [{ path: 'journey/v2.mp4', thumbnail: undefined, mediaType: 'video', durationMs: null }]); }); @@ -160,62 +174,6 @@ describe('JourneyController', () => { expect(thrown(() => ctl(svc({ reorderEntries: vi.fn().mockReturnValue(false) } as Partial)).reorderEntries(user, '9', { orderedIds: [1, 2] }))).toEqual({ status: 403, body: { error: 'Not allowed' } }); }); - /* - * The Studio book (#1973). Its access shape is the same as everything around - * it — the service answers null for a journey the caller cannot see, and that - * is a 404 rather than a 403, so the route cannot be used to find out which - * journeys exist. - */ - it('book: 404 vs a journey with no book yet, and the save status codes', () => { - // A journey with no book yet is not an error: Studio opens on it, lays a - // book out and saves. 404 here would make "no book" and "no journey" - // indistinguishable to the client. - expect(ctl(svc(), { getBook: vi.fn().mockReturnValue(null), canOpen: vi.fn().mockReturnValue(true) }).getBook(user, '9')).toEqual({ book: null }); - expect(thrown(() => ctl(svc(), { getBook: vi.fn().mockReturnValue(null), canOpen: vi.fn().mockReturnValue(false) }).getBook(user, '9'))).toEqual({ status: 404, body: { error: 'Journey not found' } }); - expect(ctl(svc(), { getBook: vi.fn().mockReturnValue({ id: 3, version: 4 }), canOpen: vi.fn() }).getBook(user, '9')).toEqual({ book: { id: 3, version: 4 } }); - - expect(thrown(() => ctl(svc(), { saveBook: vi.fn().mockReturnValue(null) }).saveBook(user, '9', { document: {} } as never))).toEqual({ status: 404, body: { error: 'Journey not found' } }); - - // The record comes back bare, not in an envelope — the contract in - // shared/src/book/book-store.schema.ts is the record itself. - const broadcastSaved = vi.fn(); - const saveBook = vi.fn().mockReturnValue({ record: { id: 3, version: 5 } }); - expect(ctl(svc(), { saveBook, broadcastSaved }).saveBook(user, '9', { title: 'T', document: { v: 1 }, baseVersion: 4 } as never, 'sock-7')).toEqual({ id: 3, version: 5 }); - expect(saveBook).toHaveBeenCalledWith(9, 1, { title: 'T', document: { v: 1 }, baseVersion: 4 }); - // Forwarded so the client that just saved does not process its own change. - expect(broadcastSaved).toHaveBeenCalledWith(9, 1, { id: 3, version: 5 }, 'sock-7'); - - // A missing title is '' rather than undefined: the column is NOT NULL and - // an untitled book is an ordinary thing to have. - const untitled = vi.fn().mockReturnValue({ record: { id: 3, version: 1 } }); - ctl(svc(), { saveBook: untitled, broadcastSaved: vi.fn() }).saveBook(user, '9', { document: {} } as never); - expect(untitled.mock.calls[0][2].title).toBe(''); - }); - - it('book: a conflict is a 409 that carries the other version', () => { - /* - * Two people editing is the normal case, not an exception. A bare 409 would - * make showing what the other version is a second round trip, at exactly - * the moment someone is worried about losing work. - */ - const broadcastSaved = vi.fn(); - const saveBook = vi.fn().mockReturnValue({ conflict: { id: 3, version: 6, title: 'Theirs' } }); - expect(thrown(() => ctl(svc(), { saveBook, broadcastSaved }).saveBook(user, '9', { document: {}, baseVersion: 4 } as never))).toEqual({ - status: 409, - body: { error: 'Book was changed by someone else', current: { id: 3, version: 6, title: 'Theirs' } }, - }); - // Nothing was written, so nobody is told anything moved. - expect(broadcastSaved).not.toHaveBeenCalled(); - }); - - it('book: delete answers 204, or 404 out of reach', () => { - expect(ctl(svc(), { deleteBook: vi.fn().mockReturnValue(true) }).deleteBook(user, '9')).toBeUndefined(); - // False means there was nothing to delete, which is still a 204 — deleting - // a book that is already gone is not a failure. - expect(ctl(svc(), { deleteBook: vi.fn().mockReturnValue(false) }).deleteBook(user, '9')).toBeUndefined(); - expect(thrown(() => ctl(svc(), { deleteBook: vi.fn().mockReturnValue(null) }).deleteBook(user, '9'))).toEqual({ status: 404, body: { error: 'Journey not found' } }); - }); - it('contributors: add 400/403, update 403, remove 403', () => { expect(thrown(() => ctl(svc()).addContributor(user, '9', {}))).toEqual({ status: 400, body: { error: 'user_id required' } }); expect(thrown(() => ctl(svc({ addContributor: vi.fn().mockReturnValue(false) } as Partial)).addContributor(user, '9', { user_id: 2 }))).toEqual({ status: 403, body: { error: 'Not allowed' } }); @@ -274,17 +232,18 @@ describe('JourneyController', () => { expect(ctl(svc({ updatePhoto: vi.fn().mockReturnValue({ id: 7 }) } as Partial)).updatePhoto(user, '7', { caption: 'x' })).toEqual({ id: 7 }); }); - it('DELETE photo unlinks the file when a path exists', () => { - const unlinkSpy = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => undefined); - try { - expect(ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 7, file_path: 'journey/a.jpg' }) } as Partial)).deletePhoto(user, '7')).toEqual({ success: true }); - expect(unlinkSpy).toHaveBeenCalledTimes(1); - // a vanished file is swallowed - unlinkSpy.mockImplementationOnce(() => { throw new Error('ENOENT'); }); - expect(ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 8, file_path: 'journey/b.jpg' }) } as Partial)).deletePhoto(user, '8')).toEqual({ success: true }); - } finally { - unlinkSpy.mockRestore(); - } + it('DELETE photo removes the storage object when a path exists', async () => { + expect(await ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 7, file_path: 'journey/a.jpg' }) } as Partial)).deletePhoto(user, '7')).toEqual({ success: true }); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'a.jpg'); + // ...and the derived thumb goes with it (spec fix #2). + expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/a.jpg')); + // a vanished object is swallowed + (storageStub.delete as ReturnType).mockRejectedValueOnce(new Error('ENOENT')); + expect(await ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 8, file_path: 'journey/b.jpg' }) } as Partial)).deletePhoto(user, '8')).toEqual({ success: true }); + // a file_path outside the journey category never reaches storage + (storageStub.delete as ReturnType).mockClear(); + expect(await ctl(svc({ deletePhoto: vi.fn().mockReturnValue({ id: 9, file_path: 'elsewhere/x.jpg' }) } as Partial)).deletePhoto(user, '9')).toEqual({ success: true }); + expect(storageStub.delete).not.toHaveBeenCalled(); }); it('gallery provider-photos: batch (with passphrase), single 400/403, success', () => { @@ -297,30 +256,37 @@ describe('JourneyController', () => { expect(ctl(svc({ addProviderPhotoToGallery: vi.fn().mockReturnValue({ id: 3 }) } as Partial)).galleryProviderPhotos(user, '9', { provider: 'immich', asset_id: 'a' })).toEqual({ id: 3 }); }); - it('DELETE gallery photo: 404, then unlinks the file when present', () => { - expect(thrown(() => ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue(null) } as Partial)).deleteGalleryPhoto(user, '7'))).toEqual({ status: 404, body: { error: 'Photo not found or not allowed' } }); - // no file_path → nothing to unlink, returns void - expect(ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 7, file_path: null }) } as Partial)).deleteGalleryPhoto(user, '7')).toBeUndefined(); - const unlinkSpy = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => undefined); - try { - ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 8, file_path: 'journey/g.jpg' }) } as Partial)).deleteGalleryPhoto(user, '8'); - expect(unlinkSpy).toHaveBeenCalledTimes(1); - unlinkSpy.mockImplementationOnce(() => { throw new Error('ENOENT'); }); - expect(ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 9, file_path: 'journey/h.jpg' }) } as Partial)).deleteGalleryPhoto(user, '9')).toBeUndefined(); - } finally { - unlinkSpy.mockRestore(); - } + it('DELETE gallery photo: 404, then removes the object when present', async () => { + expect(await thrownAsync(() => ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue(null) } as Partial)).deleteGalleryPhoto(user, '7'))).toEqual({ status: 404, body: { error: 'Photo not found or not allowed' } }); + // no file_path → nothing to delete, returns void + expect(await ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 7, file_path: null }) } as Partial)).deleteGalleryPhoto(user, '7')).toBeUndefined(); + await ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 8, file_path: 'journey/g.jpg' }) } as Partial)).deleteGalleryPhoto(user, '8'); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'g.jpg'); + expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/g.jpg')); + (storageStub.delete as ReturnType).mockRejectedValueOnce(new Error('ENOENT')); + expect(await ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 9, file_path: 'journey/h.jpg' }) } as Partial)).deleteGalleryPhoto(user, '9')).toBeUndefined(); + }); + + it('DELETE gallery video also reclaims its uploaded poster (spec fix #2 extension)', async () => { + (storageStub.delete as ReturnType).mockClear(); + await ctl( + svc({ deleteGalleryPhoto: vi.fn().mockReturnValue({ id: 10, file_path: 'journey/clip.mp4', thumbnail_path: 'journey/poster.jpg' }) } as Partial), + ).deleteGalleryPhoto(user, '10'); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'clip.mp4'); + expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/clip.mp4')); + expect(storageStub.delete).toHaveBeenCalledWith('journey', 'poster.jpg'); }); it('PATCH /:id returns the updated journey on success', () => { expect(ctl(svc({ updateJourney: vi.fn().mockReturnValue({ id: 9 }) } as Partial)).update(user, '9', { title: 'x' })).toEqual({ id: 9 }); }); - it('cover upload: 400 without file, 404 when the journey is gone, else returns the journey', () => { - expect(thrown(() => ctl(svc()).cover(user, '9', undefined))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); - expect(thrown(() => ctl(svc({ updateJourney: vi.fn().mockReturnValue(null) } as Partial)).cover(user, '9', { filename: 'c.jpg' } as Express.Multer.File))).toEqual({ status: 404, body: { error: 'Journey not found' } }); + it('cover upload: 400 without file, 404 when the journey is gone, else commits + returns the journey', async () => { + expect(await thrownAsync(() => ctl(svc()).cover(user, '9', undefined))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); + expect(await thrownAsync(() => ctl(svc({ updateJourney: vi.fn().mockReturnValue(null) } as Partial)).cover(user, '9', { filename: 'c.jpg' } as Express.Multer.File))).toEqual({ status: 404, body: { error: 'Journey not found' } }); const updateJourney = vi.fn().mockReturnValue({ id: 9, cover_image: 'journey/c.jpg' }); - expect(ctl(svc({ updateJourney } as Partial)).cover(user, '9', { filename: 'c.jpg' } as Express.Multer.File)).toEqual({ id: 9, cover_image: 'journey/c.jpg' }); + expect(await ctl(svc({ updateJourney } as Partial)).cover(user, '9', { filename: 'c.jpg' } as Express.Multer.File)).toEqual({ id: 9, cover_image: 'journey/c.jpg' }); + expect(storageStub.put).toHaveBeenCalledWith('journey', 'c.jpg', { tmpPath: undefined }); expect(updateJourney).toHaveBeenCalledWith(9, 1, { cover_image: 'journey/c.jpg' }); }); @@ -352,50 +318,105 @@ describe('JourneyController', () => { it('deleteShareLink returns success when removed', () => { expect(ctl(svc({ deleteJourneyShareLink: vi.fn().mockReturnValue(true) } as Partial)).deleteShareLink(user, '9')).toEqual({ success: true }); }); + /* + * The Studio book (#1973). Its access shape is the same as everything around + * it — the service answers null for a journey the caller cannot see, and that + * is a 404 rather than a 403, so the route cannot be used to find out which + * journeys exist. + */ + it('book: 404 vs a journey with no book yet, and the save status codes', () => { + // A journey with no book yet is not an error: Studio opens on it, lays a + // book out and saves. 404 here would make "no book" and "no journey" + // indistinguishable to the client. + expect(ctl(svc(), { getBook: vi.fn().mockReturnValue(null), canOpen: vi.fn().mockReturnValue(true) }).getBook(user, '9')).toEqual({ book: null }); + expect(thrown(() => ctl(svc(), { getBook: vi.fn().mockReturnValue(null), canOpen: vi.fn().mockReturnValue(false) }).getBook(user, '9'))).toEqual({ status: 404, body: { error: 'Journey not found' } }); + expect(ctl(svc(), { getBook: vi.fn().mockReturnValue({ id: 3, version: 4 }), canOpen: vi.fn() }).getBook(user, '9')).toEqual({ book: { id: 3, version: 4 } }); + + expect(thrown(() => ctl(svc(), { saveBook: vi.fn().mockReturnValue(null) }).saveBook(user, '9', { document: {} } as never))).toEqual({ status: 404, body: { error: 'Journey not found' } }); + + // The record comes back bare, not in an envelope — the contract in + // shared/src/book/book-store.schema.ts is the record itself. + const broadcastSaved = vi.fn(); + const saveBook = vi.fn().mockReturnValue({ record: { id: 3, version: 5 } }); + expect(ctl(svc(), { saveBook, broadcastSaved }).saveBook(user, '9', { title: 'T', document: { v: 1 }, baseVersion: 4 } as never, 'sock-7')).toEqual({ id: 3, version: 5 }); + expect(saveBook).toHaveBeenCalledWith(9, 1, { title: 'T', document: { v: 1 }, baseVersion: 4 }); + // Forwarded so the client that just saved does not process its own change. + expect(broadcastSaved).toHaveBeenCalledWith(9, 1, { id: 3, version: 5 }, 'sock-7'); + + // A missing title is '' rather than undefined: the column is NOT NULL and + // an untitled book is an ordinary thing to have. + const untitled = vi.fn().mockReturnValue({ record: { id: 3, version: 1 } }); + ctl(svc(), { saveBook: untitled, broadcastSaved: vi.fn() }).saveBook(user, '9', { document: {} } as never); + expect(untitled.mock.calls[0][2].title).toBe(''); + }); + + it('book: a conflict is a 409 that carries the other version', () => { + /* + * Two people editing is the normal case, not an exception. A bare 409 would + * make showing what the other version is a second round trip, at exactly + * the moment someone is worried about losing work. + */ + const broadcastSaved = vi.fn(); + const saveBook = vi.fn().mockReturnValue({ conflict: { id: 3, version: 6, title: 'Theirs' } }); + expect(thrown(() => ctl(svc(), { saveBook, broadcastSaved }).saveBook(user, '9', { document: {}, baseVersion: 4 } as never))).toEqual({ + status: 409, + body: { error: 'Book was changed by someone else', current: { id: 3, version: 6, title: 'Theirs' } }, + }); + // Nothing was written, so nobody is told anything moved. + expect(broadcastSaved).not.toHaveBeenCalled(); + }); + + it('book: delete answers 204, or 404 out of reach', () => { + expect(ctl(svc(), { deleteBook: vi.fn().mockReturnValue(true) }).deleteBook(user, '9')).toBeUndefined(); + // False means there was nothing to delete, which is still a 204 — deleting + // a book that is already gone is not a failure. + expect(ctl(svc(), { deleteBook: vi.fn().mockReturnValue(false) }).deleteBook(user, '9')).toBeUndefined(); + expect(thrown(() => ctl(svc(), { deleteBook: vi.fn().mockReturnValue(null) }).deleteBook(user, '9'))).toEqual({ status: 404, body: { error: 'Journey not found' } }); + }); }); describe('JourneyPublicController', () => { it('GET /:token 404 / json', () => { - expect(thrown(() => new JourneyPublicController(svc({ getPublicJourney: vi.fn().mockReturnValue(null) } as Partial)).get('tok'))).toEqual({ status: 404, body: { error: 'Not found' } }); - expect(new JourneyPublicController(svc({ getPublicJourney: vi.fn().mockReturnValue({ id: 1 }) } as Partial)).get('tok')).toEqual({ id: 1 }); + expect(thrown(() => new JourneyPublicController(svc({ getPublicJourney: vi.fn().mockReturnValue(null) } as Partial), storageStub).get('tok'))).toEqual({ status: 404, body: { error: 'Not found' } }); + expect(new JourneyPublicController(svc({ getPublicJourney: vi.fn().mockReturnValue({ id: 1 }) } as Partial), storageStub).get('tok')).toEqual({ id: 1 }); }); it('photo proxy 404 on invalid token, else streams', async () => { - expect(await thrownAsync(() => new JourneyPublicController(svc({ validateShareTokenForPhoto: vi.fn().mockReturnValue(null) } as Partial)).photo('tok', '7', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); + expect(await thrownAsync(() => new JourneyPublicController(svc({ validateShareTokenForPhoto: vi.fn().mockReturnValue(null) } as Partial), storageStub).photo('tok', '7', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); const streamPhoto = vi.fn().mockResolvedValue(undefined); const s = svc({ validateShareTokenForPhoto: vi.fn().mockReturnValue({ ownerId: 2 }), streamPhoto } as Partial); - await new JourneyPublicController(s).photo('tok', '7', 'original', {} as Response); + await new JourneyPublicController(s, storageStub).photo('tok', '7', 'original', {} as Response); expect(streamPhoto).toHaveBeenCalledWith({}, 2, 7, 'original'); }); it('legacy photo proxy: 404 invalid token, immich path streams', async () => { - expect(await thrownAsync(() => new JourneyPublicController(svc({ validateShareTokenForAsset: vi.fn().mockReturnValue(null) } as Partial)).legacyPhoto('tok', 'immich', 'a1', '2', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); + expect(await thrownAsync(() => new JourneyPublicController(svc({ validateShareTokenForAsset: vi.fn().mockReturnValue(null) } as Partial), storageStub).legacyPhoto('tok', 'immich', 'a1', '2', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); // One call for every provider now, with the ids in a ref instead of in a // per-provider argument order. const streamProviderAsset = vi.fn().mockResolvedValue(undefined); const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }), streamProviderAsset } as Partial); - await new JourneyPublicController(s).legacyPhoto('tok', 'immich', 'a1', '2', 'original', {} as Response); + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'immich', 'a1', '2', 'original', {} as Response); expect(streamProviderAsset).toHaveBeenCalledWith({}, 'immich', { userId: 5, ownerId: 5, assetId: 'a1' }, 'original'); }); it('photo proxy streams thumbnails too', async () => { const streamPhoto = vi.fn().mockResolvedValue(undefined); const s = svc({ validateShareTokenForPhoto: vi.fn().mockReturnValue({ ownerId: 3 }), streamPhoto } as Partial); - await new JourneyPublicController(s).photo('tok', '7', 'thumbnail', {} as Response); + await new JourneyPublicController(s, storageStub).photo('tok', '7', 'thumbnail', {} as Response); expect(streamPhoto).toHaveBeenCalledWith({}, 3, 7, 'thumbnail'); }); it('legacy photo proxy: synology streams, and a failure becomes a 404 json', async () => { const streamProviderAsset = vi.fn().mockResolvedValue(undefined); const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }), streamProviderAsset } as Partial); - await new JourneyPublicController(s).legacyPhoto('tok', 'synologyphotos', 'a1', '2', 'thumbnail', {} as Response); + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'synologyphotos', 'a1', '2', 'thumbnail', {} as Response); expect(streamProviderAsset).toHaveBeenCalledWith({}, 'synologyphotos', { userId: 5, ownerId: 5, assetId: 'a1' }, 'thumbnail'); const status = vi.fn().mockReturnThis(); const json = vi.fn(); const res = { status, json } as unknown as Response; const failing = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 0 }), streamProviderAsset: vi.fn().mockRejectedValue(new Error('no synology')) } as Partial); - await new JourneyPublicController(failing).legacyPhoto('tok', 'synologyphotos', 'a1', '6', 'original', res); + await new JourneyPublicController(failing, storageStub).legacyPhoto('tok', 'synologyphotos', 'a1', '6', 'original', res); expect(status).toHaveBeenCalledWith(404); expect(json).toHaveBeenCalledWith({ error: 'Provider not supported' }); }); @@ -409,7 +430,7 @@ describe('JourneyPublicController', () => { const json = vi.fn(); const res = { status, json } as unknown as Response; const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }), streamProviderAsset } as Partial); - await new JourneyPublicController(s).legacyPhoto('tok', 'photoprism', 'a1', '2', 'original', res); + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'photoprism', 'a1', '2', 'original', res); expect(status).toHaveBeenCalledWith(404); expect(json).toHaveBeenCalledWith({ error: 'Provider not supported' }); }); @@ -417,43 +438,51 @@ describe('JourneyPublicController', () => { it('legacy photo proxy: falls back to the path ownerId when the token has none', async () => { const streamProviderAsset = vi.fn().mockResolvedValue(undefined); const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 0 }), streamProviderAsset } as Partial); - await new JourneyPublicController(s).legacyPhoto('tok', 'immich', 'a1', '8', 'original', {} as Response); + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'immich', 'a1', '8', 'original', {} as Response); expect(streamProviderAsset).toHaveBeenCalledWith({}, 'immich', { userId: 8, ownerId: 8, assetId: 'a1' }, 'original'); }); - it('legacy photo proxy: local provider 404s when the resolved file does not exist', async () => { - const existsSpy = vi.spyOn(fs, 'existsSync').mockReturnValue(false); - try { - const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); - expect(await thrownAsync(() => new JourneyPublicController(s).legacyPhoto('tok', 'local', 'gone.jpg', '2', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); - } finally { - existsSpy.mockRestore(); - } + it('legacy photo proxy: local provider 404s when the object does not exist', async () => { + storageExists.mockResolvedValue(false); + const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); + expect(await thrownAsync(() => new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'local', 'gone.jpg', '2', 'thumbnail', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); + expect(storageSendToResponse).not.toHaveBeenCalled(); + }); + + it('legacy photo proxy: local provider serves through storage with the day cache header', async () => { + storageExists.mockResolvedValue(true); + storageSendToResponse.mockResolvedValue(undefined); + const set = vi.fn(); + const res = { set } as unknown as Response; + const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'local', 'photo.jpg', '2', 'original', res); + expect(storageExists).toHaveBeenCalledWith('journey', 'photo.jpg'); + expect(set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400'); + expect(storageSendToResponse).toHaveBeenCalledWith('journey', 'photo.jpg', res); }); it('legacy photo proxy: local provider cannot escape uploads/journey via a traversal asset id', async () => { - // Pretend any path exists so we can inspect exactly what would be served. - const existsSpy = vi.spyOn(fs, 'existsSync').mockReturnValue(true); - try { - const sendFile = vi.fn(); - const res = { set: vi.fn(), sendFile } as unknown as Response; - const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); + storageExists.mockResolvedValue(true); + storageSendToResponse.mockResolvedValue(undefined); + const res = { set: vi.fn() } as unknown as Response; + const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); - // Express decodes %2F in a single path param to '/', so the handler sees this. - await new JourneyPublicController(s).legacyPhoto('tok', 'local', '../../files/secret.pdf', '2', 'original', res); + // Express decodes %2F in a single path param to '/', so the handler sees this. + await new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'local', '../../files/secret.pdf', '2', 'original', res); - expect(sendFile).toHaveBeenCalledTimes(1); - // Sent as basename + root rather than as one absolute path: an absolute - // path resolves against the rewritten req.url under the Nest - // ExpressAdapter and 404s. The traversal is still collapsed — what is - // asserted is the pair, since together they are the file that gets served. - const [served, options] = sendFile.mock.calls[0] as [string, { root: string }]; - expect(served).toBe('secret.pdf'); - expect(options.root).toMatch(/[\\/]journey$/); - expect(options.root).not.toMatch(/[\\/]files/); - expect(path.join(options.root, served)).toMatch(/[\\/]journey[\\/]secret\.pdf$/); - } finally { - existsSpy.mockRestore(); - } + // basename() collapses the traversal before the storage lookup, and central + // key validation (storage-keys.ts) rejects anything that still carries a + // path — the asset can never address the sibling files/ category. + expect(storageExists).toHaveBeenCalledWith('journey', 'secret.pdf'); + expect(storageSendToResponse).toHaveBeenCalledWith('journey', 'secret.pdf', res); + }); + + it('legacy photo proxy: a bare traversal asset id reads as a miss, not a crash', async () => { + // '..' survives basename(); storage.exists rejects it as an invalid key, + // which the handler reads as a miss. + storageExists.mockRejectedValue(new Error('invalid storage key: journey/..')); + const s = svc({ validateShareTokenForAsset: vi.fn().mockReturnValue({ ownerId: 5 }) } as Partial); + expect(await thrownAsync(() => new JourneyPublicController(s, storageStub).legacyPhoto('tok', 'local', '..', '2', 'original', {} as Response))).toEqual({ status: 404, body: { error: 'Not found' } }); + expect(storageSendToResponse).not.toHaveBeenCalled(); }); }); diff --git a/server/tests/unit/nest/maps.controller.test.ts b/server/tests/unit/nest/maps.controller.test.ts index a058db614..9d245c93b 100644 --- a/server/tests/unit/nest/maps.controller.test.ts +++ b/server/tests/unit/nest/maps.controller.test.ts @@ -1,18 +1,21 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { HttpException } from '@nestjs/common'; import type { Response } from 'express'; - -const { createReadStream } = vi.hoisted(() => ({ createReadStream: vi.fn() })); -vi.mock('node:fs', () => ({ createReadStream })); +import { PassThrough } from 'node:stream'; import { MapsController } from '../../../src/nest/maps/maps.controller'; import type { MapsService } from '../../../src/nest/maps/maps.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; const user = { id: 3 } as User; +// The bytes route is the only storage consumer; everything else ignores it. +const getStream = vi.fn(); +const storageStub = { getStream } as unknown as StorageService; + function makeController(svc: Partial) { - return new MapsController(svc as MapsService); + return new MapsController(svc as MapsService, storageStub); } /** Run an async handler, expecting an HttpException; return its { status, body }. */ @@ -230,51 +233,73 @@ describe('MapsController (parity with the legacy /api/maps route)', () => { }); describe('GET /place-photo/:placeId/bytes', () => { + // A real PassThrough (like storage.service.test.ts's makeRes) rather than + // a { on, pipe } stub: pipeline() needs authentic stream/eos semantics + // (close/error/finish wiring) that a bare mock can't satisfy. function makeRes() { - const res = { + const sink = new PassThrough(); + const chunks: Buffer[] = []; + sink.on('data', (c: Buffer) => chunks.push(c)); + // Capture the real PassThrough#end BEFORE overwriting the property + // below — res and sink are the same object (Object.assign mutates and + // returns its target), so referencing sink.end inside the wrapper + // would recurse into itself. + const realEnd = sink.end.bind(sink) as (...a: unknown[]) => unknown; + const res = Object.assign(sink, { statusCode: 200, - headersSent: false, status: vi.fn(function (this: unknown, c: number) { (res as { statusCode: number }).statusCode = c; return res; }), json: vi.fn(), set: vi.fn(), type: vi.fn(), - end: vi.fn(), + end: vi.fn((...args: unknown[]) => realEnd(...args)), + body: () => Buffer.concat(chunks).toString(), + }); + return res as unknown as Response & { + status: ReturnType; + json: ReturnType; + set: ReturnType; + type: ReturnType; + end: ReturnType; + body: () => string; }; - return res as unknown as Response & { status: ReturnType; json: ReturnType; set: ReturnType; type: ReturnType; end: ReturnType }; } - beforeEach(() => createReadStream.mockReset()); + // Braced body on purpose: mockReset() returns the mock, and a function + // returned from beforeEach is invoked as a teardown callback — which would + // call getStream() bare and turn a mockRejectedValue impl into an + // unhandled rejection. + beforeEach(() => { + getStream.mockReset(); + }); // Places persist this URL in image_url, so an evicted cache entry means one // request per place on a trip render. 404 for each of them is the ban vector // from #1727 — an uncached photo answers 204 with an empty body instead. - it('204 without a body when the photo is not cached', () => { + it('204 without a body when the photo is not cached', async () => { const res = makeRes(); - makeController({ photoBytesPath: () => null }).placePhotoBytes('p1', res); + await makeController({ photoBytesKey: async () => null }).placePhotoBytes('p1', res); expect(res.status).toHaveBeenCalledWith(204); expect(res.end).toHaveBeenCalled(); expect(res.json).not.toHaveBeenCalled(); - expect(createReadStream).not.toHaveBeenCalled(); + expect(getStream).not.toHaveBeenCalled(); }); - it('streams the cached file with image/jpeg + an immutable cache header on a hit', () => { - const stream = { on: vi.fn().mockReturnThis(), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + it('streams the cached bytes with image/jpeg + an immutable cache header on a hit', async () => { + const stream = new PassThrough(); + stream.end('hello'); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); const res = makeRes(); - makeController({ photoBytesPath: () => '/cache/p1.jpg' }).placePhotoBytes('p1', res); + await makeController({ photoBytesKey: async () => 'abc.jpg' }).placePhotoBytes('p1', res); expect(res.set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=2592000, immutable'); expect(res.type).toHaveBeenCalledWith('image/jpeg'); - expect(createReadStream).toHaveBeenCalledWith('/cache/p1.jpg'); - expect(stream.pipe).toHaveBeenCalledWith(res); + expect(getStream).toHaveBeenCalledWith('photos-google', 'abc.jpg'); + expect(res.body()).toBe('hello'); }); - it('falls back to an empty 204 when the read stream errors', () => { - let onError: () => void = () => {}; - const stream = { on: vi.fn((ev: string, cb: () => void) => { if (ev === 'error') onError = cb; return stream; }), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + it('falls back to an empty 204 when the stream cannot be opened (cache-delete race)', async () => { + getStream.mockRejectedValue(new Error('storage object not found')); const res = makeRes(); - makeController({ photoBytesPath: () => '/cache/p1.jpg' }).placePhotoBytes('p1', res); - onError(); + await makeController({ photoBytesKey: async () => 'abc.jpg' }).placePhotoBytes('p1', res); expect(res.status).toHaveBeenCalledWith(204); expect(res.end).toHaveBeenCalled(); // The hit path already asked for a month of immutable caching — that header @@ -283,14 +308,44 @@ describe('MapsController (parity with the legacy /api/maps route)', () => { expect(res.set).toHaveBeenLastCalledWith('Cache-Control', 'no-store'); }); - it('does not re-send a 204 when the stream errors after headers were flushed', () => { - let onError: () => void = () => {}; - const stream = { on: vi.fn((ev: string, cb: () => void) => { if (ev === 'error') onError = cb; return stream; }), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + it('falls back to an empty 204 when the stream errors before headers were flushed', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); const res = makeRes(); - (res as { headersSent: boolean }).headersSent = true; - makeController({ photoBytesPath: () => '/cache/p1.jpg' }).placePhotoBytes('p1', res); - onError(); + const pending = makeController({ photoBytesKey: async () => 'abc.jpg' }).placePhotoBytes('p1', res); + // Let pipeline() attach its listeners before erroring the source, so + // the event isn't missed to a scheduling race. + await new Promise((resolve) => setImmediate(resolve)); + // No bytes reached res yet — pipeline rejects before any write. + stream.destroy(new Error('source boom')); + await pending; + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + expect(res.set).toHaveBeenLastCalledWith('Cache-Control', 'no-store'); + }); + + it('swallows an abort-like error once headers are flushed (client walked away)', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); + const res = makeRes(); + (res as unknown as { headersSent: boolean }).headersSent = true; + const pending = makeController({ photoBytesKey: async () => 'abc.jpg' }).placePhotoBytes('p1', res); + await new Promise((resolve) => setImmediate(resolve)); + stream.destroy(Object.assign(new Error('reset'), { code: 'ECONNRESET' })); + await expect(pending).resolves.toBeUndefined(); + expect(res.status).not.toHaveBeenCalled(); + expect(res.end).not.toHaveBeenCalled(); + }); + + it('re-throws a real (non-abort) error once headers are flushed', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); + const res = makeRes(); + (res as unknown as { headersSent: boolean }).headersSent = true; + const pending = makeController({ photoBytesKey: async () => 'abc.jpg' }).placePhotoBytes('p1', res); + await new Promise((resolve) => setImmediate(resolve)); + stream.destroy(Object.assign(new Error('disk read error'), { code: 'EIO' })); + await expect(pending).rejects.toThrow('disk read error'); expect(res.status).not.toHaveBeenCalled(); expect(res.end).not.toHaveBeenCalled(); }); diff --git a/server/tests/unit/nest/maps.service.test.ts b/server/tests/unit/nest/maps.service.test.ts index 93bfb7e02..fecd1ab6c 100644 --- a/server/tests/unit/nest/maps.service.test.ts +++ b/server/tests/unit/nest/maps.service.test.ts @@ -129,7 +129,7 @@ const photoCacheStub = { markError: (placeId: string, kind?: string) => mockCacheMarkError(placeId, kind), getInFlight: (placeId: string) => mockCacheGetInFlight(placeId), setInFlight: (placeId: string, p: Promise) => mockCacheSetInFlight(placeId, p), - serveFilePath: (placeId: string) => mockServeFilePath(placeId), + serveKey: (placeId: string) => mockServeFilePath(placeId), } as unknown as PlacePhotoCacheService; import { db } from '../../../src/db/database'; @@ -1964,9 +1964,8 @@ describe('getPlacePhoto (fetch stubbed)', () => { it('MAPS-043b: returns cached photo when disk cache returns a hit', async () => { const placeId = `coords:cache-test-${Date.now()}`; const cachedUrl = `/api/maps/place-photo/${encodeURIComponent(placeId)}/bytes`; - mockCacheGet.mockReturnValue({ + mockCacheGet.mockResolvedValue({ photoUrl: cachedUrl, - filePath: `/tmp/${placeId}.jpg`, attribution: null, }); const fetchMock = vi.fn(); @@ -2446,16 +2445,16 @@ describe('kill-switch settings reads', () => { }); }); -describe('photoBytesPath', () => { - it('returns the cached file path from placePhotoCache', () => { - mockServeFilePath.mockReturnValue('/cache/p1.jpg'); - expect(svc.photoBytesPath('p1')).toBe('/cache/p1.jpg'); +describe('photoBytesKey', () => { + it('returns the cached storage name from placePhotoCache', () => { + mockServeFilePath.mockReturnValue('abc.jpg'); + expect(svc.photoBytesKey('p1')).toBe('abc.jpg'); expect(mockServeFilePath).toHaveBeenCalledWith('p1'); }); it('returns null when nothing is cached', () => { mockServeFilePath.mockReturnValue(null); - expect(svc.photoBytesPath('p1')).toBeNull(); + expect(svc.photoBytesKey('p1')).toBeNull(); }); }); diff --git a/server/tests/unit/nest/notification-preferences.service.test.ts b/server/tests/unit/nest/notification-preferences.service.test.ts index 95fa97483..6b9165ca9 100644 --- a/server/tests/unit/nest/notification-preferences.service.test.ts +++ b/server/tests/unit/nest/notification-preferences.service.test.ts @@ -118,11 +118,12 @@ describe('getPreferencesMatrix', () => { expect(event_types.length).toBe(12); }); - it('NPREF-005b — admin scope returns only version_available', () => { + it('NPREF-005b — admin scope returns the admin-scoped events', () => { const { user } = createAdmin(testDb); const { event_types } = getPreferencesMatrix(user.id, 'admin', 'admin'); expect(event_types).toContain('version_available'); - expect(event_types.length).toBe(1); + expect(event_types).toContain('replica_failure'); + expect(event_types.length).toBe(2); }); it('NPREF-006 — returns default true for all preferences when no stored prefs', () => { diff --git a/server/tests/unit/nest/notifications.service.test.ts b/server/tests/unit/nest/notifications.service.test.ts index 47ffab34b..050a94cc2 100644 --- a/server/tests/unit/nest/notifications.service.test.ts +++ b/server/tests/unit/nest/notifications.service.test.ts @@ -388,6 +388,49 @@ describe('send() — in-app notification content', () => { expect(notifs[0].navigate_target).toBe('/admin'); expect(notifs[0].title_key).toBe('notif.version_available.title'); }); + + it('NOTIF-RF-002 suppressed > 0 stores the textSuppressed key; 0 keeps the plain key', async () => { + const { user: admin } = createAdmin(testDb); + setNotificationChannels(testDb, 'none'); + + await send({ + event: 'replica_failure', + actorId: null, + scope: 'admin', + targetId: 0, + params: { backend: 'minio', op: 'put', key: 'trips/1/a.jpg', error: 'ECONNRESET', suppressed: '3' }, + }); + + await send({ + event: 'replica_failure', + actorId: null, + scope: 'admin', + targetId: 0, + params: { backend: 'minio', op: 'put', key: 'trips/1/b.jpg', error: 'ECONNRESET', suppressed: '0' }, + }); + + const notifs = getInAppNotifications(admin.id); + expect(notifs.length).toBe(2); + expect(notifs[0].text_key).toBe('notif.replica_failure.textSuppressed'); + expect(notifs[1].text_key).toBe('notif.replica_failure.text'); + }); + + it('NOTIF-RF-003 a missing suppressed param is NOT treated as suppressed (truthy guard, not just !== "0")', async () => { + const { user: admin } = createAdmin(testDb); + setNotificationChannels(testDb, 'none'); + + await send({ + event: 'replica_failure', + actorId: null, + scope: 'admin', + targetId: 0, + params: { backend: 'minio', op: 'put', key: 'trips/1/c.jpg', error: 'ECONNRESET' }, + }); + + const notifs = getInAppNotifications(admin.id); + expect(notifs.length).toBe(1); + expect(notifs[0].text_key).toBe('notif.replica_failure.text'); + }); }); // ───────────────────────────────────────────────────────────────────────────── diff --git a/server/tests/unit/nest/notifications/replica-failure-debouncer.test.ts b/server/tests/unit/nest/notifications/replica-failure-debouncer.test.ts new file mode 100644 index 000000000..204dd6245 --- /dev/null +++ b/server/tests/unit/nest/notifications/replica-failure-debouncer.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect } from 'vitest'; +import { ReplicaFailureDebouncer } from '../../../../src/nest/notifications/replica-failure-debouncer'; + +const HOUR = 60 * 60 * 1000; + +describe('ReplicaFailureDebouncer', () => { + it('DEB-001 first failure per backend is admitted with suppressed=0', () => { + let now = 1_000_000; + const debouncer = new ReplicaFailureDebouncer(HOUR, () => now); + expect(debouncer.admit('s3-bkp')).toBe(0); + }); + + it('DEB-002 failures inside the window are suppressed; the next one after it carries the count', () => { + let now = 1_000_000; + const debouncer = new ReplicaFailureDebouncer(HOUR, () => now); + expect(debouncer.admit('s3-bkp')).toBe(0); + now += 1000; + expect(debouncer.admit('s3-bkp')).toBeNull(); + expect(debouncer.admit('s3-bkp')).toBeNull(); + now += HOUR; + expect(debouncer.admit('s3-bkp')).toBe(2); // the two suppressed ones ride the summary + now += 1000; + expect(debouncer.admit('s3-bkp')).toBeNull(); // window reset + }); + + it('DEB-003 backends debounce independently', () => { + let now = 1_000_000; + const debouncer = new ReplicaFailureDebouncer(HOUR, () => now); + expect(debouncer.admit('a')).toBe(0); + expect(debouncer.admit('b')).toBe(0); + expect(debouncer.admit('a')).toBeNull(); + expect(debouncer.admit('b')).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/notifications/storage-health-notifier.service.test.ts b/server/tests/unit/nest/notifications/storage-health-notifier.service.test.ts new file mode 100644 index 000000000..6881c966c --- /dev/null +++ b/server/tests/unit/nest/notifications/storage-health-notifier.service.test.ts @@ -0,0 +1,34 @@ +import { describe, it, expect, vi } from 'vitest'; +import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; +import { StorageHealthNotifierService } from '../../../../src/nest/notifications/storage-health-notifier.service'; +import type { NotificationsService } from '../../../../src/nest/notifications/notifications.service'; + +describe('StorageHealthNotifierService', () => { + it('NOTIF-001 subscribes on bootstrap and sends an admin-scoped replica_failure with params', () => { + const events = new StorageEventsService(); + const send = vi.fn().mockResolvedValue(undefined); + const notifier = new StorageHealthNotifierService(events, { send } as unknown as NotificationsService); + notifier.onApplicationBootstrap(); + events.emitReplicaFailure({ backend: 's3-bkp', key: 'backups/db.zip', op: 'put', error: 'timeout', at: 1 }); + expect(send).toHaveBeenCalledWith({ + event: 'replica_failure', + actorId: null, + scope: 'admin', + targetId: 0, + params: { backend: 's3-bkp', key: 'backups/db.zip', op: 'put', error: 'timeout', suppressed: '0' }, + }); + // Second failure inside the window: suppressed, no second send. + events.emitReplicaFailure({ backend: 's3-bkp', key: 'backups/x.zip', op: 'put', error: 'timeout', at: 2 }); + expect(send).toHaveBeenCalledTimes(1); + }); + + it('NOTIF-002 a rejected send never throws into the emitter (write path stays safe)', () => { + const events = new StorageEventsService(); + const send = vi.fn().mockRejectedValue(new Error('smtp down')); + const notifier = new StorageHealthNotifierService(events, { send } as unknown as NotificationsService); + notifier.onApplicationBootstrap(); + expect(() => + events.emitReplicaFailure({ backend: 'b', key: 'k', op: 'delete', error: 'e', at: 3 }), + ).not.toThrow(); + }); +}); diff --git a/server/tests/unit/nest/oidc.service.test.ts b/server/tests/unit/nest/oidc.service.test.ts index d707d954e..30f8dc056 100644 --- a/server/tests/unit/nest/oidc.service.test.ts +++ b/server/tests/unit/nest/oidc.service.test.ts @@ -103,6 +103,7 @@ import { AuthService } from '../../../src/nest/auth/auth.service'; import { WebauthnConfigService } from '../../../src/nest/auth/webauthn-config.service'; import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; import { OidcService } from '../../../src/nest/oidc/oidc.service'; import { MailerService } from '../../../src/nest/notifications/mailer/mailer.service'; @@ -123,6 +124,7 @@ const auth = new AuthService( new UserCleanupService(new DatabaseService(testDb), new BudgetService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new ExchangeRatesService(), new RealtimeService())), mailerStub, new EphemeralTokenService(), + new AllowedFileTypesService(new DatabaseService(testDb)), ); const svc = new OidcService(new DatabaseService(testDb), auth, membership); diff --git a/server/tests/unit/nest/passkey.service.test.ts b/server/tests/unit/nest/passkey.service.test.ts index f84ed289e..f330b68e6 100644 --- a/server/tests/unit/nest/passkey.service.test.ts +++ b/server/tests/unit/nest/passkey.service.test.ts @@ -84,6 +84,7 @@ import { PasskeyService } from '../../../src/nest/auth/passkey.service'; import { WebauthnConfigService } from '../../../src/nest/auth/webauthn-config.service'; import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; +import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; import { MailerService } from '../../../src/nest/notifications/mailer/mailer.service'; // MailerService is injected since the notifications fold — a stub instead of a @@ -113,6 +114,7 @@ const auth = new AuthService( new UserCleanupService(new DatabaseService(testDb), new BudgetService(new DatabaseService(testDb), new PermissionsService(new DatabaseService(testDb)), new ExchangeRatesService(), new RealtimeService())), mailerStub, new EphemeralTokenService(), + new AllowedFileTypesService(new DatabaseService(testDb)), ); const svc = new PasskeyService(new DatabaseService(testDb), auth, webauthn); diff --git a/server/tests/unit/nest/photo-capture-backfill.service.test.ts b/server/tests/unit/nest/photo-capture-backfill.service.test.ts index 48452aa5f..b389a1acc 100644 --- a/server/tests/unit/nest/photo-capture-backfill.service.test.ts +++ b/server/tests/unit/nest/photo-capture-backfill.service.test.ts @@ -11,6 +11,13 @@ import exifr from 'exifr'; import { PhotoCaptureBackfillService } from '../../../src/nest/memories/photo-capture-backfill.service'; import type { PhotoResolverService } from '../../../src/nest/memories/photo-resolver.service'; import type { TrekPhotosRepository } from '../../../src/nest/photos/trek-photos.repository'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; + +// The storage layer's job here is only to hand the EXIF reader a real path; +// materialization (local fast-path vs remote temp download) has its own tests. +const storageStub = { + withLocalFile: vi.fn(async (_category: string, name: string, fn: (absPath: string) => Promise) => fn(`/uploads/journey/${name}`)), +} as unknown as StorageService; type Row = { id: number; provider?: string; file_path?: string | null; taken_at?: string | null; lat?: number | null; lng?: number | null }; @@ -24,7 +31,7 @@ function build(rows: Row[], info: Record) { recordCaptureMetadata, } as unknown as TrekPhotosRepository; const resolver = { getPhotoInfo } as unknown as PhotoResolverService; - return { svc: new PhotoCaptureBackfillService(resolver, photos), recordCaptureMetadata, getPhotoInfo }; + return { svc: new PhotoCaptureBackfillService(resolver, photos, storageStub), recordCaptureMetadata, getPhotoInfo }; } describe('PhotoCaptureBackfillService', () => { @@ -109,7 +116,7 @@ describe('PhotoCaptureBackfillService — local files', () => { recordCaptureMetadata, } as unknown as TrekPhotosRepository; const resolver = { getPhotoInfo } as unknown as PhotoResolverService; - return { svc: new PhotoCaptureBackfillService(resolver, photos), recordCaptureMetadata, getPhotoInfo }; + return { svc: new PhotoCaptureBackfillService(resolver, photos, storageStub), recordCaptureMetadata, getPhotoInfo }; } it('CAPTURE-007: reads a local file rather than asking a provider', async () => { @@ -199,7 +206,7 @@ describe('PhotoCaptureBackfillService — local files', () => { resolve: () => { throw new Error('db gone'); }, recordCaptureMetadata, } as unknown as TrekPhotosRepository; - const svc = new PhotoCaptureBackfillService({} as PhotoResolverService, photos); + const svc = new PhotoCaptureBackfillService({} as PhotoResolverService, photos, storageStub); await expect(svc.run([7], 1)).resolves.toBeUndefined(); expect(recordCaptureMetadata).not.toHaveBeenCalled(); diff --git a/server/tests/unit/nest/photo-resolver.service.test.ts b/server/tests/unit/nest/photo-resolver.service.test.ts index 9b443c413..8861b3296 100644 --- a/server/tests/unit/nest/photo-resolver.service.test.ts +++ b/server/tests/unit/nest/photo-resolver.service.test.ts @@ -63,6 +63,9 @@ const cache = { setInFlight: vi.fn(), put: vi.fn(), }; +// Local bytes go through the storage facade (slice 3); the resolver only ever +// addresses the 'journey' category with the 'journey/' prefix stripped. +const storage = { exists: vi.fn(), sendToResponse: vi.fn() }; const dbs = new DatabaseService(testDb); const repo = new TrekPhotosRepository(dbs); @@ -80,6 +83,7 @@ const svc = new PhotoResolverService( thumbnails as unknown as ThumbnailService, cache as unknown as TrekPhotoCacheService, providers, + storage as unknown as import('../../../src/nest/storage/storage.service').StorageService, ); function makeRes() { @@ -111,6 +115,8 @@ beforeEach(() => { cache.serveFresh.mockReturnValue(false); cache.getInFlight.mockReturnValue(undefined); cache.cacheKey.mockReturnValue('k'); + storage.exists.mockResolvedValue(false); + storage.sendToResponse.mockResolvedValue(undefined); }); afterAll(() => testDb.close()); @@ -168,6 +174,76 @@ describe('streamPhoto — dispatch', () => { expect(row).toEqual({ thumbnail_path: 'journey/thumbs/abc.jpg', width: 800, height: 600 }); }); + it('RESOLVE-014: a local thumbnail hit streams through storage with the immutable headers', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'journey/x.jpg', thumbnail_path: 'journey/thumbs/h.jpg' }); + storage.exists.mockResolvedValue(true); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'thumbnail'); + + expect(storage.exists).toHaveBeenCalledWith('journey', 'thumbs/h.jpg'); + expect(res.set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400, immutable'); + expect(res.set).toHaveBeenCalledWith('X-Content-Type-Options', 'nosniff'); + expect(storage.sendToResponse).toHaveBeenCalledWith('journey', 'thumbs/h.jpg', res); + }); + + it('RESOLVE-015: a local original hit streams through storage with the day-long cache header', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'journey/x.jpg' }); + storage.exists.mockResolvedValue(true); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'original'); + + expect(res.set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400'); + expect(res.set).toHaveBeenCalledWith('X-Content-Type-Options', 'nosniff'); + expect(storage.sendToResponse).toHaveBeenCalledWith('journey', 'x.jpg', res); + }); + + it('RESOLVE-016: a missing thumbnail falls through to the original for images', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'journey/x.jpg', thumbnail_path: 'journey/thumbs/h.jpg' }); + storage.exists.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'thumbnail'); + + expect(storage.sendToResponse).toHaveBeenCalledWith('journey', 'x.jpg', res); + }); + + it('RESOLVE-017: a video whose recorded poster is gone 404s instead of falling through (#823)', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'journey/clip.mp4', thumbnail_path: 'journey/thumbs/t.jpg', media_type: 'video' }); + storage.exists.mockResolvedValue(false); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'thumbnail'); + + expect(res.status).toHaveBeenCalledWith(404); + expect(res.json).toHaveBeenCalledWith({ error: 'No poster available' }); + expect(storage.sendToResponse).not.toHaveBeenCalled(); + }); + + it('RESOLVE-019: a rejecting exists check (invalid key) reads as a local miss for thumb and original', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'journey/x.jpg', thumbnail_path: 'journey/thumbs/h.jpg' }); + storage.exists.mockRejectedValue(new Error('invalid storage key')); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'thumbnail'); + + expect(res.status).toHaveBeenCalledWith(404); + expect(res.json).toHaveBeenCalledWith({ error: 'File not found' }); + expect(storage.sendToResponse).not.toHaveBeenCalled(); + }); + + it('RESOLVE-018: a file_path outside journey/ reads as a local miss without touching storage', async () => { + const id = insertPhoto({ provider: 'local', file_path: 'nope/other.jpg' }); + const res = makeRes(); + + await svc.streamPhoto(res as never, 1, id, 'original'); + + expect(res.status).toHaveBeenCalledWith(404); + expect(res.json).toHaveBeenCalledWith({ error: 'File not found' }); + expect(storage.exists).not.toHaveBeenCalled(); + }); + it('RESOLVE-006: an immich original goes straight to the provider with the range header', async () => { const id = insertPhoto({ provider: 'immich', asset_id: 'a1', owner_id: 5 }); const res = makeRes(); diff --git a/server/tests/unit/nest/place-enrichment.service.test.ts b/server/tests/unit/nest/place-enrichment.service.test.ts index 7754088a4..ba1e62555 100644 --- a/server/tests/unit/nest/place-enrichment.service.test.ts +++ b/server/tests/unit/nest/place-enrichment.service.test.ts @@ -668,17 +668,17 @@ describe('creditLine', () => { ); }); - it('ENRICH-029: reads a stored credit back by cache key', () => { + it('ENRICH-029: reads a stored credit back by cache key', async () => { const cache = cacheStub({ get: vi.fn((key: string) => - key === 'ChIJmuseum~p0' ? { photoUrl: '/x', filePath: '/tmp/x', attribution: 'Alice · CC BY-SA 4.0' } : null, + key === 'ChIJmuseum~p0' ? { photoUrl: '/x', attribution: 'Alice · CC BY-SA 4.0' } : null, ), }); const svc = make(mapsStub(), cache); - expect(svc.credit('ChIJmuseum~p0')).toEqual({ credit: 'Alice · CC BY-SA 4.0' }); + expect(await svc.credit('ChIJmuseum~p0')).toEqual({ credit: 'Alice · CC BY-SA 4.0' }); // An uploaded image or a key that was swept has nothing to name. - expect(svc.credit('unknown')).toEqual({ credit: null }); + expect(await svc.credit('unknown')).toEqual({ credit: null }); }); }); diff --git a/server/tests/unit/nest/place-photos/place-photo-cache.service.test.ts b/server/tests/unit/nest/place-photos/place-photo-cache.service.test.ts index 850a58b95..9414b21a0 100644 --- a/server/tests/unit/nest/place-photos/place-photo-cache.service.test.ts +++ b/server/tests/unit/nest/place-photos/place-photo-cache.service.test.ts @@ -1,31 +1,28 @@ /** - * Unit tests for placePhotoCache — PPC-001 through PPC-012. + * Unit tests for PlacePhotoCacheService — PPC-001 through PPC-014. * Covers the downscale guard in put(), removeIfUnreferenced(), sweepOrphans(), - * and the two negative-cache windows. - * Uses a real in-memory SQLite DB and a throwaway temp upload dir - * (TREK_PLACE_PHOTO_DIR) so the real uploads tree is never touched. + * the two negative-cache windows, and the serveKey/get miss handling. + * + * Slice 4: the cache is fully async and addresses storage as + * ('photos-google', '.jpg'). The whole suite runs in BOTH registry + * modes — mode A (TREK_PLACE_PHOTO_DIR unset: uploads-local backend, + * 'photos/google/' prefix) and mode B (dir set: place-photos-local backend, + * bare keys) — via the shared stub-registry fixture. The real mode flip is + * pinned by the storage-registry tests; here the two prefixes prove the cache + * itself is mode-agnostic. */ import { describe, it, expect, beforeAll, beforeEach, afterAll, vi } from 'vitest'; import path from 'node:path'; import fs from 'node:fs'; -import os from 'node:os'; import crypto from 'node:crypto'; import { Jimp, JimpMime } from 'jimp'; -import Database from 'better-sqlite3'; -// Throwaway upload dir, hoisted with the DB below: the service is a static -// import now, so vi.mock's factory runs before any top-level statement. -const { TMP_DIR, testDb } = vi.hoisted(() => { - const nodeFs = require('node:fs'); - const nodePath = require('node:path'); - const nodeOs = require('node:os'); +const { testDb } = vi.hoisted(() => { const Db = require('better-sqlite3'); - const dir = nodeFs.mkdtempSync(nodePath.join(nodeOs.tmpdir(), 'ppc-')); - process.env.TREK_PLACE_PHOTO_DIR = dir; - return { TMP_DIR: dir, testDb: new Db(':memory:') }; + return { testDb: new Db(':memory:') }; }); -// Minimal real DB with just the tables the cache touches. isReferenced now +// Minimal real DB with just the tables the cache touches. isReferenced // UNIONs collection_places (#1081 photo-cache fix), so the bare fixture must // declare it too or the reference check would throw "no such table". testDb.exec(` @@ -49,178 +46,238 @@ testDb.exec(` vi.mock('../../../../src/db/database', () => ({ db: testDb })); -function filePathFor(placeId: string): string { - const hash = crypto.createHash('sha1').update(placeId).digest('hex'); - return path.join(TMP_DIR, `${hash}.jpg`); -} +import { PlacePhotoCacheService } from '../../../../src/nest/place-photos/place-photo-cache.service'; +import { DatabaseService } from '../../../../src/nest/database/database.service'; +import { makeStorageFixture, type StorageFixture } from '../../../helpers/storage-fixture'; async function makeJpeg(width: number, height: number): Promise { const img = new Jimp({ width, height, color: 0xff0000ff }); return img.getBuffer(JimpMime.jpeg, { quality: 80 }); } -import { PlacePhotoCacheService } from '../../../../src/nest/place-photos/place-photo-cache.service'; -import { DatabaseService } from '../../../../src/nest/database/database.service'; -import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +function nameFor(placeId: string): string { + return `${crypto.createHash('sha1').update(placeId).digest('hex')}.jpg`; +} -let cache: PlacePhotoCacheService; +describe.each([ + ['mode A (photos/google/ prefix)', 'photos/google/'], + ['mode B (bare keys)', ''], +])('PlacePhotoCacheService — %s', (_label, keyPrefix) => { + let fx: StorageFixture; + let cache: PlacePhotoCacheService; -beforeAll(() => { - // onModuleInit does what the module-scope mkdirSync used to do on import. - cache = new PlacePhotoCacheService(new DatabaseService(testDb as never), new RuntimeEnvService()); - cache.onModuleInit(); -}); + const prefixDir = () => path.join(fx.root, ...keyPrefix.split('/').filter(Boolean)); + const filePathFor = (placeId: string) => path.join(prefixDir(), nameFor(placeId)); + const writeObject = (placeId: string, bytes: Buffer | string) => { + fs.mkdirSync(prefixDir(), { recursive: true }); + fs.writeFileSync(filePathFor(placeId), bytes); + }; -beforeEach(() => { - testDb.exec('DELETE FROM places; DELETE FROM google_place_photo_meta;'); - for (const f of fs.readdirSync(TMP_DIR)) fs.rmSync(path.join(TMP_DIR, f), { force: true }); -}); - -afterAll(() => { - testDb.close(); - fs.rmSync(TMP_DIR, { recursive: true, force: true }); -}); - -describe('placePhotoCache.put() downscale guard', () => { - it('PPC-001: downscales an oversized image to <= 800px', async () => { - const big = await makeJpeg(1600, 1200); - await cache.put('big-place', big, 'Alice'); - - const written = fs.readFileSync(filePathFor('big-place')); - const decoded = await Jimp.read(written); - expect(Math.max(decoded.bitmap.width, decoded.bitmap.height)).toBeLessThanOrEqual(800); - expect(written.length).toBeLessThan(big.length); + beforeAll(() => { + fx = makeStorageFixture(keyPrefix); + cache = new PlacePhotoCacheService(new DatabaseService(testDb as never), fx.storage); }); - it('PPC-002: passes a small image through unchanged', async () => { - const small = await makeJpeg(100, 100); - await cache.put('small-place', small, null); - - const written = fs.readFileSync(filePathFor('small-place')); - expect(written.equals(small)).toBe(true); - }); - - it('PPC-003: falls back to original bytes when the input is not a decodable image', async () => { - const garbage = Buffer.from('definitely not an image'); - await cache.put('garbage-place', garbage, null); - - const written = fs.readFileSync(filePathFor('garbage-place')); - expect(written.equals(garbage)).toBe(true); - }); -}); - -describe('placePhotoCache.removeIfUnreferenced()', () => { - it('PPC-004: removes a cache entry that no place references', async () => { - await cache.put('orphan', await makeJpeg(50, 50), null); - expect(fs.existsSync(filePathFor('orphan'))).toBe(true); - - cache.removeIfUnreferenced('orphan'); - - expect(fs.existsSync(filePathFor('orphan'))).toBe(false); - expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('orphan')).toBeUndefined(); - }); - - it('PPC-005: keeps an entry still referenced by google_place_id', async () => { - await cache.put('gid-1', await makeJpeg(50, 50), null); - testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('gid-1'); - - cache.removeIfUnreferenced('gid-1'); - - expect(fs.existsSync(filePathFor('gid-1'))).toBe(true); - }); - - it('PPC-006: keeps an entry referenced by a coords proxy URL in image_url', async () => { - const id = 'coords:48.8:2.3'; - await cache.put(id, await makeJpeg(50, 50), null); - const proxy = `/api/maps/place-photo/${encodeURIComponent(id)}/bytes`; - testDb.prepare('INSERT INTO places (image_url) VALUES (?)').run(proxy); - - cache.removeIfUnreferenced(id); - - expect(fs.existsSync(filePathFor(id))).toBe(true); - }); -}); - -describe('placePhotoCache.sweepOrphans()', () => { - it('PPC-007: removes orphaned meta rows + files, keeps referenced ones, deletes stray files', async () => { - await cache.put('keep-gid', await makeJpeg(50, 50), null); - await cache.put('drop-me', await makeJpeg(50, 50), null); - testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('keep-gid'); - - // A stray .jpg on disk with no meta row (e.g. a crash between write and upsert). - const strayPath = path.join(TMP_DIR, 'deadbeef'.padEnd(40, '0') + '.jpg'); - fs.writeFileSync(strayPath, 'stray'); - - const removed = cache.sweepOrphans(); - - expect(fs.existsSync(filePathFor('keep-gid'))).toBe(true); - expect(fs.existsSync(filePathFor('drop-me'))).toBe(false); - expect(fs.existsSync(strayPath)).toBe(false); - expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('drop-me')).toBeUndefined(); - expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('keep-gid')).toBeDefined(); - expect(removed).toBe(2); // drop-me (orphan meta+file) + stray file - }); - - it('PPC-008: returns 0 when every entry is referenced', async () => { - await cache.put('ref-a', await makeJpeg(50, 50), null); - testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('ref-a'); - - expect(cache.sweepOrphans()).toBe(0); - expect(fs.existsSync(filePathFor('ref-a'))).toBe(true); - }); -}); - -describe('placePhotoCache negative cache', () => { - function ageError(placeId: string, ms: number): void { - testDb - .prepare('UPDATE google_place_photo_meta SET error_at = ? WHERE place_id = ?') - .run(Date.now() - ms, placeId); - } - - it('PPC-009: a place with no photo stays remembered well past the old five-minute window', () => { - cache.markError('photo-less'); - - ageError('photo-less', 30 * 60 * 1000); - expect(cache.getErrored('photo-less')).toBe(true); - - ageError('photo-less', 5 * 60 * 60 * 1000); - expect(cache.getErrored('photo-less')).toBe(true); - }); - - it('PPC-010: the miss expires once it is a day old', () => { - cache.markError('stale-miss'); - ageError('stale-miss', 24 * 60 * 60 * 1000); - - expect(cache.getErrored('stale-miss')).toBe(false); - }); - - // A failed provider call says nothing about the place, so it must not inherit the - // long window a real "this place has no photo" answer gets. - it('PPC-011: a failed provider call is forgotten after minutes and never persisted', () => { - vi.useFakeTimers(); - try { - cache.markError('flaky', 'provider-error'); - expect(cache.getErrored('flaky')).toBe(true); - // Nothing on disk — a restart retries instead of inheriting someone's outage. - expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('flaky')).toBeUndefined(); - - vi.advanceTimersByTime(5 * 60 * 1000); - expect(cache.getErrored('flaky')).toBe(false); - - // The long window belongs to the other case: same age, still remembered. - cache.markError('photo-less-too'); - ageError('photo-less-too', 5 * 60 * 1000); - expect(cache.getErrored('photo-less-too')).toBe(true); - } finally { - vi.useRealTimers(); + beforeEach(() => { + testDb.exec('DELETE FROM places; DELETE FROM collection_places; DELETE FROM google_place_photo_meta;'); + for (const f of fs.readdirSync(fx.root)) { + if (f === '.tmp') continue; + fs.rmSync(path.join(fx.root, f), { recursive: true, force: true }); } }); - it('PPC-012: a cached photo clears an earlier failed attempt', async () => { - cache.markError('recovered', 'provider-error'); - expect(cache.getErrored('recovered')).toBe(true); + afterAll(() => { + fx.cleanup(); + }); - await cache.put('recovered', await makeJpeg(40, 40), null); - expect(cache.getErrored('recovered')).toBe(false); + describe('put() downscale guard', () => { + it('PPC-001: downscales an oversized image to <= 800px', async () => { + const big = await makeJpeg(1600, 1200); + await cache.put('big-place', big, 'Alice'); + + const written = fs.readFileSync(filePathFor('big-place')); + const decoded = await Jimp.read(written); + expect(Math.max(decoded.bitmap.width, decoded.bitmap.height)).toBeLessThanOrEqual(800); + expect(written.length).toBeLessThan(big.length); + }); + + it('PPC-002: passes a small image through unchanged', async () => { + const small = await makeJpeg(100, 100); + await cache.put('small-place', small, null); + + const written = fs.readFileSync(filePathFor('small-place')); + expect(written.equals(small)).toBe(true); + }); + + it('PPC-003: falls back to original bytes when the input is not a decodable image', async () => { + const garbage = Buffer.from('definitely not an image'); + await cache.put('garbage-place', garbage, null); + + const written = fs.readFileSync(filePathFor('garbage-place')); + expect(written.equals(garbage)).toBe(true); + }); + }); + + describe('serveKey()', () => { + it('returns the bare .jpg storage name when the photo object exists', async () => { + writeObject('p-key', await makeJpeg(10, 10)); + expect(await cache.serveKey('p-key')).toBe(nameFor('p-key')); + }); + + it('returns null when nothing is cached', async () => { + expect(await cache.serveKey('p-missing')).toBeNull(); + }); + }); + + describe('get()', () => { + it('PPC-013: purges the meta row and returns null when the object vanished', async () => { + testDb + .prepare('INSERT INTO google_place_photo_meta (place_id, attribution, fetched_at) VALUES (?, ?, ?)') + .run('gone-place', 'Bob', Date.now()); + + expect(await cache.get('gone-place')).toBeNull(); + expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('gone-place')).toBeUndefined(); + }); + + it('PPC-014: returns the proxy URL + attribution for a cached photo', async () => { + await cache.put('hit-place', await makeJpeg(20, 20), 'Carol'); + + const hit = await cache.get('hit-place'); + expect(hit).toEqual({ + photoUrl: `/api/maps/place-photo/${encodeURIComponent('hit-place')}/bytes`, + attribution: 'Carol', + }); + }); + }); + + describe('removeIfUnreferenced()', () => { + it('PPC-004: removes a cache entry that no place references', async () => { + await cache.put('orphan', await makeJpeg(50, 50), null); + expect(fs.existsSync(filePathFor('orphan'))).toBe(true); + + await cache.removeIfUnreferenced('orphan'); + + expect(fs.existsSync(filePathFor('orphan'))).toBe(false); + expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('orphan')).toBeUndefined(); + }); + + it('PPC-005: keeps an entry still referenced by google_place_id', async () => { + await cache.put('gid-1', await makeJpeg(50, 50), null); + testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('gid-1'); + + await cache.removeIfUnreferenced('gid-1'); + + expect(fs.existsSync(filePathFor('gid-1'))).toBe(true); + }); + + it('PPC-006: keeps an entry referenced by a coords proxy URL in image_url', async () => { + const id = 'coords:48.8:2.3'; + await cache.put(id, await makeJpeg(50, 50), null); + const proxy = `/api/maps/place-photo/${encodeURIComponent(id)}/bytes`; + testDb.prepare('INSERT INTO places (image_url) VALUES (?)').run(proxy); + + await cache.removeIfUnreferenced(id); + + expect(fs.existsSync(filePathFor(id))).toBe(true); + }); + }); + + describe('sweepOrphans()', () => { + it('PPC-007: removes orphaned meta rows + files, keeps referenced ones, deletes stray files', async () => { + await cache.put('keep-gid', await makeJpeg(50, 50), null); + await cache.put('drop-me', await makeJpeg(50, 50), null); + testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('keep-gid'); + + // A stray .jpg on disk with no meta row (e.g. a crash between write and upsert). + const strayPath = path.join(prefixDir(), 'deadbeef'.padEnd(40, '0') + '.jpg'); + fs.writeFileSync(strayPath, 'stray'); + + const removed = await cache.sweepOrphans(); + + expect(fs.existsSync(filePathFor('keep-gid'))).toBe(true); + expect(fs.existsSync(filePathFor('drop-me'))).toBe(false); + expect(fs.existsSync(strayPath)).toBe(false); + expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('drop-me')).toBeUndefined(); + expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('keep-gid')).toBeDefined(); + expect(removed).toBe(2); // drop-me (orphan meta+file) + stray file + }); + + it('PPC-008: returns 0 when every entry is referenced', async () => { + await cache.put('ref-a', await makeJpeg(50, 50), null); + testDb.prepare('INSERT INTO places (google_place_id) VALUES (?)').run('ref-a'); + + expect(await cache.sweepOrphans()).toBe(0); + expect(fs.existsSync(filePathFor('ref-a'))).toBe(true); + }); + + it('PPC-015: never touches nested entries or non-jpg files (readdir parity — list() recurses)', async () => { + const nestedDir = path.join(prefixDir(), 'sub'); + fs.mkdirSync(nestedDir, { recursive: true }); + const nested = path.join(nestedDir, 'nested.jpg'); + fs.writeFileSync(nested, 'nested'); + const note = path.join(prefixDir(), 'note.txt'); + fs.writeFileSync(note, 'not a photo'); + + const removed = await cache.sweepOrphans(); + + expect(fs.existsSync(nested)).toBe(true); + expect(fs.existsSync(note)).toBe(true); + expect(removed).toBe(0); + }); + }); + + describe('negative cache', () => { + function ageError(placeId: string, ms: number): void { + testDb + .prepare('UPDATE google_place_photo_meta SET error_at = ? WHERE place_id = ?') + .run(Date.now() - ms, placeId); + } + + it('PPC-009: a place with no photo stays remembered well past the old five-minute window', () => { + cache.markError('photo-less'); + + ageError('photo-less', 30 * 60 * 1000); + expect(cache.getErrored('photo-less')).toBe(true); + + ageError('photo-less', 5 * 60 * 60 * 1000); + expect(cache.getErrored('photo-less')).toBe(true); + }); + + it('PPC-010: the miss expires once it is a day old', () => { + cache.markError('stale-miss'); + ageError('stale-miss', 24 * 60 * 60 * 1000); + + expect(cache.getErrored('stale-miss')).toBe(false); + }); + + // A failed provider call says nothing about the place, so it must not inherit the + // long window a real "this place has no photo" answer gets. + it('PPC-011: a failed provider call is forgotten after minutes and never persisted', () => { + vi.useFakeTimers(); + try { + cache.markError('flaky', 'provider-error'); + expect(cache.getErrored('flaky')).toBe(true); + // Nothing on disk — a restart retries instead of inheriting someone's outage. + expect(testDb.prepare('SELECT 1 FROM google_place_photo_meta WHERE place_id = ?').get('flaky')).toBeUndefined(); + + vi.advanceTimersByTime(5 * 60 * 1000); + expect(cache.getErrored('flaky')).toBe(false); + + // The long window belongs to the other case: same age, still remembered. + cache.markError('photo-less-too'); + ageError('photo-less-too', 5 * 60 * 1000); + expect(cache.getErrored('photo-less-too')).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + + it('PPC-012: a cached photo clears an earlier failed attempt', async () => { + cache.markError('recovered', 'provider-error'); + expect(cache.getErrored('recovered')).toBe(true); + + await cache.put('recovered', await makeJpeg(40, 40), null); + expect(cache.getErrored('recovered')).toBe(false); + }); }); }); diff --git a/server/tests/unit/nest/places.controller.test.ts b/server/tests/unit/nest/places.controller.test.ts index 71cfcae9b..5a86e25bc 100644 --- a/server/tests/unit/nest/places.controller.test.ts +++ b/server/tests/unit/nest/places.controller.test.ts @@ -3,8 +3,11 @@ import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.serv import { HttpException } from '@nestjs/common'; import { PlacesController } from '../../../src/nest/places/places.controller'; import type { PlacesService } from '../../../src/nest/places/places.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as StorageService; + const user = { id: 1, role: 'user', email: 'u@example.test' } as User; const trip = { user_id: 1 }; @@ -45,14 +48,14 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou // by calling the method directly. it('GET / lists with filters', () => { const list = vi.fn().mockReturnValue([{ id: 1 }]); - expect(new PlacesController(svc({ list } as Partial), new RuntimeEnvService()).list(user, '5', 'beach', 'cat', 'tag')).toEqual({ places: [{ id: 1 }] }); + expect(new PlacesController(svc({ list } as Partial), new RuntimeEnvService(), storageStub).list(user, '5', 'beach', 'cat', 'tag')).toEqual({ places: [{ id: 1 }] }); expect(list).toHaveBeenCalledWith('5', { search: 'beach', category: 'cat', tag: 'tag' }); }); describe('POST / (create)', () => { it('400 on an over-long name (length guard before permission)', () => { const canEdit = vi.fn().mockReturnValue(false); // would 403 if reached - expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService()).create(user, '5', { name: 'x'.repeat(201) }))).toEqual({ + expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService(), storageStub).create(user, '5', { name: 'x'.repeat(201) }))).toEqual({ status: 400, body: { error: 'name must be 200 characters or less' }, }); expect(canEdit).not.toHaveBeenCalled(); @@ -62,10 +65,10 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou // pins `name`, so the ZodValidationPipe rejects a nameless body before the // handler runs (see the e2e suite for the envelope it produces). it('403 without place_edit, then creates + hooks', () => { - expect(thrown(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).create(user, '5', { name: 'Spot' }))).toEqual({ status: 403, body: { error: 'No permission' } }); + expect(thrown(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService(), storageStub).create(user, '5', { name: 'Spot' }))).toEqual({ status: 403, body: { error: 'No permission' } }); const create = vi.fn().mockReturnValue({ id: 9 }); const broadcast = vi.fn(); const onCreated = vi.fn(); const s = svc({ create, broadcast, onCreated } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).create(user, '5', { name: 'Spot' }, 'sock')).toEqual({ place: { id: 9 } }); + expect(new PlacesController(s, new RuntimeEnvService(), storageStub).create(user, '5', { name: 'Spot' }, 'sock')).toEqual({ place: { id: 9 } }); expect(broadcast).toHaveBeenCalledWith('5', 'place:created', { place: { id: 9 } }, 'sock'); expect(onCreated).toHaveBeenCalledWith('5', 9); }); @@ -74,22 +77,22 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou describe('POST /import/gpx', () => { const file = { buffer: Buffer.from('gpx'), originalname: 'r.gpx' } as Express.Multer.File; it('400 without a file', () => { - expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService()).importGpx(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); + expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).importGpx(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); }); it('400 when all import types are disabled', () => { - expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService()).importGpx(user, '5', file, { importWaypoints: 'false', importRoutes: 'false', importTracks: 'false' }))).toEqual({ + expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).importGpx(user, '5', file, { importWaypoints: 'false', importRoutes: 'false', importTracks: 'false' }))).toEqual({ status: 400, body: { error: 'No import types selected' }, }); }); it('400 when the GPX yields nothing', () => { - expect(thrown(() => new PlacesController(svc({ importGpx: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService()).importGpx(user, '5', file, {}))).toEqual({ + expect(thrown(() => new PlacesController(svc({ importGpx: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService(), storageStub).importGpx(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'No matching places found in GPX file' }, }); }); it('imports and broadcasts per place', () => { const broadcast = vi.fn(); const s = svc({ importGpx: vi.fn().mockReturnValue({ places: [{ id: 1 }, { id: 2 }], count: 2, skipped: 0 }), broadcast } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).importGpx(user, '5', file, {}, 'sock')).toEqual({ places: [{ id: 1 }, { id: 2 }], count: 2, skipped: 0 }); + expect(new PlacesController(s, new RuntimeEnvService(), storageStub).importGpx(user, '5', file, {}, 'sock')).toEqual({ places: [{ id: 1 }, { id: 2 }], count: 2, skipped: 0 }); expect(broadcast).toHaveBeenCalledTimes(2); }); }); @@ -97,23 +100,23 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou describe('POST /import/map', () => { const file = { buffer: Buffer.from(''), originalname: 'm.kml' } as Express.Multer.File; it('400 without a file', async () => { - expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService()).importMap(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); + expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).importMap(user, '5', undefined, {}))).toEqual({ status: 400, body: { error: 'No file uploaded' } }); }); it('403 without place_edit (permission runs before the file check)', async () => { const importMapFile = vi.fn(); const s = svc({ canEdit: vi.fn().mockReturnValue(false), importMapFile } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'No permission' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}))).toEqual({ status: 403, body: { error: 'No permission' } }); expect(importMapFile).not.toHaveBeenCalled(); }); it('400 when both import types are disabled', async () => { - expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService()).importMap(user, '5', file, { importPoints: 'false', importPaths: 'false' }))).toEqual({ + expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).importMap(user, '5', file, { importPoints: 'false', importPaths: 'false' }))).toEqual({ status: 400, body: { error: 'No import types selected' }, }); }); it('400 when the map file has no Placemarks (and carries the summary through)', async () => { const summary = { totalPlacemarks: 0 }; const s = svc({ importMapFile: vi.fn().mockResolvedValue({ places: [], summary }) } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}))).toEqual({ + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'No valid Placemarks found in map file', summary }, }); }); @@ -121,26 +124,26 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou const broadcast = vi.fn(); const result = { places: [{ id: 1 }, { id: 2 }], summary: { totalPlacemarks: 2 }, count: 2 }; const s = svc({ importMapFile: vi.fn().mockResolvedValue(result), broadcast } as Partial); - expect(await new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}, 'sock')).toEqual(result); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}, 'sock')).toEqual(result); expect(broadcast).toHaveBeenCalledTimes(2); expect(broadcast).toHaveBeenCalledWith('5', 'place:created', { place: { id: 1 } }, 'sock'); }); it('passes a missing summary through (no zero-placemark guard) and still imports', async () => { const result = { places: [{ id: 7 }] }; const s = svc({ importMapFile: vi.fn().mockResolvedValue(result), broadcast: vi.fn() } as Partial); - expect(await new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {})).toEqual(result); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {})).toEqual(result); }); it('wraps a thrown Error from the service in a 400 with its message', async () => { const s = svc({ importMapFile: vi.fn().mockRejectedValue(new Error('bad kml')) } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'bad kml' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'bad kml' } }); }); it('falls back to a generic 400 message for a non-Error rejection', async () => { const s = svc({ importMapFile: vi.fn().mockRejectedValue('boom') } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'Failed to import map file' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}))).toEqual({ status: 400, body: { error: 'Failed to import map file' } }); }); it('re-throws an HttpException raised inside the try untouched', async () => { const s = svc({ importMapFile: vi.fn().mockRejectedValue(new HttpException({ error: 'teapot' }, 418)) } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importMap(user, '5', file, {}))).toEqual({ status: 418, body: { error: 'teapot' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importMap(user, '5', file, {}))).toEqual({ status: 418, body: { error: 'teapot' } }); }); }); @@ -150,29 +153,29 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou // handler runs. it('maps a service { error, status } to the same response', async () => { const s = svc({ importGoogleList: vi.fn().mockResolvedValue({ error: 'List is empty', status: 400 }) } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importGoogle(user, '5', { url: 'http://x' }))).toEqual({ status: 400, body: { error: 'List is empty' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importGoogle(user, '5', { url: 'http://x' }))).toEqual({ status: 400, body: { error: 'List is empty' } }); }); it('imports a naver list and returns the count + listName', async () => { const s = svc({ importNaverList: vi.fn().mockResolvedValue({ places: [{ id: 1 }], listName: 'Trip', skipped: 2 }), broadcast: vi.fn() } as Partial); - expect(await new PlacesController(s, new RuntimeEnvService()).importNaver(user, '5', { url: 'http://x' })).toEqual({ places: [{ id: 1 }], count: 1, listName: 'Trip', skipped: 2 }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).importNaver(user, '5', { url: 'http://x' })).toEqual({ places: [{ id: 1 }], count: 1, listName: 'Trip', skipped: 2 }); }); it('forwards the enrich flag + userId and broadcasts each imported place', async () => { const importGoogleList = vi.fn().mockResolvedValue({ places: [{ id: 1 }, { id: 2 }], listName: 'L', skipped: 0 }); const broadcast = vi.fn(); const s = svc({ importGoogleList, broadcast } as Partial); - expect(await new PlacesController(s, new RuntimeEnvService()).importGoogle(user, '5', { url: 'http://x', enrich: true }, 'sock')).toEqual({ places: [{ id: 1 }, { id: 2 }], count: 2, listName: 'L', skipped: 0 }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).importGoogle(user, '5', { url: 'http://x', enrich: true }, 'sock')).toEqual({ places: [{ id: 1 }, { id: 2 }], count: 2, listName: 'L', skipped: 0 }); expect(importGoogleList).toHaveBeenCalledWith('5', 'http://x', { enrich: true, userId: 1 }); expect(broadcast).toHaveBeenCalledTimes(2); }); it('wraps a thrown Error in the provider-specific 400 (Google)', async () => { const s = svc({ importGoogleList: vi.fn().mockRejectedValue(new Error('network down')) } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importGoogle(user, '5', { url: 'http://x' }))).toEqual({ + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importGoogle(user, '5', { url: 'http://x' }))).toEqual({ status: 400, body: { error: 'Failed to import Google Maps list. Make sure the list is shared publicly.' }, }); }); it('wraps a non-Error rejection in the provider-specific 400 (Naver)', async () => { const s = svc({ importNaverList: vi.fn().mockRejectedValue('weird') } as Partial); - expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService()).importNaver(user, '5', { url: 'http://x' }))).toEqual({ + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).importNaver(user, '5', { url: 'http://x' }))).toEqual({ status: 400, body: { error: 'Failed to import Naver Maps list. Make sure the list is shared publicly.' }, }); }); @@ -182,28 +185,28 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou // The legacy 'ids must be an array of numbers' 400 is gone: // placeBulkDeleteRequestSchema types the array, so the pipe rejects a bad // element before the handler runs. - it('returns empty for an empty list without touching the service', () => { + it('returns empty for an empty list without touching the service', async () => { const removeMany = vi.fn(); - expect(new PlacesController(svc({ removeMany } as Partial), new RuntimeEnvService()).bulkDelete(user, '5', { ids: [] })).toEqual({ deleted: [], count: 0 }); + expect(await new PlacesController(svc({ removeMany } as Partial), new RuntimeEnvService(), storageStub).bulkDelete(user, '5', { ids: [] })).toEqual({ deleted: [], count: 0 }); expect(removeMany).not.toHaveBeenCalled(); }); - it('deletes, fires hooks + broadcasts per deleted id', () => { + it('deletes, fires hooks + broadcasts per deleted id', async () => { const removeMany = vi.fn().mockReturnValue([1, 2]); const onDeleted = vi.fn(); const broadcast = vi.fn(); const scopedIds = vi.fn().mockReturnValue([1, 2]); const s = svc({ removeMany, onDeleted, broadcast, scopedIds } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).bulkDelete(user, '5', { ids: [1, 2] }, 'sock')).toEqual({ deleted: [1, 2], count: 2 }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).bulkDelete(user, '5', { ids: [1, 2] }, 'sock')).toEqual({ deleted: [1, 2], count: 2 }); expect(onDeleted).toHaveBeenCalledTimes(2); expect(broadcast).toHaveBeenCalledTimes(2); }); // #1745: the hook keys on the place id alone, so an id from another trip // would detach that trip's journey entries even though removeMany skips it. - it('fires the journey hook only for ids that belong to the trip, ahead of the delete', () => { + it('fires the journey hook only for ids that belong to the trip, ahead of the delete', async () => { const removeMany = vi.fn().mockReturnValue([1]); const scopedIds = vi.fn().mockReturnValue([1]); const onDeleted = vi.fn(); const s = svc({ removeMany, scopedIds, onDeleted, broadcast: vi.fn() } as Partial); - new PlacesController(s, new RuntimeEnvService()).bulkDelete(user, '5', { ids: [1, 99] }); + await new PlacesController(s, new RuntimeEnvService(), storageStub).bulkDelete(user, '5', { ids: [1, 99] }); expect(scopedIds).toHaveBeenCalledWith('5', [1, 99]); expect(onDeleted).toHaveBeenCalledTimes(1); expect(onDeleted).toHaveBeenCalledWith(1); @@ -211,12 +214,12 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou }); // #1298: the link is gone once the place is, so the ids have to be read first. - it('announces the expenses the deleted places took with them', () => { + it('announces the expenses the deleted places took with them', async () => { const removeMany = vi.fn().mockReturnValue([1, 2]); const linkedExpenseIds = vi.fn().mockReturnValue([77]); const broadcast = vi.fn(); const s = svc({ removeMany, linkedExpenseIds, broadcast } as Partial); - new PlacesController(s, new RuntimeEnvService()).bulkDelete(user, '5', { ids: [1, 2] }, 'sock'); + await new PlacesController(s, new RuntimeEnvService(), storageStub).bulkDelete(user, '5', { ids: [1, 2] }, 'sock'); expect(linkedExpenseIds).toHaveBeenCalledWith('5', [1, 2]); expect(linkedExpenseIds.mock.invocationCallOrder[0]).toBeLessThan(removeMany.mock.invocationCallOrder[0]); @@ -225,79 +228,79 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou }); describe('POST /bulk-update', () => { - it('404 when trip not accessible, 403 without place_edit (before any write)', () => { - expect(thrown(() => new PlacesController(svc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }), new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [1], category_id: 3 }))).toEqual({ status: 404, body: { error: 'Trip not found' } }); - expect(thrown(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [1], category_id: 3 }))).toEqual({ status: 403, body: { error: 'No permission' } }); + it('404 when trip not accessible, 403 without place_edit (before any write)', async () => { + expect(await thrownAsync(() => new PlacesController(svc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }), new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [1], category_id: 3 }))).toEqual({ status: 404, body: { error: 'Trip not found' } }); + expect(await thrownAsync(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [1], category_id: 3 }))).toEqual({ status: 403, body: { error: 'No permission' } }); }); // Same as bulk-delete: placeBulkUpdateRequestSchema types `ids`, so the // pipe owns that 400 now. `.min(1)` was deliberately left off the schema so // the empty-list short-circuit below stays reachable. - it('400 when no patch field is present', () => { - expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [1] }))).toEqual({ status: 400, body: { error: 'Provide at least one field to update' } }); + it('400 when no patch field is present', async () => { + expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [1] }))).toEqual({ status: 400, body: { error: 'Provide at least one field to update' } }); }); - it('returns empty for an empty list without touching the service', () => { + it('returns empty for an empty list without touching the service', async () => { const updateMany = vi.fn(); - expect(new PlacesController(svc({ updateMany } as Partial), new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [] })).toEqual({ updated: [], count: 0 }); + expect(await new PlacesController(svc({ updateMany } as Partial), new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [] })).toEqual({ updated: [], count: 0 }); expect(updateMany).not.toHaveBeenCalled(); }); - it('updates, fires hooks + broadcasts per updated place', () => { + it('updates, fires hooks + broadcasts per updated place', async () => { const updateMany = vi.fn().mockReturnValue([{ id: 1 }, { id: 2 }]); const onUpdated = vi.fn(); const broadcast = vi.fn(); const s = svc({ updateMany, onUpdated, broadcast } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [1, 2], category_id: 3 }, 'sock')).toEqual({ updated: [1, 2], count: 2 }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [1, 2], category_id: 3 }, 'sock')).toEqual({ updated: [1, 2], count: 2 }); expect(updateMany).toHaveBeenCalledWith('5', [1, 2], { category_id: 3 }); expect(onUpdated).toHaveBeenCalledTimes(2); expect(broadcast).toHaveBeenCalledWith('5', 'place:updated', { place: { id: 1 } }, 'sock'); }); - it('passes category_id: null through to clear the category', () => { + it('passes category_id: null through to clear the category', async () => { const updateMany = vi.fn().mockReturnValue([{ id: 1 }]); const s = svc({ updateMany } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).bulkUpdate(user, '5', { ids: [1], category_id: null })).toEqual({ updated: [1], count: 1 }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).bulkUpdate(user, '5', { ids: [1], category_id: null })).toEqual({ updated: [1], count: 1 }); expect(updateMany).toHaveBeenCalledWith('5', [1], { category_id: null }); }); }); it('GET /:id returns the place when found, 404 when missing', () => { - expect(thrown(() => new PlacesController(svc({ get: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService()).get(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Place not found' } }); + expect(thrown(() => new PlacesController(svc({ get: vi.fn().mockReturnValue(undefined) } as Partial), new RuntimeEnvService(), storageStub).get(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Place not found' } }); const s = svc({ get: vi.fn().mockReturnValue({ id: 9 }) } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).get(user, '5', '9')).toEqual({ place: { id: 9 } }); + expect(new PlacesController(s, new RuntimeEnvService(), storageStub).get(user, '5', '9')).toEqual({ place: { id: 9 } }); }); - it('PUT /:id 404 when missing, else updates + hooks', () => { - expect(thrown(() => new PlacesController(svc({ update: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService()).update(user, '5', '9', { name: 'X' }))).toEqual({ status: 404, body: { error: 'Place not found' } }); + it('PUT /:id 404 when missing, else updates + hooks', async () => { + expect(await thrownAsync(() => new PlacesController(svc({ update: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService(), storageStub).update(user, '5', '9', { name: 'X' }))).toEqual({ status: 404, body: { error: 'Place not found' } }); const update = vi.fn().mockReturnValue({ id: 9 }); const onUpdated = vi.fn(); const broadcast = vi.fn(); const s = svc({ update, onUpdated, broadcast } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).update(user, '5', '9', { name: 'X' }, 'sock')).toEqual({ place: { id: 9 } }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).update(user, '5', '9', { name: 'X' }, 'sock')).toEqual({ place: { id: 9 } }); expect(onUpdated).toHaveBeenCalledWith(9); }); describe('route_color (#776)', () => { - it('400s on anything that is not a hex colour, before the permission check', () => { + it('400s on anything that is not a hex colour, before the permission check', async () => { const canEdit = vi.fn().mockReturnValue(false); // would 403 if it got that far const err = { status: 400, body: { error: 'route_color must be a hex colour like #4f46e5' } }; - expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService()).update(user, '5', '9', { route_color: 'red' }))).toEqual(err); - expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService()).update(user, '5', '9', { route_color: '#12345' }))).toEqual(err); - expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService()).update(user, '5', '9', { route_color: 123 }))).toEqual(err); - expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService()).create(user, '5', { name: 'T', route_color: 'red' }))).toEqual(err); + expect(await thrownAsync(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: 'red' }))).toEqual(err); + expect(await thrownAsync(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: '#12345' }))).toEqual(err); + expect(await thrownAsync(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: 123 }))).toEqual(err); + expect(thrown(() => new PlacesController(svc({ canEdit }), new RuntimeEnvService(), storageStub).create(user, '5', { name: 'T', route_color: 'red' }))).toEqual(err); expect(canEdit).not.toHaveBeenCalled(); }); - it('passes a valid hex through, and null through as the reset to auto', () => { + it('passes a valid hex through, and null through as the reset to auto', async () => { const update = vi.fn().mockReturnValue({ id: 9, route_color: '#e11d48' }); const broadcast = vi.fn(); const s = svc({ update, broadcast } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).update(user, '5', '9', { route_color: '#e11d48' }, 'sock')) + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: '#e11d48' }, 'sock')) .toEqual({ place: { id: 9, route_color: '#e11d48' } }); expect(update).toHaveBeenCalledWith('5', '9', expect.objectContaining({ route_color: '#e11d48' }), undefined); // The colour has to reach the other members too, not just the DB. expect(broadcast).toHaveBeenCalledWith('5', 'place:updated', { place: { id: 9, route_color: '#e11d48' } }, 'sock'); - new PlacesController(svc({ update } as Partial), new RuntimeEnvService()).update(user, '5', '9', { route_color: null }); + await new PlacesController(svc({ update } as Partial), new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: null }); expect(update).toHaveBeenLastCalledWith('5', '9', expect.objectContaining({ route_color: null }), undefined); }); - it('accepts the short #abc form', () => { + it('accepts the short #abc form', async () => { const update = vi.fn().mockReturnValue({ id: 9 }); - expect(new PlacesController(svc({ update } as Partial), new RuntimeEnvService()).update(user, '5', '9', { route_color: '#abc' })).toEqual({ place: { id: 9 } }); + expect(await new PlacesController(svc({ update } as Partial), new RuntimeEnvService(), storageStub).update(user, '5', '9', { route_color: '#abc' })).toEqual({ place: { id: 9 } }); }); }); @@ -305,12 +308,13 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou // URL — the thumbnail into hand-built marker HTML, the homepage into // window.open. The write body is an open record, so the Zod pipe never sees // either one; they are checked here for the same reason route_color is. + // (update() is async since the storage slices; create() is not.) describe('image_url and website', () => { const imageErr = { status: 400, body: { error: 'image_url must be an uploaded path, a photo-proxy path, an inline image or an https URL' } }; const siteErr = { status: 400, body: { error: 'website must be an http or https URL' } }; - const ctl = (over: Partial = {}) => new PlacesController(svc(over), new RuntimeEnvService()); + const ctl = (over: Partial = {}) => new PlacesController(svc(over), new RuntimeEnvService(), storageStub); - it('400s an image_url the marker builders were never meant to receive, before the permission check', () => { + it('400s an image_url the marker builders were never meant to receive, before the permission check', async () => { const canEdit = vi.fn().mockReturnValue(false); // would 403 if it got that far for (const image_url of [ 'javascript:alert(1)', @@ -319,13 +323,13 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou '//evil.example/p.png', 123, ]) { - expect(thrown(() => ctl({ canEdit }).update(user, '5', '9', { image_url }))).toEqual(imageErr); + expect(await thrownAsync(() => ctl({ canEdit }).update(user, '5', '9', { image_url }))).toEqual(imageErr); } expect(thrown(() => ctl({ canEdit }).create(user, '5', { name: 'T', image_url: 'javascript:alert(1)' }))).toEqual(imageErr); expect(canEdit).not.toHaveBeenCalled(); }); - it('keeps accepting every shape the app actually stores', () => { + it('keeps accepting every shape the app actually stores', async () => { const update = vi.fn().mockReturnValue({ id: 9 }); for (const image_url of [ '/uploads/places/eiffel.jpg', @@ -334,31 +338,31 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou 'https://images.example/photo.jpg', null, ]) { - expect(ctl({ update } as Partial).update(user, '5', '9', { image_url })).toEqual({ place: { id: 9 } }); + expect(await ctl({ update } as Partial).update(user, '5', '9', { image_url })).toEqual({ place: { id: 9 } }); } }); - it('400s a website that window.open would not treat as a page', () => { + it('400s a website that window.open would not treat as a page', async () => { const canEdit = vi.fn().mockReturnValue(false); for (const website of ['javascript:fetch("/api/trips")', 'data:text/html,x', 'louvre.fr', 42]) { - expect(thrown(() => ctl({ canEdit }).update(user, '5', '9', { website }))).toEqual(siteErr); + expect(await thrownAsync(() => ctl({ canEdit }).update(user, '5', '9', { website }))).toEqual(siteErr); } expect(canEdit).not.toHaveBeenCalled(); }); - it('accepts http and https, and treats the empty string as clearing the field', () => { + it('accepts http and https, and treats the empty string as clearing the field', async () => { const update = vi.fn().mockReturnValue({ id: 9 }); for (const website of ['https://louvre.fr', 'http://pension.at', '', null]) { - expect(ctl({ update } as Partial).update(user, '5', '9', { website })).toEqual({ place: { id: 9 } }); + expect(await ctl({ update } as Partial).update(user, '5', '9', { website })).toEqual({ place: { id: 9 } }); } }); }); - it('PUT /:id forwards the base-version token and 409s on a conflict (#1135)', () => { + it('PUT /:id forwards the base-version token and 409s on a conflict (#1135)', async () => { const update = vi.fn().mockReturnValue({ conflict: true, server: { id: 9, name: 'Theirs' } }); const onUpdated = vi.fn(); const broadcast = vi.fn(); const s = svc({ update, onUpdated, broadcast } as Partial); - expect(thrown(() => new PlacesController(s, new RuntimeEnvService()).update(user, '5', '9', { name: 'Mine' }, 'sock', '2026-01-01 00:00:00'))).toEqual({ + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).update(user, '5', '9', { name: 'Mine' }, 'sock', '2026-01-01 00:00:00'))).toEqual({ status: 409, body: { error: 'conflict', server: { id: 9, name: 'Theirs' } }, }); expect(update).toHaveBeenCalledWith('5', '9', expect.objectContaining({ name: 'Mine' }), '2026-01-01 00:00:00'); @@ -366,72 +370,73 @@ describe('PlacesController (parity with the legacy /api/trips/:tripId/places rou expect(onUpdated).not.toHaveBeenCalled(); }); - it('DELETE /:id fires the hook then 404 / success', () => { + it('DELETE /:id fires the hook then 404 / success', async () => { const onDeleted = vi.fn(); const remove = vi.fn().mockReturnValue(false); - expect(thrown(() => new PlacesController(svc({ remove, onDeleted } as Partial), new RuntimeEnvService()).remove(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Place not found' } }); + expect(await thrownAsync(() => new PlacesController(svc({ remove, onDeleted } as Partial), new RuntimeEnvService(), storageStub).remove(user, '5', '9'))).toEqual({ status: 404, body: { error: 'Place not found' } }); expect(onDeleted).toHaveBeenCalledWith(9); expect(onDeleted.mock.invocationCallOrder[0]).toBeLessThan(remove.mock.invocationCallOrder[0]); const s = svc({ remove: vi.fn().mockReturnValue(true), broadcast: vi.fn() } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).remove(user, '5', '9')).toEqual({ success: true }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).remove(user, '5', '9')).toEqual({ success: true }); }); // #1745: a place on another trip must 404 without the hook ever running — // onPlaceDeleted keys on the place id alone, so it would detach that trip's // journey entries. - it('DELETE /:id 404s a foreign place before the journey hook runs', () => { + it('DELETE /:id 404s a foreign place before the journey hook runs', async () => { const onDeleted = vi.fn(); const remove = vi.fn(); const s = svc({ get: vi.fn().mockReturnValue(null), onDeleted, remove } as Partial); - expect(thrown(() => new PlacesController(s, new RuntimeEnvService()).remove(user, '5', '99'))).toEqual({ status: 404, body: { error: 'Place not found' } }); + expect(await thrownAsync(() => new PlacesController(s, new RuntimeEnvService(), storageStub).remove(user, '5', '99'))).toEqual({ status: 404, body: { error: 'Place not found' } }); expect(onDeleted).not.toHaveBeenCalled(); expect(remove).not.toHaveBeenCalled(); }); it('GET /:id/image maps service error + returns photos', async () => { const s = svc({ searchImage: vi.fn().mockResolvedValue({ photos: [{ url: 'x' }] }) } as Partial); - expect(await new PlacesController(s, new RuntimeEnvService()).image(user, '5', '9')).toEqual({ photos: [{ url: 'x' }] }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).image(user, '5', '9')).toEqual({ photos: [{ url: 'x' }] }); const e = svc({ searchImage: vi.fn().mockResolvedValue({ error: 'No key', status: 400 }) } as Partial); - expect(await thrownAsync(() => new PlacesController(e, new RuntimeEnvService()).image(user, '5', '9'))).toEqual({ status: 400, body: { error: 'No key' } }); + expect(await thrownAsync(() => new PlacesController(e, new RuntimeEnvService(), storageStub).image(user, '5', '9'))).toEqual({ status: 400, body: { error: 'No key' } }); }); it('GET /:id/image turns an unexpected throw into a 500, but re-throws an HttpException as-is', async () => { const boom = svc({ searchImage: vi.fn().mockRejectedValue(new Error('Unsplash down')) } as Partial); - expect(await thrownAsync(() => new PlacesController(boom, new RuntimeEnvService()).image(user, '5', '9'))).toEqual({ status: 500, body: { error: 'Error searching for image' } }); + expect(await thrownAsync(() => new PlacesController(boom, new RuntimeEnvService(), storageStub).image(user, '5', '9'))).toEqual({ status: 500, body: { error: 'Error searching for image' } }); const http = svc({ searchImage: vi.fn().mockRejectedValue(new HttpException({ error: 'rate limited' }, 429)) } as Partial); - expect(await thrownAsync(() => new PlacesController(http, new RuntimeEnvService()).image(user, '5', '9'))).toEqual({ status: 429, body: { error: 'rate limited' } }); + expect(await thrownAsync(() => new PlacesController(http, new RuntimeEnvService(), storageStub).image(user, '5', '9'))).toEqual({ status: 429, body: { error: 'rate limited' } }); }); describe('POST /:id/image (custom place image #1136)', () => { const file = { filename: 'abc.jpg' } as Express.Multer.File; - it('404 when the trip is not accessible, 403 without place_edit', () => { - expect(thrown(() => new PlacesController(svc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }), new RuntimeEnvService()).uploadImage(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); - expect(thrown(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService()).uploadImage(user, '5', '9', file))).toEqual({ status: 403, body: { error: 'No permission' } }); + it('404 when the trip is not accessible, 403 without place_edit', async () => { + expect(await thrownAsync(() => new PlacesController(svc({ verifyTripAccess: vi.fn().mockReturnValue(undefined) }), new RuntimeEnvService(), storageStub).uploadImage(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); + expect(await thrownAsync(() => new PlacesController(svc({ canEdit: vi.fn().mockReturnValue(false) }), new RuntimeEnvService(), storageStub).uploadImage(user, '5', '9', file))).toEqual({ status: 403, body: { error: 'No permission' } }); }); - it('400 when no file was uploaded', () => { - expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService()).uploadImage(user, '5', '9', undefined))).toEqual({ status: 400, body: { error: 'No image uploaded' } }); + it('400 when no file was uploaded', async () => { + expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).uploadImage(user, '5', '9', undefined))).toEqual({ status: 400, body: { error: 'No image uploaded' } }); }); - it('404 when the place is missing (service returns null)', () => { - expect(thrown(() => new PlacesController(svc({ update: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService()).uploadImage(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Place not found' } }); + it('404 when the place is missing (service returns null)', async () => { + expect(await thrownAsync(() => new PlacesController(svc({ update: vi.fn().mockReturnValue(null) } as Partial), new RuntimeEnvService(), storageStub).uploadImage(user, '5', '9', file))).toEqual({ status: 404, body: { error: 'Place not found' } }); }); - it('stores the uploaded file as image_url, broadcasts + fires the update hook', () => { + it('stores the uploaded file as image_url, broadcasts + fires the update hook', async () => { const update = vi.fn().mockReturnValue({ id: 9 }); const onUpdated = vi.fn(); const broadcast = vi.fn(); const s = svc({ update, onUpdated, broadcast } as Partial); - expect(new PlacesController(s, new RuntimeEnvService()).uploadImage(user, '5', '9', file, 'sock')).toEqual({ place: { id: 9 } }); + expect(await new PlacesController(s, new RuntimeEnvService(), storageStub).uploadImage(user, '5', '9', file, 'sock')).toEqual({ place: { id: 9 } }); + expect(storageStub.put).toHaveBeenCalledWith('places', 'abc.jpg', { tmpPath: undefined }); expect(update).toHaveBeenCalledWith('5', '9', { image_url: '/uploads/places/abc.jpg' }); expect(broadcast).toHaveBeenCalledWith('5', 'place:updated', { place: { id: 9 } }, 'sock'); expect(onUpdated).toHaveBeenCalledWith(9); }); - it('403 in demo mode for a demo account', () => { + it('403 in demo mode for a demo account', async () => { const prev = process.env.DEMO_MODE; process.env.DEMO_MODE = 'true'; const demo = { ...user, email: 'demo@trek.app' } as User; try { - expect(thrown(() => new PlacesController(svc(), new RuntimeEnvService()).uploadImage(demo, '5', '9', file))).toEqual({ + expect(await thrownAsync(() => new PlacesController(svc(), new RuntimeEnvService(), storageStub).uploadImage(demo, '5', '9', file))).toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' }, }); } finally { diff --git a/server/tests/unit/nest/places.service.test.ts b/server/tests/unit/nest/places.service.test.ts index 4b5a50303..5de63f564 100644 --- a/server/tests/unit/nest/places.service.test.ts +++ b/server/tests/unit/nest/places.service.test.ts @@ -76,7 +76,7 @@ import { MapsService } from '../../../src/nest/maps/maps.service'; import { QueryHelpersService } from '../../../src/nest/query-helpers/query-helpers.service'; import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.service'; import { TrekPhotosRepository } from '../../../src/nest/photos/trek-photos.repository'; -import { PLACE_IMAGES_DIR } from '../../../src/nest/places/place-image'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; const GPX_FIXTURE = path.join(__dirname, '../../fixtures/test.gpx'); const KML_FIXTURE = path.join(__dirname, '../../fixtures/test.kml'); @@ -92,6 +92,8 @@ const dbs = new DatabaseService(testDb); * creates a journey, so every hook returns on its first lookup. * `maps` is a parameter because the enrichment cases hand in their own provider. */ +const placesStorageFx = makeStorageFixture(''); + function makePlacesService(maps: MapsService = new MapsService(dbs, photoCacheStub)): PlacesService { return new PlacesService( dbs, @@ -99,9 +101,10 @@ function makePlacesService(maps: MapsService = new MapsService(dbs, photoCacheSt new RealtimeService(), maps, new QueryHelpersService(dbs), - new UnsplashService(dbs, new RuntimeEnvService()), + new UnsplashService(dbs, new RuntimeEnvService(), placesStorageFx.storage), photoCacheStub, new JourneyDomainService(dbs, new RealtimeService(), new TrekPhotosRepository(dbs)), + placesStorageFx.storage, ); } @@ -268,71 +271,71 @@ describe('get', () => { // ── update ──────────────────────────────────────────────────────────────────── describe('update', () => { - it('PLACE-SVC-013 — updates place name and lat/lng', () => { + it('PLACE-SVC-013 — updates place name and lat/lng', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Old', lat: 0, lng: 0 }) as any; - const updated = svc.update(String(trip.id), String(place.id), { name: 'New', lat: 48.8, lng: 2.3 }) as any; + const updated = await svc.update(String(trip.id), String(place.id), { name: 'New', lat: 48.8, lng: 2.3 }) as any; expect(updated.name).toBe('New'); expect(updated.lat).toBe(48.8); expect(updated.lng).toBe(2.3); }); - it('PLACE-SVC-014 — returns null for non-existent place', () => { + it('PLACE-SVC-014 — returns null for non-existent place', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); - expect(svc.update(String(trip.id), '99999', { name: 'Ghost' })).toBeNull(); + expect(await svc.update(String(trip.id), '99999', { name: 'Ghost' })).toBeNull(); }); - it('PLACE-SVC-015 — updates tags (replaces old set)', () => { + it('PLACE-SVC-015 — updates tags (replaces old set)', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const tag1 = createTag(testDb, user.id, { name: 'Old Tag' }) as any; const tag2 = createTag(testDb, user.id, { name: 'New Tag' }) as any; const place = svc.create(String(trip.id), { name: 'Taggable', tags: [tag1.id] }) as any; - const updated = svc.update(String(trip.id), String(place.id), { tags: [tag2.id] }) as any; + const updated = await svc.update(String(trip.id), String(place.id), { tags: [tag2.id] }) as any; expect(updated.tags).toHaveLength(1); expect(updated.tags[0].id).toBe(tag2.id); }); - it('PLACE-SVC-016 — clears tags when tags: [] is passed', () => { + it('PLACE-SVC-016 — clears tags when tags: [] is passed', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const tag = createTag(testDb, user.id, { name: 'Temp' }) as any; const place = svc.create(String(trip.id), { name: 'Untaggable', tags: [tag.id] }) as any; - const updated = svc.update(String(trip.id), String(place.id), { tags: [] }) as any; + const updated = await svc.update(String(trip.id), String(place.id), { tags: [] }) as any; expect(updated.tags).toHaveLength(0); }); // ── Track colour (#776) ───────────────────────────────────────────────────── - it('PLACE-SVC-052 — stores a picked route_color', () => { + it('PLACE-SVC-052 — stores a picked route_color', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Walk' }) as any; - const updated = svc.update(String(trip.id), String(place.id), { route_color: '#e11d48' }) as any; + const updated = await svc.update(String(trip.id), String(place.id), { route_color: '#e11d48' }) as any; expect(updated.route_color).toBe('#e11d48'); }); - it('PLACE-SVC-053 — an explicit null clears it again (the reset to auto)', () => { + it('PLACE-SVC-053 — an explicit null clears it again (the reset to auto)', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Walk' }) as any; - svc.update(String(trip.id), String(place.id), { route_color: '#e11d48' }); + await svc.update(String(trip.id), String(place.id), { route_color: '#e11d48' }); // Guards the COALESCE trap: name/currency/transport_mode can never be // emptied, and route_color built that way would be a one-way door. - const cleared = svc.update(String(trip.id), String(place.id), { route_color: null }) as any; + const cleared = await svc.update(String(trip.id), String(place.id), { route_color: null }) as any; expect(cleared.route_color).toBeNull(); }); - it('PLACE-SVC-054 — an unrelated update leaves the colour alone', () => { + it('PLACE-SVC-054 — an unrelated update leaves the colour alone', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Walk' }) as any; - svc.update(String(trip.id), String(place.id), { route_color: '#059669' }); - const renamed = svc.update(String(trip.id), String(place.id), { name: 'Hike' }) as any; + await svc.update(String(trip.id), String(place.id), { route_color: '#059669' }); + const renamed = await svc.update(String(trip.id), String(place.id), { name: 'Hike' }) as any; expect(renamed.name).toBe('Hike'); expect(renamed.route_color).toBe('#059669'); }); @@ -353,14 +356,14 @@ describe('update', () => { // ── updateMany ──────────────────────────────────────────────────────────────── describe('updateMany', () => { - it('PLACE-SVC-039 — applies the same fields to many places, preserving the rest', () => { + it('PLACE-SVC-039 — applies the same fields to many places, preserving the rest', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const a = createPlace(testDb, trip.id, { name: 'A' }) as any; const b = createPlace(testDb, trip.id, { name: 'B' }) as any; const c = createPlace(testDb, trip.id, { name: 'C' }) as any; - const updated = svc.updateMany(String(trip.id), [a.id, b.id, c.id], { notes: 'visited', transport_mode: 'walking' }); + const updated = await svc.updateMany(String(trip.id), [a.id, b.id, c.id], { notes: 'visited', transport_mode: 'walking' }); expect(updated).toHaveLength(3); for (const p of updated) { @@ -371,14 +374,14 @@ describe('updateMany', () => { expect(updated.map(p => (p as any).name).sort()).toEqual(['A', 'B', 'C']); }); - it('PLACE-SVC-040 — skips ids that are not in the trip and reports the rest', () => { + it('PLACE-SVC-040 — skips ids that are not in the trip and reports the rest', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const other = createTrip(testDb, user.id); const mine = createPlace(testDb, trip.id, { name: 'Mine' }) as any; const foreign = createPlace(testDb, other.id, { name: 'Foreign' }) as any; - const updated = svc.updateMany(String(trip.id), [mine.id, foreign.id, 99999], { notes: 'tagged' }); + const updated = await svc.updateMany(String(trip.id), [mine.id, foreign.id, 99999], { notes: 'tagged' }); expect(updated).toHaveLength(1); expect((updated[0] as any).id).toBe(mine.id); @@ -386,42 +389,42 @@ describe('updateMany', () => { expect((svc.get(String(other.id), String(foreign.id)) as any).notes).toBeNull(); }); - it('PLACE-SVC-041 — returns [] for an empty id list', () => { + it('PLACE-SVC-041 — returns [] for an empty id list', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); - expect(svc.updateMany(String(trip.id), [], { notes: 'x' })).toEqual([]); + expect(await svc.updateMany(String(trip.id), [], { notes: 'x' })).toEqual([]); }); }); // ── remove ──────────────────────────────────────────────────────────────────── describe('remove', () => { - it('PLACE-SVC-017 — deletes a place and returns true', () => { + it('PLACE-SVC-017 — deletes a place and returns true', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'To Delete' }) as any; - expect(svc.remove(String(trip.id), String(place.id))).toBe(true); + expect(await svc.remove(String(trip.id), String(place.id))).toBe(true); expect(svc.get(String(trip.id), String(place.id))).toBeNull(); }); - it('PLACE-SVC-018 — returns false for non-existent place', () => { + it('PLACE-SVC-018 — returns false for non-existent place', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); - expect(svc.remove(String(trip.id), '99999')).toBe(false); + expect(await svc.remove(String(trip.id), '99999')).toBe(false); }); - it('PLACE-SVC-019 — deleting one place does not remove others', () => { + it('PLACE-SVC-019 — deleting one place does not remove others', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const p1 = createPlace(testDb, trip.id, { name: 'Keep' }) as any; const p2 = createPlace(testDb, trip.id, { name: 'Remove' }) as any; - svc.remove(String(trip.id), String(p2.id)); + await svc.remove(String(trip.id), String(p2.id)); const remaining = svc.list(String(trip.id), {}) as any[]; expect(remaining).toHaveLength(1); expect(remaining[0].id).toBe(p1.id); }); - it('PLACE-SVC-019c — the linked expense goes with the place (#1298)', () => { + it('PLACE-SVC-019c — the linked expense goes with the place (#1298)', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Louvre' }) as any; @@ -432,13 +435,13 @@ describe('remove', () => { // Read the link before the delete — that is what the controller broadcasts. expect(svc.linkedExpenseIds(trip.id, [place.id])).toEqual([linked]); - expect(svc.remove(String(trip.id), String(place.id))).toBe(true); + expect(await svc.remove(String(trip.id), String(place.id))).toBe(true); const rows = testDb.prepare('SELECT id FROM budget_items ORDER BY id').all() as { id: number }[]; expect(rows.map(r => r.id)).toEqual([untouched, standalone]); }); - it('PLACE-SVC-019d — removeMany takes the expense of every deleted place with it', () => { + it('PLACE-SVC-019d — removeMany takes the expense of every deleted place with it', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const a = createPlace(testDb, trip.id, { name: 'A' }) as any; @@ -449,7 +452,7 @@ describe('remove', () => { } expect(svc.linkedExpenseIds(trip.id, [a.id, b.id])).toHaveLength(2); - svc.removeMany(String(trip.id), [a.id, b.id]); + await svc.removeMany(String(trip.id), [a.id, b.id]); const rows = testDb.prepare('SELECT place_id FROM budget_items').all() as { place_id: number }[]; expect(rows.map(r => r.place_id)).toEqual([keep.id]); @@ -466,14 +469,14 @@ describe('remove', () => { expect(svc.linkedExpenseIds(trip.id, [])).toEqual([]); }); - it('PLACE-SVC-019b — reclaims the photo cache for the deleted place', () => { + it('PLACE-SVC-019b — reclaims the photo cache for the deleted place', async () => { removeIfUnreferencedSpy.mockClear(); const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'With Photo' }) as any; testDb.prepare('UPDATE places SET google_place_id = ? WHERE id = ?').run('ChIJgid', place.id); - svc.remove(String(trip.id), String(place.id)); + await svc.remove(String(trip.id), String(place.id)); expect(removeIfUnreferencedSpy).toHaveBeenCalledWith('ChIJgid'); }); @@ -482,7 +485,7 @@ describe('remove', () => { // ── removeMany ──────────────────────────────────────────────────────────────── describe('removeMany', () => { - it('PLACE-SVC-056 — deletes the trip-scoped ids in one transaction and reports them', () => { + it('PLACE-SVC-056 — deletes the trip-scoped ids in one transaction and reports them', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const other = createTrip(testDb, user.id); @@ -490,16 +493,16 @@ describe('removeMany', () => { const b = createPlace(testDb, trip.id, { name: 'B' }) as any; const foreign = createPlace(testDb, other.id, { name: 'Foreign' }) as any; - const deleted = svc.removeMany(String(trip.id), [a.id, b.id, foreign.id, 99999]); + const deleted = await svc.removeMany(String(trip.id), [a.id, b.id, foreign.id, 99999]); expect(deleted.sort()).toEqual([a.id, b.id].sort()); expect(svc.get(String(other.id), String(foreign.id))).not.toBeNull(); }); - it('PLACE-SVC-057 — returns [] for an empty id list', () => { + it('PLACE-SVC-057 — returns [] for an empty id list', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); - expect(svc.removeMany(String(trip.id), [])).toEqual([]); + expect(await svc.removeMany(String(trip.id), [])).toEqual([]); }); }); @@ -940,13 +943,12 @@ describe('importKmlPlaces deduplication', () => { describe('custom place image reclaim', () => { function writePlaceImage(name: string): string { - fs.mkdirSync(PLACE_IMAGES_DIR, { recursive: true }); - const filePath = path.join(PLACE_IMAGES_DIR, name); + const filePath = path.join(placesStorageFx.root, name); fs.writeFileSync(filePath, 'jpeg-bytes'); return filePath; } - it('PLACE-SVC-046 — replacing image_url unlinks the previous upload once unreferenced', () => { + it('PLACE-SVC-046 — replacing image_url unlinks the previous upload once unreferenced', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any; @@ -954,11 +956,11 @@ describe('custom place image reclaim', () => { testDb.prepare('UPDATE places SET image_url = ? WHERE id = ?').run('/uploads/places/svc-replace-a.jpg', place.id); expect(fs.existsSync(fileA)).toBe(true); - svc.update(String(trip.id), String(place.id), { image_url: '/uploads/places/svc-replace-b.jpg' }); + await svc.update(String(trip.id), String(place.id), { image_url: '/uploads/places/svc-replace-b.jpg' }); expect(fs.existsSync(fileA)).toBe(false); }); - it('PLACE-SVC-047 — clearing image_url to null unlinks the previous upload', () => { + it('PLACE-SVC-047 — clearing image_url to null unlinks the previous upload', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any; @@ -966,11 +968,11 @@ describe('custom place image reclaim', () => { testDb.prepare('UPDATE places SET image_url = ? WHERE id = ?').run('/uploads/places/svc-clear.jpg', place.id); expect(fs.existsSync(fileA)).toBe(true); - svc.update(String(trip.id), String(place.id), { image_url: null } as any); + await svc.update(String(trip.id), String(place.id), { image_url: null } as any); expect(fs.existsSync(fileA)).toBe(false); }); - it('PLACE-SVC-048 — remove unlinks the uploaded image', () => { + it('PLACE-SVC-048 — remove unlinks the uploaded image', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any; @@ -978,11 +980,11 @@ describe('custom place image reclaim', () => { testDb.prepare('UPDATE places SET image_url = ? WHERE id = ?').run('/uploads/places/svc-delete.jpg', place.id); expect(fs.existsSync(fileA)).toBe(true); - svc.remove(String(trip.id), String(place.id)); + await svc.remove(String(trip.id), String(place.id)); expect(fs.existsSync(fileA)).toBe(false); }); - it('PLACE-SVC-049 — a collection_places reference keeps the file when the trip place is deleted', () => { + it('PLACE-SVC-049 — a collection_places reference keeps the file when the trip place is deleted', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Shared Photo' }) as any; @@ -994,7 +996,7 @@ describe('custom place image reclaim', () => { .run(col.lastInsertRowid, user.id, 'Shared Photo', '/uploads/places/svc-shared.jpg'); expect(fs.existsSync(fileA)).toBe(true); - svc.remove(String(trip.id), String(place.id)); + await svc.remove(String(trip.id), String(place.id)); expect(fs.existsSync(fileA)).toBe(true); // resetTestDb does not clear collections; drop what this test inserted and its file. @@ -1303,17 +1305,17 @@ describe('zero-valued numeric fields', () => { expect((svc.create(String(trip.id), { name: 'Default' }) as any).duration_minutes).toBe(60); }); - it('PLACE-SVC-067 — update can set duration_minutes to 0', () => { + it('PLACE-SVC-067 — update can set duration_minutes to 0', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = createPlace(testDb, trip.id, { name: 'Stop' }) as any; testDb.prepare('UPDATE places SET duration_minutes = 90 WHERE id = ?').run(place.id); - const zeroed = svc.update(String(trip.id), String(place.id), { duration_minutes: 0 }) as any; + const zeroed = await svc.update(String(trip.id), String(place.id), { duration_minutes: 0 }) as any; expect(zeroed.duration_minutes).toBe(0); // An omitted duration still leaves the stored value alone (COALESCE). - const untouched = svc.update(String(trip.id), String(place.id), { name: 'Stop 2' }) as any; + const untouched = await svc.update(String(trip.id), String(place.id), { name: 'Stop 2' }) as any; expect(untouched.duration_minutes).toBe(0); }); }); diff --git a/server/tests/unit/nest/platform.controller.test.ts b/server/tests/unit/nest/platform.controller.test.ts index af139f74e..b48c44b9a 100644 --- a/server/tests/unit/nest/platform.controller.test.ts +++ b/server/tests/unit/nest/platform.controller.test.ts @@ -5,23 +5,27 @@ import { NotFoundException } from '@nestjs/common'; const h = vi.hoisted(() => ({ verifyJwtAndLoadUser: vi.fn(), dbPrepare: vi.fn(), - existsSync: vi.fn(), + exists: vi.fn(), + sendToResponse: vi.fn(), })); vi.mock('../../../src/nest/auth/jwt-verify', () => ({ verifyJwtAndLoadUser: h.verifyJwtAndLoadUser })); vi.mock('../../../src/db/database', () => ({ db: { prepare: h.dbPrepare } })); -vi.mock('node:fs', async (orig) => { - const real = (await orig()) as Record; - return { ...real, default: { ...(real.default as object), existsSync: h.existsSync }, existsSync: h.existsSync }; -}); - import { applyPlatformUploads, applyPlatformSpa, applyPlatformStatic, + storageStaticHandler, } from '../../../src/nest/platform/platform.routes'; import { SpaFallbackFilter } from '../../../src/nest/platform/spa-fallback.filter'; +import { StorageNotFoundError, StorageInvalidKeyError } from '../../../src/nest/storage/storage.types'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; + +// The serving swap addresses files as (category, name) on the injected facade; +// these unit tests only assert routing/auth/error mapping, so a two-method stub +// is the whole storage surface. +const storage = { exists: h.exists, sendToResponse: h.sendToResponse } as unknown as StorageService; // Tagged sentinel for express.static — we only need to know it was registered on // the right path, not run it. @@ -76,18 +80,24 @@ beforeEach(() => { }); describe('applyPlatformUploads', () => { - it('registers the static avatar/cover/journey mounts + the files block', () => { + it('registers the four static mounts + the files block', () => { const { app, calls } = fakeApp(); - applyPlatformUploads(app); + applyPlatformUploads(app, storage); const paths = calls.filter((c) => c.method === 'use').map((c) => c.path); expect(paths).toEqual( - expect.arrayContaining(['/uploads/avatars', '/uploads/covers', '/uploads/journey', '/uploads/files']), + expect.arrayContaining([ + '/uploads/avatars', + '/uploads/covers', + '/uploads/journey', + '/uploads/places', + '/uploads/files', + ]), ); }); it('the /uploads/files block always answers 401', () => { const { app, calls } = fakeApp(); - applyPlatformUploads(app); + applyPlatformUploads(app, storage); const filesBlock = calls.find((c) => c.path === '/uploads/files')!.handlers[0]; const res = makeRes(); filesBlock({}, res); @@ -98,103 +108,248 @@ describe('applyPlatformUploads', () => { describe('GET /uploads/photos/:filename', () => { function photoHandler() { const { app, calls } = fakeApp(); - applyPlatformUploads(app); + applyPlatformUploads(app, storage); return calls.find((c) => c.method === 'get' && c.path === '/uploads/photos/:filename')!.handlers[0]; } + const next = vi.fn(); - it('403 when the resolved path escapes the photos dir', () => { - // basename() strips the traversal, but feed a name that resolves outside by - // stubbing path indirectly is hard — instead exercise the existsSync 404 etc. - // The startsWith guard is defensive; cover it via a filename of '..'. - const handler = photoHandler(); + it('403 when the basename is a bare traversal segment', async () => { + // Parity pin: the old resolve()+startsWith guard could only fire after + // basename() when the remaining segment was '..'. const res = makeRes(); - // path.basename('..') === '..' -> join(photos,'..') resolves to uploads -> not under photos - handler({ params: { filename: '..' }, headers: {}, query: {} }, res); + await photoHandler()({ params: { filename: '..' }, headers: {}, query: {} }, res, next); expect(res.statusCode).toBe(403); expect(res.body).toBe('Forbidden'); + expect(h.exists).not.toHaveBeenCalled(); }); - it('404 when the file does not exist', () => { - h.existsSync.mockReturnValue(false); + it('404 when the object does not exist — checked before auth', async () => { + h.exists.mockResolvedValue(false); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: {} }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: {} }, res, next); + expect(h.exists).toHaveBeenCalledWith('photos', 'a.jpg'); + expect(res.statusCode).toBe(404); + expect(res.body).toBe('Not found'); + expect(h.verifyJwtAndLoadUser).not.toHaveBeenCalled(); + }); + + it('404 when the key is invalid (exists rejects) — still before auth', async () => { + h.exists.mockRejectedValue(new StorageInvalidKeyError('photos/.')); + const res = makeRes(); + await photoHandler()({ params: { filename: '.' }, headers: {}, query: {} }, res, next); expect(res.statusCode).toBe(404); expect(res.body).toBe('Not found'); }); - it('401 when no token is supplied', () => { - h.existsSync.mockReturnValue(true); + it('401 when no token is supplied', async () => { + h.exists.mockResolvedValue(true); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: {} }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: {} }, res, next); expect(res.statusCode).toBe(401); expect(res.body).toBe('Authentication required'); }); - it('serves the file for a valid JWT session (Bearer header)', () => { - h.existsSync.mockReturnValue(true); + it('serves the file for a valid JWT session (Bearer header)', async () => { + h.exists.mockResolvedValue(true); + h.sendToResponse.mockResolvedValue(undefined); h.verifyJwtAndLoadUser.mockReturnValue({ id: 1 }); const res = makeRes(); - photoHandler()( + await photoHandler()( { params: { filename: 'a.jpg' }, headers: { authorization: 'Bearer jwt123' }, query: {} }, res, + next, ); expect(h.verifyJwtAndLoadUser).toHaveBeenCalledWith('jwt123'); - expect(String(res.body)).toContain('FILE:'); + expect(h.sendToResponse).toHaveBeenCalledWith('photos', 'a.jpg', res); }); - it('reads the token from the query string when there is no Bearer header', () => { - h.existsSync.mockReturnValue(true); + it('reads the token from the query string when there is no Bearer header', async () => { + h.exists.mockResolvedValue(true); + h.sendToResponse.mockResolvedValue(undefined); h.verifyJwtAndLoadUser.mockReturnValue({ id: 1 }); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'qtok' } }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'qtok' } }, res, next); expect(h.verifyJwtAndLoadUser).toHaveBeenCalledWith('qtok'); - expect(String(res.body)).toContain('FILE:'); + expect(h.sendToResponse).toHaveBeenCalledWith('photos', 'a.jpg', res); }); - it('401 when the token is not a session and the photo row is missing', () => { - h.existsSync.mockReturnValue(true); + it('401 when the token is not a session and the photo row is missing', async () => { + h.exists.mockResolvedValue(true); h.verifyJwtAndLoadUser.mockReturnValue(null); h.dbPrepare.mockReturnValue({ get: vi.fn().mockReturnValue(undefined) }); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res, next); expect(res.statusCode).toBe(401); }); - it('401 when a share token does not cover the photo trip', () => { - h.existsSync.mockReturnValue(true); + it('401 when a share token does not cover the photo trip', async () => { + h.exists.mockResolvedValue(true); h.verifyJwtAndLoadUser.mockReturnValue(null); const photoStmt = { get: vi.fn().mockReturnValue({ trip_id: 7 }) }; const shareStmt = { get: vi.fn().mockReturnValue({ trip_id: 8 }) }; h.dbPrepare.mockImplementationOnce(() => photoStmt).mockImplementationOnce(() => shareStmt); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res, next); expect(res.statusCode).toBe(401); }); - it('401 when there is no matching share token at all', () => { - h.existsSync.mockReturnValue(true); + it('401 when there is no matching share token at all', async () => { + h.exists.mockResolvedValue(true); h.verifyJwtAndLoadUser.mockReturnValue(null); const photoStmt = { get: vi.fn().mockReturnValue({ trip_id: 7 }) }; const shareStmt = { get: vi.fn().mockReturnValue(undefined) }; h.dbPrepare.mockImplementationOnce(() => photoStmt).mockImplementationOnce(() => shareStmt); const res = makeRes(); - photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res); + await photoHandler()({ params: { filename: 'a.jpg' }, headers: {}, query: { token: 'share1' } }, res, next); expect(res.statusCode).toBe(401); }); - it('serves the file when the share token covers the photo trip', () => { - h.existsSync.mockReturnValue(true); + it('serves the file when the share token covers the photo trip', async () => { + h.exists.mockResolvedValue(true); + h.sendToResponse.mockResolvedValue(undefined); h.verifyJwtAndLoadUser.mockReturnValue(null); const photoStmt = { get: vi.fn().mockReturnValue({ trip_id: 7 }) }; const shareStmt = { get: vi.fn().mockReturnValue({ trip_id: 7 }) }; h.dbPrepare.mockImplementationOnce(() => photoStmt).mockImplementationOnce(() => shareStmt); const res = makeRes(); - photoHandler()( + await photoHandler()( { params: { filename: 'a.jpg' }, headers: { authorization: 'Bearer share1' }, query: {} }, res, + next, ); - expect(String(res.body)).toContain('FILE:'); + expect(h.sendToResponse).toHaveBeenCalledWith('photos', 'a.jpg', res); }); + + it('404 when the object vanishes between the exists check and the send', async () => { + // Approved deviation D7: the delete race maps to the same 404 text. + h.exists.mockResolvedValue(true); + h.verifyJwtAndLoadUser.mockReturnValue({ id: 1 }); + h.sendToResponse.mockRejectedValue(new StorageNotFoundError('photos/a.jpg')); + const res = makeRes(); + const resAny = res as unknown as { headersSent?: boolean }; + resAny.headersSent = false; + await photoHandler()( + { params: { filename: 'a.jpg' }, headers: { authorization: 'Bearer jwt123' }, query: {} }, + res, + next, + ); + expect(res.statusCode).toBe(404); + expect(res.body).toBe('Not found'); + }); + + it('rethrows a non-miss send failure to the route error handler', async () => { + h.exists.mockResolvedValue(true); + h.verifyJwtAndLoadUser.mockReturnValue({ id: 1 }); + const boom = new Error('disk on fire'); + h.sendToResponse.mockRejectedValue(boom); + const res = makeRes(); + const localNext = vi.fn(); + await photoHandler()( + { params: { filename: 'a.jpg' }, headers: { authorization: 'Bearer jwt123' }, query: {} }, + res, + localNext, + ); + expect(localNext).toHaveBeenCalledWith(boom); + expect(res.statusCode).toBe(200); // untouched — finalhandler owns it now + }); + + it('does not write a second 404 when the send fails after headers were flushed', async () => { + h.exists.mockResolvedValue(true); + h.verifyJwtAndLoadUser.mockReturnValue({ id: 1 }); + h.sendToResponse.mockRejectedValue(new StorageNotFoundError('photos/a.jpg')); + const res = makeRes(); + (res as unknown as { headersSent: boolean }).headersSent = true; + const localNext = vi.fn(); + await photoHandler()( + { params: { filename: 'a.jpg' }, headers: { authorization: 'Bearer jwt123' }, query: {} }, + res, + localNext, + ); + expect(res.status).not.toHaveBeenCalled(); + expect(localNext).toHaveBeenCalledWith(expect.any(StorageNotFoundError)); + }); + }); +}); + +describe('storageStaticHandler', () => { + const req = (over: Record = {}) => ({ method: 'GET', path: '/x.png', ...over }); + + function run(over: Record = {}) { + const handler = storageStaticHandler(storage, 'avatars'); + const res = makeRes(); + const next = vi.fn(); + const out = handler(req(over) as never, res as never, next as never); + return { res, next, out }; + } + + it('serves a hit through sendToResponse with the decoded name', async () => { + h.sendToResponse.mockResolvedValue(undefined); + const { res, next, out } = run({ path: '//caf%C3%A9.png' }); + await out; + expect(h.sendToResponse).toHaveBeenCalledWith('avatars', 'café.png', res); + expect(next).not.toHaveBeenCalled(); + }); + + it('non-GET/HEAD methods fall through without touching storage', async () => { + const { next } = run({ method: 'POST' }); + expect(next).toHaveBeenCalledWith(); + expect(h.sendToResponse).not.toHaveBeenCalled(); + }); + + it('undecodable percent-encoding falls through', async () => { + const { next } = run({ path: '/%ZZ' }); + expect(next).toHaveBeenCalledWith(); + expect(h.sendToResponse).not.toHaveBeenCalled(); + }); + + it('a miss calls next() with no args', async () => { + h.sendToResponse.mockRejectedValue(new StorageNotFoundError('avatars/x.png')); + const { next, out } = run(); + await out; + expect(next).toHaveBeenCalledWith(); + }); + + it('an invalid key calls next() with no args', async () => { + h.sendToResponse.mockRejectedValue(new StorageInvalidKeyError('avatars/..')); + const { next, out } = run(); + await out; + expect(next).toHaveBeenCalledWith(); + }); + + it('a send-layer 404 (stat→send race) falls through', async () => { + h.sendToResponse.mockRejectedValue(Object.assign(new Error('ENOENT'), { status: 404 })); + const { next, out } = run(); + await out; + expect(next).toHaveBeenCalledWith(); + }); + + it('EISDIR falls through', async () => { + h.sendToResponse.mockRejectedValue(Object.assign(new Error('dir'), { code: 'EISDIR' })); + const { next, out } = run(); + await out; + expect(next).toHaveBeenCalledWith(); + }); + + it('a client abort is swallowed', async () => { + h.sendToResponse.mockRejectedValue(Object.assign(new Error('aborted'), { code: 'ECONNABORTED' })); + const { next, out } = run(); + await out; + expect(next).not.toHaveBeenCalled(); + }); + + it('a mid-stream write error is swallowed', async () => { + h.sendToResponse.mockRejectedValue(Object.assign(new Error('write EPIPE'), { syscall: 'write' })); + const { next, out } = run(); + await out; + expect(next).not.toHaveBeenCalled(); + }); + + it('anything else goes to next(err) for finalhandler', async () => { + const boom = new Error('boom'); + h.sendToResponse.mockRejectedValue(boom); + const { next, out } = run(); + await out; + expect(next).toHaveBeenCalledWith(boom); }); }); diff --git a/server/tests/unit/nest/share.controller.test.ts b/server/tests/unit/nest/share.controller.test.ts index a794a54e4..d1f02b819 100644 --- a/server/tests/unit/nest/share.controller.test.ts +++ b/server/tests/unit/nest/share.controller.test.ts @@ -1,14 +1,17 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { HttpException } from '@nestjs/common'; import type { Response } from 'express'; - -const { createReadStream } = vi.hoisted(() => ({ createReadStream: vi.fn() })); -vi.mock('node:fs', () => ({ createReadStream })); +import { PassThrough } from 'node:stream'; import { TripShareController, SharedController } from '../../../src/nest/share/share.controller'; import type { ShareService } from '../../../src/nest/share/share.service'; +import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; +// Only the shared place-photo proxy consumes storage. +const getStream = vi.fn(); +const storageStub = { getStream } as unknown as StorageService; + const user = { id: 1, role: 'user', email: 'u@example.test' } as User; function svc(o: Partial = {}): ShareService { @@ -74,69 +77,130 @@ describe('TripShareController', () => { describe('SharedController', () => { it('404 for an invalid token, else returns the snapshot', () => { - expect(thrown(() => new SharedController(svc({ getSharedTripData: vi.fn().mockReturnValue(null) } as Partial)).read('bad'))).toEqual({ status: 404, body: { error: 'Invalid or expired link' } }); - expect(new SharedController(svc({ getSharedTripData: vi.fn().mockReturnValue({ trip: { id: 9 } }) } as Partial)).read('tok')).toEqual({ trip: { id: 9 } }); + expect(thrown(() => new SharedController(svc({ getSharedTripData: vi.fn().mockReturnValue(null) } as Partial), storageStub).read('bad'))).toEqual({ status: 404, body: { error: 'Invalid or expired link' } }); + expect(new SharedController(svc({ getSharedTripData: vi.fn().mockReturnValue({ trip: { id: 9 } }) } as Partial), storageStub).read('tok')).toEqual({ trip: { id: 9 } }); }); describe('place-photo proxy', () => { + // A real PassThrough (like storage.service.test.ts's makeRes) rather than + // a { on, pipe } stub: pipeline() needs authentic stream/eos semantics + // (close/error/finish wiring) that a bare mock can't satisfy. function photoRes() { - const r = { + const sink = new PassThrough(); + const chunks: Buffer[] = []; + sink.on('data', (c: Buffer) => chunks.push(c)); + // Capture the real PassThrough#end BEFORE overwriting the property + // below — r and sink are the same object (Object.assign mutates and + // returns its target), so referencing sink.end inside the wrapper + // would recurse into itself. + const realEnd = sink.end.bind(sink) as (...a: unknown[]) => unknown; + const r = Object.assign(sink, { statusCode: 200, - headersSent: false, status: vi.fn(function (this: unknown, c: number) { (r as { statusCode: number }).statusCode = c; return r; }), json: vi.fn(), set: vi.fn(), type: vi.fn(), + end: vi.fn((...args: unknown[]) => realEnd(...args)), + body: () => Buffer.concat(chunks).toString(), + }); + return r as unknown as Response & { + status: ReturnType; + json: ReturnType; + set: ReturnType; + type: ReturnType; + end: ReturnType; + body: () => string; }; - return r as unknown as Response & { status: ReturnType; json: ReturnType; set: ReturnType; type: ReturnType }; } - beforeEach(() => createReadStream.mockReset()); - - function controller(path: string | null) { - return new SharedController(svc({ getSharedPlacePhotoPath: vi.fn().mockReturnValue(path) } as Partial)); - } - - it('404 without streaming when the photo is not cached for the token', () => { - const res = photoRes(); - controller(null).placePhotoBytes('tok', 'p1', res); - expect(res.status).toHaveBeenCalledWith(404); - expect(res.json).toHaveBeenCalledWith({ error: 'Photo not cached' }); - expect(createReadStream).not.toHaveBeenCalled(); + // Braced body on purpose: mockReset() returns the mock, and a function + // returned from beforeEach is invoked as a teardown callback — which would + // call getStream() bare and turn a rejecting impl into an unhandled + // rejection. + beforeEach(() => { + getStream.mockReset(); }); - it('streams the cached file with image/jpeg + an immutable cache header on a hit', () => { - const stream = { on: vi.fn().mockReturnThis(), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + function controller(key: string | null) { + return new SharedController(svc({ getSharedPlacePhotoKey: vi.fn().mockReturnValue(key) } as Partial), storageStub); + } + + // #1727's rationale extended to public share pages: shared payloads keep + // this URL in place image_urls, so an evicted cache entry means one request + // per place per shared-page render — an empty 204 instead of 404 noise. + it('204 without a body when the photo is not cached for the token', async () => { const res = photoRes(); - controller('/cache/p1.jpg').placePhotoBytes('tok', 'p1', res); + await controller(null).placePhotoBytes('tok', 'p1', res); + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + expect(res.json).not.toHaveBeenCalled(); + expect(res.set).toHaveBeenLastCalledWith('Cache-Control', 'no-store'); + expect(getStream).not.toHaveBeenCalled(); + }); + + it('streams the cached bytes with image/jpeg + an immutable cache header on a hit', async () => { + const stream = new PassThrough(); + stream.end('hello'); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); + const res = photoRes(); + await controller('abc.jpg').placePhotoBytes('tok', 'p1', res); expect(res.set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=2592000, immutable'); expect(res.type).toHaveBeenCalledWith('image/jpeg'); - expect(createReadStream).toHaveBeenCalledWith('/cache/p1.jpg'); - expect(stream.pipe).toHaveBeenCalledWith(res); + expect(getStream).toHaveBeenCalledWith('photos-google', 'abc.jpg'); + expect(res.body()).toBe('hello'); }); - it('falls back to 404 when the read stream errors before headers were sent', () => { - let onError: () => void = () => {}; - const stream = { on: vi.fn((ev: string, cb: () => void) => { if (ev === 'error') onError = cb; return stream; }), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + it('falls back to an empty 204 when the stream cannot be opened (cache-delete race)', async () => { + getStream.mockRejectedValue(new Error('storage object not found')); const res = photoRes(); - controller('/cache/p1.jpg').placePhotoBytes('tok', 'p1', res); - onError(); - expect(res.status).toHaveBeenCalledWith(404); - expect(res.json).toHaveBeenCalledWith({ error: 'Photo not cached' }); + await controller('abc.jpg').placePhotoBytes('tok', 'p1', res); + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + // The hit path already asked for a month of immutable caching — that + // header must not survive onto the empty answer. + expect(res.set).toHaveBeenLastCalledWith('Cache-Control', 'no-store'); }); - it('does not re-send a 404 when the stream errors after headers were flushed', () => { - let onError: () => void = () => {}; - const stream = { on: vi.fn((ev: string, cb: () => void) => { if (ev === 'error') onError = cb; return stream; }), pipe: vi.fn() }; - createReadStream.mockReturnValue(stream); + it('falls back to an empty 204 when the stream errors before headers were sent', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); const res = photoRes(); - (res as { headersSent: boolean }).headersSent = true; - controller('/cache/p1.jpg').placePhotoBytes('tok', 'p1', res); - onError(); + const pending = controller('abc.jpg').placePhotoBytes('tok', 'p1', res); + // Let pipeline() attach its listeners before erroring the source, so + // the event isn't missed to a scheduling race. + await new Promise((resolve) => setImmediate(resolve)); + stream.destroy(new Error('source boom')); + await pending; + expect(res.status).toHaveBeenCalledWith(204); + expect(res.end).toHaveBeenCalled(); + expect(res.set).toHaveBeenLastCalledWith('Cache-Control', 'no-store'); + }); + + it('swallows an abort-like error once headers are flushed (client walked away)', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); + const res = photoRes(); + (res as unknown as { headersSent: boolean }).headersSent = true; + const pending = controller('abc.jpg').placePhotoBytes('tok', 'p1', res); + await new Promise((resolve) => setImmediate(resolve)); + stream.destroy(Object.assign(new Error('reset'), { code: 'ECONNRESET' })); + await expect(pending).resolves.toBeUndefined(); expect(res.status).not.toHaveBeenCalled(); expect(res.json).not.toHaveBeenCalled(); + expect(res.end).not.toHaveBeenCalled(); + }); + + it('re-throws a real (non-abort) error once headers are flushed', async () => { + const stream = new PassThrough(); + getStream.mockResolvedValue({ stream, stat: { key: 'photos/google/abc.jpg', size: 5, mtimeMs: 0 } }); + const res = photoRes(); + (res as unknown as { headersSent: boolean }).headersSent = true; + const pending = controller('abc.jpg').placePhotoBytes('tok', 'p1', res); + await new Promise((resolve) => setImmediate(resolve)); + stream.destroy(Object.assign(new Error('disk read error'), { code: 'EIO' })); + await expect(pending).rejects.toThrow('disk read error'); + expect(res.status).not.toHaveBeenCalled(); + expect(res.end).not.toHaveBeenCalled(); }); }); }); diff --git a/server/tests/unit/nest/share.service.test.ts b/server/tests/unit/nest/share.service.test.ts index eca53a484..168e4e371 100644 --- a/server/tests/unit/nest/share.service.test.ts +++ b/server/tests/unit/nest/share.service.test.ts @@ -45,8 +45,8 @@ const checkPermission = vi.fn(); const permissionsStub = { checkPermission } as unknown as PermissionsService; // Injected stub since the photo-cache fold (was a path mock of the module). -const serveFilePath = vi.fn(); -const photoCacheStub = { serveFilePath } as unknown as PlacePhotoCacheService; +const serveKey = vi.fn(); +const photoCacheStub = { serveKey } as unknown as PlacePhotoCacheService; import { createTables } from '../../../src/db/schema'; import { runMigrations } from '../../../src/db/migrations'; @@ -71,7 +71,7 @@ beforeAll(() => { beforeEach(() => { resetTestDb(testDb); checkPermission.mockReset(); - serveFilePath.mockReset(); + serveKey.mockReset(); }); afterAll(() => { @@ -377,41 +377,41 @@ describe('getSharedTripData', () => { }); }); -// ── getSharedPlacePhotoPath ─────────────────────────────────────────────────── +// ── getSharedPlacePhotoKey ─────────────────────────────────────────────────── -describe('getSharedPlacePhotoPath', () => { - it('SHARE-SVC-022: returns null for an unknown or expired token', () => { - expect(svc.getSharedPlacePhotoPath('nope', 'ChIJabc')).toBeNull(); +describe('getSharedPlacePhotoKey', () => { + it('SHARE-SVC-022: returns null for an unknown or expired token', async () => { + expect(await svc.getSharedPlacePhotoKey('nope', 'ChIJabc')).toBeNull(); const { trip, token } = seedSharedTrip(); testDb.prepare('UPDATE share_tokens SET expires_at = ? WHERE trip_id = ?').run('2020-01-01T00:00:00.000Z', trip.id); - expect(svc.getSharedPlacePhotoPath(token, 'ChIJabc')).toBeNull(); - expect(serveFilePath).not.toHaveBeenCalled(); + expect(await svc.getSharedPlacePhotoKey(token, 'ChIJabc')).toBeNull(); + expect(serveKey).not.toHaveBeenCalled(); }); - it('SHARE-SVC-023: returns null when the owner disabled the map section', () => { + it('SHARE-SVC-023: returns null when the owner disabled the map section', async () => { const { trip, token } = seedSharedTrip({ share_map: false }); const place = createPlace(testDb, trip.id); testDb.prepare('UPDATE places SET image_url = ? WHERE id = ?').run('/api/maps/place-photo/ChIJabc/bytes', place.id); - expect(svc.getSharedPlacePhotoPath(token, 'ChIJabc')).toBeNull(); - expect(serveFilePath).not.toHaveBeenCalled(); + expect(await svc.getSharedPlacePhotoKey(token, 'ChIJabc')).toBeNull(); + expect(serveKey).not.toHaveBeenCalled(); }); - it('SHARE-SVC-024: returns null when no place in the trip carries the proxy URL', () => { + it('SHARE-SVC-024: returns null when no place in the trip carries the proxy URL', async () => { const { token } = seedSharedTrip(); - expect(svc.getSharedPlacePhotoPath(token, 'ChIJabc')).toBeNull(); - expect(serveFilePath).not.toHaveBeenCalled(); + expect(await svc.getSharedPlacePhotoKey(token, 'ChIJabc')).toBeNull(); + expect(serveKey).not.toHaveBeenCalled(); }); - it('SHARE-SVC-025: resolves via serveFilePath when the encoded placeId matches the stored URL', () => { + it('SHARE-SVC-025: resolves via serveKey when the encoded placeId matches the stored URL', async () => { const { trip, token } = seedSharedTrip(); const place = createPlace(testDb, trip.id); // Wikimedia pseudo-IDs contain characters that must round-trip encoded. const placeId = 'coords:48.8,2.3'; testDb.prepare('UPDATE places SET image_url = ? WHERE id = ?') .run(`/api/maps/place-photo/${encodeURIComponent(placeId)}/bytes`, place.id); - serveFilePath.mockReturnValue('/cache/abc.jpg'); - expect(svc.getSharedPlacePhotoPath(token, placeId)).toBe('/cache/abc.jpg'); - expect(serveFilePath).toHaveBeenCalledWith(placeId); + serveKey.mockReturnValue('abc.jpg'); + expect(await svc.getSharedPlacePhotoKey(token, placeId)).toBe('abc.jpg'); + expect(serveKey).toHaveBeenCalledWith(placeId); }); }); diff --git a/server/tests/unit/nest/storage/content-type.test.ts b/server/tests/unit/nest/storage/content-type.test.ts new file mode 100644 index 000000000..f2b7b4130 --- /dev/null +++ b/server/tests/unit/nest/storage/content-type.test.ts @@ -0,0 +1,14 @@ +import { describe, it, expect } from 'vitest'; +import { contentTypeFor } from '../../../../src/nest/storage/content-type'; + +describe('contentTypeFor', () => { + it('CTYPE-001 maps common extensions, case-insensitively, and falls back to octet-stream', () => { + expect(contentTypeFor('files/a1b2.pdf')).toBe('application/pdf'); + expect(contentTypeFor('covers/x.JPG')).toBe('image/jpeg'); + expect(contentTypeFor('journey/clip.mp4')).toBe('video/mp4'); + expect(contentTypeFor('files/notes.txt')).toBe('text/plain'); + expect(contentTypeFor('files/archive.zip')).toBe('application/zip'); + expect(contentTypeFor('files/no-extension')).toBe('application/octet-stream'); + expect(contentTypeFor('files/weird.xyzzy')).toBe('application/octet-stream'); + }); +}); diff --git a/server/tests/unit/nest/storage/http-serving.test.ts b/server/tests/unit/nest/storage/http-serving.test.ts new file mode 100644 index 000000000..ca4d5c026 --- /dev/null +++ b/server/tests/unit/nest/storage/http-serving.test.ts @@ -0,0 +1,179 @@ +import { describe, it, expect } from 'vitest'; +import { + decideRange, + isNotModified, + validatorsFor, + type Validators, +} from '../../../../src/nest/storage/http-serving'; + +// The pure half of the remote-serving parity work (task C4): every decision a +// proxying byte-path makes BEFORE it touches a driver or a socket. Kept +// dependency-free so the state machine is provable in isolation — the +// service-level suite then only has to prove the wiring. + +const LM_5000 = new Date(5000).toUTCString(); + +describe('validatorsFor', () => { + it('prefers the driver-supplied ETag verbatim', () => { + expect(validatorsFor({ key: 'a.bin', size: 10, mtimeMs: 5000, etag: '"abc123"' })).toEqual({ + etag: '"abc123"', + lastModified: LM_5000, + }); + }); + + it('derives a weak size-mtime ETag when the driver supplies none (send/express parity)', () => { + expect(validatorsFor({ key: 'a.bin', size: 26, mtimeMs: 4096 })).toEqual({ + etag: 'W/"1a-1000"', + lastModified: new Date(4096).toUTCString(), + }); + }); + + it('emits NO validators at all when mtimeMs is 0 (S3 can report it — never a 1970 header)', () => { + expect(validatorsFor({ key: 'a.bin', size: 10, mtimeMs: 0 })).toEqual({}); + }); + + it('still emits a driver ETag when mtimeMs is 0, but no Last-Modified', () => { + expect(validatorsFor({ key: 'a.bin', size: 10, mtimeMs: 0, etag: '"abc123"' })).toEqual({ + etag: '"abc123"', + }); + }); + + it('ignores an empty driver ETag and falls back to the derived one', () => { + expect(validatorsFor({ key: 'a.bin', size: 1, mtimeMs: 16, etag: '' }).etag).toBe('W/"1-10"'); + }); + + it('floors a fractional mtime into the derived tag', () => { + expect(validatorsFor({ key: 'a.bin', size: 1, mtimeMs: 16.9 }).etag).toBe('W/"1-10"'); + }); +}); + +describe('isNotModified', () => { + const v: Validators = { etag: '"abc"', lastModified: LM_5000 }; + + it('matches If-None-Match exactly', () => { + expect(isNotModified({ 'if-none-match': '"abc"' }, v)).toBe(true); + expect(isNotModified({ 'if-none-match': '"other"' }, v)).toBe(false); + }); + + it('matches a list of candidate tags', () => { + expect(isNotModified({ 'if-none-match': '"x", "abc" ,"y"' }, v)).toBe(true); + }); + + it('compares weakly — W/ prefixes on either side are stripped', () => { + expect(isNotModified({ 'if-none-match': 'W/"abc"' }, v)).toBe(true); + expect(isNotModified({ 'if-none-match': '"1a-1000"' }, { etag: 'W/"1a-1000"' })).toBe(true); + }); + + it('honours the * wildcard', () => { + expect(isNotModified({ 'if-none-match': '*' }, v)).toBe(true); + expect(isNotModified({ 'if-none-match': '*' }, {})).toBe(true); + }); + + it('never matches when we have no ETag to compare against', () => { + expect(isNotModified({ 'if-none-match': '"abc"' }, {})).toBe(false); + }); + + it('lets If-None-Match win over If-Modified-Since even when the date says fresh', () => { + expect(isNotModified({ 'if-none-match': '"stale"', 'if-modified-since': LM_5000 }, v)).toBe(false); + }); + + it('matches If-Modified-Since at second granularity', () => { + // 5400ms truncates to the same second as 5000ms — a 1970-second-boundary + // stand-in for the sub-second mtimes Last-Modified cannot express. + expect(isNotModified({ 'if-modified-since': LM_5000 }, { lastModified: new Date(5400).toUTCString() })).toBe(true); + expect(isNotModified({ 'if-modified-since': LM_5000 }, { lastModified: new Date(6000).toUTCString() })).toBe(false); + }); + + it('treats an older If-Modified-Since as modified', () => { + expect(isNotModified({ 'if-modified-since': new Date(4000).toUTCString() }, v)).toBe(false); + }); + + it('ignores an unparseable If-Modified-Since', () => { + expect(isNotModified({ 'if-modified-since': 'not-a-date' }, v)).toBe(false); + }); + + it('is false when there is no Last-Modified to compare (mtimeMs 0)', () => { + expect(isNotModified({ 'if-modified-since': LM_5000 }, {})).toBe(false); + }); + + it('is false with no conditional headers at all', () => { + expect(isNotModified({}, v)).toBe(false); + }); + + it('reads the first value of a repeated header', () => { + expect(isNotModified({ 'if-none-match': ['"abc"', '"zzz"'] }, v)).toBe(true); + }); +}); + +describe('decideRange', () => { + it('is full with no Range header', () => { + expect(decideRange({}, 10, '"abc"')).toEqual({ kind: 'full' }); + }); + + it('resolves a closed range inclusively', () => { + expect(decideRange({ range: 'bytes=2-5' }, 10, '"abc"')).toEqual({ kind: 'partial', start: 2, end: 5 }); + }); + + it('resolves an open-ended range to the last byte', () => { + expect(decideRange({ range: 'bytes=5-' }, 10, '"abc"')).toEqual({ kind: 'partial', start: 5, end: 9 }); + }); + + it('clamps an end past EOF', () => { + expect(decideRange({ range: 'bytes=5-999' }, 10, '"abc"')).toEqual({ kind: 'partial', start: 5, end: 9 }); + }); + + it('computes a suffix range from the pre-stat size', () => { + expect(decideRange({ range: 'bytes=-4' }, 10, '"abc"')).toEqual({ kind: 'partial', start: 6, end: 9 }); + }); + + it('clamps a suffix larger than the object to the whole object', () => { + expect(decideRange({ range: 'bytes=-999' }, 10, '"abc"')).toEqual({ kind: 'partial', start: 0, end: 9 }); + }); + + it('is unsatisfiable past EOF, on an inverted range, and on a zero-length suffix', () => { + expect(decideRange({ range: 'bytes=10-12' }, 10, '"abc"')).toEqual({ kind: 'unsatisfiable' }); + expect(decideRange({ range: 'bytes=5-3' }, 10, '"abc"')).toEqual({ kind: 'unsatisfiable' }); + expect(decideRange({ range: 'bytes=-0' }, 10, '"abc"')).toEqual({ kind: 'unsatisfiable' }); + }); + + it('is unsatisfiable for any range against a zero-byte object', () => { + expect(decideRange({ range: 'bytes=0-' }, 0, '"abc"')).toEqual({ kind: 'unsatisfiable' }); + expect(decideRange({ range: 'bytes=-1' }, 0, '"abc"')).toEqual({ kind: 'unsatisfiable' }); + }); + + it('falls back to a full 200 for a multi-range request (no multipart/byteranges)', () => { + expect(decideRange({ range: 'bytes=0-1,4-5' }, 10, '"abc"')).toEqual({ kind: 'full' }); + }); + + it('falls back to a full 200 for malformed or non-byte units', () => { + for (const range of ['bytes=-', 'bytes=abc', 'items=0-1', 'bytes 0-1', 'bytes=0-1;x', '']) { + expect(decideRange({ range }, 10, '"abc"')).toEqual({ kind: 'full' }); + } + }); + + it('honours a strong-exact If-Range match', () => { + expect(decideRange({ range: 'bytes=2-5', 'if-range': '"abc"' }, 10, '"abc"')).toEqual({ + kind: 'partial', + start: 2, + end: 5, + }); + }); + + it('serves the full entity when If-Range does not strong-match', () => { + expect(decideRange({ range: 'bytes=2-5', 'if-range': '"stale"' }, 10, '"abc"')).toEqual({ kind: 'full' }); + // A weak validator can never satisfy If-Range (RFC 9110 §13.1.5). + expect(decideRange({ range: 'bytes=2-5', 'if-range': 'W/"abc"' }, 10, 'W/"abc"')).toEqual({ kind: 'full' }); + // Neither can an HTTP-date form — we only carry the ETag here. + expect(decideRange({ range: 'bytes=2-5', 'if-range': LM_5000 }, 10, '"abc"')).toEqual({ kind: 'full' }); + // …nor an If-Range sent for a resource we have no ETag for. + expect(decideRange({ range: 'bytes=2-5', 'if-range': '"abc"' }, 10, undefined)).toEqual({ kind: 'full' }); + }); + + it('reads the first value of a repeated Range header', () => { + expect(decideRange({ range: ['bytes=2-5', 'bytes=0-1'] }, 10, '"abc"')).toEqual({ + kind: 'partial', + start: 2, + end: 5, + }); + }); +}); diff --git a/server/tests/unit/nest/storage/local.driver.test.ts b/server/tests/unit/nest/storage/local.driver.test.ts new file mode 100644 index 000000000..c61f77b6d --- /dev/null +++ b/server/tests/unit/nest/storage/local.driver.test.ts @@ -0,0 +1,138 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import { StorageInvalidKeyError } from '../../../../src/nest/storage/storage.types'; +import { describeStorageDriver, type DriverHarness } from './storage-driver.contract'; + +function makeTmpDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-')); +} + +async function makeHarness(): Promise { + const root = makeTmpDir(); + const tmpFiles = makeTmpDir(); + const driver = new LocalDriver({ id: 'test-local', root }); + driver.init({ cleanSpool: true }); + let counter = 0; + return { + driver, + makeTempFile: async (contents) => { + const p = path.join(tmpFiles, `src-${counter++}.tmp`); + fs.writeFileSync(p, contents); + return p; + }, + cleanup: async () => { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(tmpFiles, { recursive: true, force: true }); + }, + }; +} + +describeStorageDriver('LocalDriver', makeHarness); + +describe('LocalDriver specifics', () => { + const dirs: string[] = []; + const tmp = (): string => { + const d = makeTmpDir(); + dirs.push(d); + return d; + }; + + afterEach(() => { + vi.restoreAllMocks(); + while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); + }); + + it('realpaths a symlinked root and confines keys to the real directory', async () => { + // Docker symlinks BOTH storage anchors (/app/server/uploads → /app/uploads, + // /app/server/data → /app/data), so this is the deployed shape, not an edge. + const realdir = tmp(); + const link = path.join(tmp(), 'uploads-link'); + fs.symlinkSync(realdir, link); + + const driver = new LocalDriver({ id: 'sym', root: link }); + driver.init(); + await driver.put('files/x.bin', Readable.from('sym')); + + const local = driver.getLocalPath('files/x.bin'); + expect(local.startsWith(fs.realpathSync(realdir) + path.sep)).toBe(true); + expect(fs.readFileSync(local, 'utf8')).toBe('sym'); + expect(() => driver.getLocalPath('../escape')).toThrow(StorageInvalidKeyError); + }); + + it('cleans aged spool leftovers only at boot, never on reload-style init', () => { + const driver = new LocalDriver({ id: 'spool', root: tmp() }); + driver.init(); + const stray = path.join(driver.getSpoolDir(), 'leftover.part'); + fs.writeFileSync(stray, 'in-flight upload'); + // Crash leftover: old enough to clear the reap age gate. + const old = new Date(Date.now() - 2 * 60 * 60 * 1000); + fs.utimesSync(stray, old, old); + + driver.init(); // reload(): must NOT delete an in-flight upload's spool file + expect(fs.existsSync(stray)).toBe(true); + + driver.init({ cleanSpool: true }); // boot: crash leftovers are reclaimed + expect(fs.existsSync(stray)).toBe(false); + }); + + it('the boot spool sweep spares fresh entries (another process may be mid-upload)', () => { + const driver = new LocalDriver({ id: 'spool-fresh', root: tmp() }); + driver.init(); + const inflight = path.join(driver.getSpoolDir(), 'inflight.part'); + fs.writeFileSync(inflight, 'spooling right now'); + + driver.init({ cleanSpool: true }); + expect(fs.existsSync(inflight)).toBe(true); + }); + + it('creates its root, spool, and category prefix dirs at init', () => { + const root = path.join(tmp(), 'nested', 'uploads'); + const driver = new LocalDriver({ id: 'pfx', root }); + driver.init({ ensurePrefixes: ['files', 'photos/trek'] }); + + expect(fs.statSync(path.join(root, '.tmp')).isDirectory()).toBe(true); + expect(fs.statSync(path.join(root, 'files')).isDirectory()).toBe(true); + expect(fs.statSync(path.join(root, 'photos/trek')).isDirectory()).toBe(true); + }); + + it('falls back to copy+unlink when a temp-file rename crosses volumes (EXDEV)', async () => { + const root = tmp(); + const driver = new LocalDriver({ id: 'exdev', root }); + driver.init(); + const src = path.join(tmp(), 'other-volume.bin'); + fs.writeFileSync(src, 'exdev payload'); + + vi.spyOn(fs.promises, 'rename').mockRejectedValueOnce( + Object.assign(new Error('EXDEV: cross-device link not permitted'), { code: 'EXDEV' }), + ); + + await driver.put('files/moved.bin', { tmpPath: src }); + expect(fs.readFileSync(path.join(root, 'files/moved.bin'), 'utf8')).toBe('exdev payload'); + expect(fs.existsSync(src)).toBe(false); + }); + + it('commits stream puts via a same-volume spool rename', async () => { + const root = tmp(); + const driver = new LocalDriver({ id: 'samevol', root }); + driver.init(); + + const renameSpy = vi.spyOn(fs.promises, 'rename'); + await driver.put('files/spooled.bin', Readable.from('data')); + + const [from, to] = renameSpy.mock.calls.at(-1)!; + expect(String(from).startsWith(path.join(fs.realpathSync(root), '.tmp') + path.sep)).toBe(true); + expect(String(to)).toBe(path.join(fs.realpathSync(root), 'files/spooled.bin')); + }); + + it('lists a missing prefix as empty', async () => { + const driver = new LocalDriver({ id: 'empty', root: tmp() }); + driver.init(); + const seen: unknown[] = []; + for await (const stat of driver.list('nope/')) seen.push(stat); + expect(seen).toEqual([]); + }); +}); diff --git a/server/tests/unit/nest/storage/mirror-backfill.test.ts b/server/tests/unit/nest/storage/mirror-backfill.test.ts new file mode 100644 index 000000000..a57eaff41 --- /dev/null +++ b/server/tests/unit/nest/storage/mirror-backfill.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import { MirrorDriver, type ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver'; + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-backfill-')); + tmpDirs.push(dir); + return dir; +} +afterEach(() => { + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +function makeLocal(id: string): { driver: LocalDriver; root: string } { + const root = makeTmpDir(); + const driver = new LocalDriver({ id, root }); + driver.init({ ensurePrefixes: [], cleanSpool: false }); + return { driver, root }; +} + +async function put(driver: LocalDriver, key: string, body: string): Promise { + await driver.put(key, Readable.from(body)); +} + +function makeMirror(primary: LocalDriver, replicas: LocalDriver[], onReplicaFailure?: (f: ReplicaFailure) => void): MirrorDriver { + return new MirrorDriver({ id: 'm', primary, replicas, tempDir: () => makeTmpDir(), onReplicaFailure }); +} + +const HOOKS = { onProgress: () => undefined, isCancelled: () => false }; + +function emptyAsyncIterable(): AsyncIterable { + return { + [Symbol.asyncIterator]() { + return { next: () => Promise.resolve({ done: true, value: undefined }) }; + }, + }; +} + +describe('MirrorDriver.backfill', () => { + it('BKFL-001 copies missing objects to every replica, scoped to the given prefixes', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: replica, root: replicaRoot } = makeLocal('r'); + await put(primary, 'backups/a.zip', 'aaa'); + await put(primary, 'covers/skip.jpg', 'not in scope'); + const result = await makeMirror(primary, [replica]).backfill(['backups/'], HOOKS); + expect(result).toMatchObject({ done: 1, total: 1, copied: 1, skipped: 0, failed: 0, deleted: 0, cancelled: false }); + expect(fs.readFileSync(path.join(replicaRoot, 'backups/a.zip'), 'utf8')).toBe('aaa'); + expect(fs.existsSync(path.join(replicaRoot, 'covers/skip.jpg'))).toBe(false); // scope rule + }); + + it('BKFL-002 skips size-matched objects and re-copies size mismatches', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: replica, root: replicaRoot } = makeLocal('r'); + await put(primary, 'backups/same.zip', 'equal'); + await put(replica, 'backups/same.zip', 'eq2al'); // same size → skipped + await put(primary, 'backups/diff.zip', 'longer-content'); + await put(replica, 'backups/diff.zip', 'short'); // size mismatch → re-copied + const result = await makeMirror(primary, [replica]).backfill(['backups/'], HOOKS); + expect(result).toMatchObject({ done: 2, copied: 1, skipped: 1, failed: 0, deleted: 0 }); + expect(fs.readFileSync(path.join(replicaRoot, 'backups/diff.zip'), 'utf8')).toBe('longer-content'); + expect(fs.readFileSync(path.join(replicaRoot, 'backups/same.zip'), 'utf8')).toBe('eq2al'); // untouched + }); + + it('BKFL-003 a failing replica reports through the failure hook, counts failed, and the run continues', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: good, root: goodRoot } = makeLocal('good'); + const bad = { + id: 'bad', + stat: vi.fn().mockResolvedValue(null), + put: vi.fn().mockRejectedValue(new Error('replica on fire')), + list: () => emptyAsyncIterable(), // holds nothing itself — keeps the sweep phase a no-op for this mock + } as unknown as LocalDriver; + const failures: ReplicaFailure[] = []; + await put(primary, 'backups/a.zip', 'aaa'); + await put(primary, 'backups/b.zip', 'bbb'); + const result = await makeMirror(primary, [bad, good], (f) => failures.push(f)).backfill(['backups/'], HOOKS); + expect(result).toMatchObject({ done: 2, copied: 2, failed: 2, deleted: 0, cancelled: false }); // good got both; bad failed both + expect(failures).toHaveLength(2); + expect(failures[0]).toMatchObject({ backend: 'bad', op: 'put' }); + expect(failures[0]!.error).toContain('replica on fire'); + expect(fs.existsSync(path.join(goodRoot, 'backups/b.zip'))).toBe(true); + }); + + it('BKFL-004 cancel stops after the in-flight key', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: replica } = makeLocal('r'); + for (let i = 0; i < 5; i++) await put(primary, `backups/${i}.zip`, `content-${i}`); + let examined = 0; + const hooks = { + onProgress: (p: { done: number }) => { + examined = p.done; + }, + isCancelled: () => examined >= 2, + }; + const result = await makeMirror(primary, [replica]).backfill(['backups/'], hooks); + expect(result.cancelled).toBe(true); + expect(result.done).toBeLessThan(5); + }); + + it('BKFL-005 a primary error aborts (propagates) instead of being swallowed', async () => { + const { driver: replica } = makeLocal('r'); + const explodingPrimary = { + id: 'p', + list: () => ({ + [Symbol.asyncIterator]() { + return { + next: () => Promise.reject(new Error('primary listing failed')), + }; + }, + }), + } as unknown as LocalDriver; + await expect(makeMirror(explodingPrimary, [replica]).backfill(['backups/'], HOOKS)).rejects.toThrow( + 'primary listing failed', + ); + }); + + it('BKFL-006 the sweep deletes replica keys absent from the primary key-set', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: replica, root: replicaRoot } = makeLocal('r'); + await put(primary, 'backups/a.zip', 'aaa'); + await put(replica, 'backups/a.zip', 'aaa'); // matches the primary — untouched + await put(replica, 'backups/stale.zip', 'gone'); // absent from the primary — swept + const result = await makeMirror(primary, [replica]).backfill(['backups/'], HOOKS); + expect(result).toMatchObject({ deleted: 1, failed: 0, cancelled: false }); + expect(fs.existsSync(path.join(replicaRoot, 'backups/stale.zip'))).toBe(false); + expect(fs.existsSync(path.join(replicaRoot, 'backups/a.zip'))).toBe(true); + }); + + it('BKFL-007 the race gate spares a replica key the primary regained since the copy loop', async () => { + const { driver: primary } = makeLocal('p'); + const { driver: replica, root: replicaRoot } = makeLocal('r'); + await put(replica, 'backups/regained.zip', 'back'); // on the replica only, per the copy loop's enumeration + // Simulate the primary regaining the object between the copy loop and the + // sweep phase: list() never yields it (so it never enters the primary + // key-set), but stat() — the sweep's race gate — reports it exists. + const wrapped = { + id: primary.id, + list: (prefix: string) => primary.list(prefix), + getStream: (key: string, range?: unknown) => (primary.getStream as (k: string, r?: unknown) => unknown)(key, range), + stat: async (key: string) => + key === 'backups/regained.zip' ? { key, size: 4, mtimeMs: Date.now() } : primary.stat(key), + } as unknown as LocalDriver; + const result = await makeMirror(wrapped, [replica]).backfill(['backups/'], HOOKS); + expect(result).toMatchObject({ deleted: 0, failed: 0, cancelled: false }); + expect(fs.existsSync(path.join(replicaRoot, 'backups/regained.zip'))).toBe(true); // spared by the race gate + }); + + it('BKFL-008 a replica list failure during the sweep is reported, skips that replica, and the run continues', async () => { + const { driver: primary } = makeLocal('p'); + await put(primary, 'backups/a.zip', 'aaa'); + const bad = { + id: 'bad', + stat: vi.fn().mockResolvedValue(null), + put: vi.fn().mockResolvedValue(undefined), + list: () => { + throw new Error('replica listing failed'); + }, + } as unknown as LocalDriver; + const { driver: good, root: goodRoot } = makeLocal('good'); + await put(good, 'backups/stale.zip', 'stale'); // extra on the good replica — still swept once bad is skipped + const failures: ReplicaFailure[] = []; + const result = await makeMirror(primary, [bad, good], (f) => failures.push(f)).backfill(['backups/'], HOOKS); + expect(failures.some((f) => f.backend === 'bad' && f.op === 'list')).toBe(true); + expect(fs.existsSync(path.join(goodRoot, 'backups/stale.zip'))).toBe(false); // the good replica's sweep still ran + expect(result.deleted).toBe(1); + }); + + it('BKFL-009 cancel mid-sweep stops after the in-flight key', async () => { + const { driver: primary } = makeLocal('p'); // empty — every replica key is a sweep candidate + const { driver: replica } = makeLocal('r'); + for (let i = 0; i < 5; i++) await put(replica, `backups/stale-${i}.zip`, `x`); + let deleted = 0; + const hooks = { + onProgress: (p: { deleted: number }) => { + deleted = p.deleted; + }, + isCancelled: () => deleted >= 2, + }; + const result = await makeMirror(primary, [replica]).backfill(['backups/'], hooks); + expect(result.cancelled).toBe(true); + expect(result.deleted).toBeGreaterThanOrEqual(2); + expect(result.deleted).toBeLessThan(5); + }); +}); diff --git a/server/tests/unit/nest/storage/mirror.driver.test.ts b/server/tests/unit/nest/storage/mirror.driver.test.ts new file mode 100644 index 000000000..5b5274198 --- /dev/null +++ b/server/tests/unit/nest/storage/mirror.driver.test.ts @@ -0,0 +1,343 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import { + MirrorDriver, + type BackfillHooks, + type ReplicaFailure, +} from '../../../../src/nest/storage/drivers/mirror.driver'; +import { + StorageBackendError, + StorageNotFoundError, + type StorageDriver, +} from '../../../../src/nest/storage/storage.types'; +import { describeStorageDriver, type DriverHarness } from './storage-driver.contract'; + +function makeTmpDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'trek-mirror-')); +} + +function makeLocal(id: string): { driver: LocalDriver; root: string } { + const root = makeTmpDir(); + const driver = new LocalDriver({ id, root }); + driver.init({ cleanSpool: true }); + return { driver, root }; +} + +interface MirrorFixture { + mirror: MirrorDriver; + primary: LocalDriver; + replica: LocalDriver; + spool: string; + failures: ReplicaFailure[]; + dirs: string[]; +} + +function makeMirror(): MirrorFixture { + const p = makeLocal('primary-local'); + const r = makeLocal('replica-local'); + const spool = makeTmpDir(); + const failures: ReplicaFailure[] = []; + const mirror = new MirrorDriver({ + id: 'backup-mirror', + primary: p.driver, + replicas: [r.driver], + tempDir: () => spool, + onReplicaFailure: (f) => failures.push(f), + }); + return { mirror, primary: p.driver, replica: r.driver, spool, failures, dirs: [p.root, r.root, spool] }; +} + +async function makeHarness(): Promise { + const fx = makeMirror(); + const tmpFiles = makeTmpDir(); + let counter = 0; + return { + driver: fx.mirror, + makeTempFile: async (contents) => { + const p = path.join(tmpFiles, `src-${counter++}.tmp`); + fs.writeFileSync(p, contents); + return p; + }, + cleanup: async () => { + for (const dir of [...fx.dirs, tmpFiles]) fs.rmSync(dir, { recursive: true, force: true }); + }, + }; +} + +describeStorageDriver('MirrorDriver', makeHarness); + +async function readAll(driver: StorageDriver, key: string): Promise { + const { stream } = await driver.getStream(key); + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk as Buffer)); + return Buffer.concat(chunks).toString(); +} + +describe('MirrorDriver specifics', () => { + const cleanups: string[] = []; + const track = (fx: MirrorFixture): MirrorFixture => { + cleanups.push(...fx.dirs); + return fx; + }; + + afterEach(() => { + vi.restoreAllMocks(); + while (cleanups.length) fs.rmSync(cleanups.pop()!, { recursive: true, force: true }); + }); + + it('lands a stream put in the primary AND every replica', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-1.zip', Readable.from('zip bytes')); + expect(await readAll(fx.primary, 'backup-1.zip')).toBe('zip bytes'); + expect(await readAll(fx.replica, 'backup-1.zip')).toBe('zip bytes'); + }); + + it('resolves put and reports when a replica write fails', async () => { + const fx = track(makeMirror()); + vi.spyOn(fx.replica, 'put').mockRejectedValueOnce(new StorageBackendError('replica disk full')); + + await expect(fx.mirror.put('backup-2.zip', Readable.from('payload'))).resolves.toBeUndefined(); + + expect(await readAll(fx.primary, 'backup-2.zip')).toBe('payload'); + expect(fx.failures).toHaveLength(1); + expect(fx.failures[0]).toMatchObject({ backend: 'replica-local', key: 'backup-2.zip', op: 'put' }); + expect(fx.failures[0].error).toContain('replica disk full'); + }); + + it("reports a replica failure with the wrapped cause chain, not just the wrapper's message", async () => { + const fx = track(makeMirror()); + const raw = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:9000'), { code: 'ECONNREFUSED' }); + vi.spyOn(fx.replica, 'put').mockRejectedValueOnce( + new StorageBackendError("put failed for 'backup-3.zip' on 's3'", raw), + ); + + await expect(fx.mirror.put('backup-3.zip', Readable.from('payload'))).resolves.toBeUndefined(); + + expect(fx.failures).toHaveLength(1); + expect(fx.failures[0].error).toContain("put failed for 'backup-3.zip' on 's3'"); + expect(fx.failures[0].error).toContain('ECONNREFUSED'); + }); + + it('rejects put and never touches replicas when the primary write fails', async () => { + const fx = track(makeMirror()); + vi.spyOn(fx.primary, 'put').mockRejectedValueOnce(new StorageBackendError('primary gone')); + const replicaPut = vi.spyOn(fx.replica, 'put'); + + await expect(fx.mirror.put('backup-3.zip', Readable.from('x'))).rejects.toBeInstanceOf(StorageBackendError); + expect(replicaPut).not.toHaveBeenCalled(); + expect(fx.failures).toHaveLength(0); + }); + + it('falls back to a replica when the primary read ERRORS', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-4.zip', Readable.from('mirrored')); + vi.spyOn(fx.primary, 'getStream').mockRejectedValueOnce(new StorageBackendError('primary io error')); + + expect(await readAll(fx.mirror, 'backup-4.zip')).toBe('mirrored'); + }); + + it('falls back to a replica on a plain primary miss (binding decision: a read succeeds if ANY member holds it)', async () => { + const fx = track(makeMirror()); + // Plant the object only on the replica: per the widened invariant a + // primary miss now DOES consult replicas, for both getStream and stat. + await fx.replica.put('only-on-replica.zip', Readable.from('ghost')); + const replicaGet = vi.spyOn(fx.replica, 'getStream'); + + expect(await readAll(fx.mirror, 'only-on-replica.zip')).toBe('ghost'); + expect(replicaGet).toHaveBeenCalled(); + expect((await fx.mirror.stat('only-on-replica.zip'))?.size).toBe('ghost'.length); + }); + + it('stays a genuine miss when neither the primary nor any replica holds the object', async () => { + const fx = track(makeMirror()); + await expect(fx.mirror.getStream('nowhere.zip')).rejects.toBeInstanceOf(StorageNotFoundError); + expect(await fx.mirror.stat('nowhere.zip')).toBeNull(); + }); + + it('falls back for stat and list on primary error', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-5.zip', Readable.from('12345')); + + vi.spyOn(fx.primary, 'stat').mockRejectedValueOnce(new StorageBackendError('io')); + expect((await fx.mirror.stat('backup-5.zip'))!.size).toBe(5); + + vi.spyOn(fx.primary, 'list').mockImplementationOnce(() => { + throw new StorageBackendError('io'); + }); + const keys: string[] = []; + for await (const stat of fx.mirror.list('')) keys.push(stat.key); + expect(keys).toEqual(['backup-5.zip']); + }); + + it('fans delete out to all targets, tolerating a miss on either side', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-6.zip', Readable.from('bye')); + await fx.replica.delete('backup-6.zip'); // already gone on the replica + + await fx.mirror.delete('backup-6.zip'); + expect(await fx.primary.stat('backup-6.zip')).toBeNull(); + expect(await fx.replica.stat('backup-6.zip')).toBeNull(); + expect(fx.failures).toHaveLength(0); // idempotent miss is not a failure + }); + + it('has no spool dir of its own and delegates getLocalPath to the primary', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-7.zip', Readable.from('here')); + + expect(fx.mirror.getSpoolDir()).toBeNull(); + const local = fx.mirror.getLocalPath('backup-7.zip'); + expect(local).toBe(fx.primary.getLocalPath('backup-7.zip')); + }); + + it('leaves no spool leftovers in tempDir on success or failure', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('backup-8.zip', Readable.from('ok')); + expect(fs.readdirSync(fx.spool)).toEqual([]); + + vi.spyOn(fx.primary, 'put').mockRejectedValueOnce(new StorageBackendError('nope')); + await expect(fx.mirror.put('backup-9.zip', Readable.from('fail'))).rejects.toThrow(); + expect(fs.readdirSync(fx.spool)).toEqual([]); + }); +}); + +describe('MirrorDriver.stat null-continue fallback loop', () => { + interface TriFixture { + mirror: MirrorDriver; + primary: LocalDriver; + replicaA: LocalDriver; + replicaB: LocalDriver; + dirs: string[]; + } + + function makeTriMirror(): TriFixture { + const p = makeLocal('primary-local'); + const a = makeLocal('replica-a'); + const b = makeLocal('replica-b'); + const spool = makeTmpDir(); + const mirror = new MirrorDriver({ + id: 'tri-mirror', + primary: p.driver, + replicas: [a.driver, b.driver], + tempDir: () => spool, + }); + return { mirror, primary: p.driver, replicaA: a.driver, replicaB: b.driver, dirs: [p.root, a.root, b.root, spool] }; + } + + const cleanups: string[] = []; + afterEach(() => { + vi.restoreAllMocks(); + while (cleanups.length) fs.rmSync(cleanups.pop()!, { recursive: true, force: true }); + }); + + it('a replica returning null does not short-circuit the search — the next replica still gets a look', async () => { + const fx = makeTriMirror(); + cleanups.push(...fx.dirs); + await fx.replicaB.put('deep.zip', Readable.from('found on b')); + vi.spyOn(fx.primary, 'stat').mockRejectedValueOnce(new StorageBackendError('primary io error')); + const statA = vi.spyOn(fx.replicaA, 'stat'); + + const result = await fx.mirror.stat('deep.zip'); + + expect(result?.size).toBe('found on b'.length); + expect(statA).toHaveBeenCalled(); // replicaA WAS consulted (and missed) before replicaB answered + }); + + it('a replica that itself errors during the search is treated as a miss, not a hard failure', async () => { + const fx = makeTriMirror(); + cleanups.push(...fx.dirs); + await fx.replicaB.put('deep-2.zip', Readable.from('found on b again')); + vi.spyOn(fx.primary, 'stat').mockRejectedValueOnce(new StorageBackendError('primary io error')); + vi.spyOn(fx.replicaA, 'stat').mockRejectedValueOnce(new StorageBackendError('replica-a unreachable')); + + const result = await fx.mirror.stat('deep-2.zip'); + + expect(result?.size).toBe('found on b again'.length); + }); + + it('rethrows the remembered primary error only when every replica also misses', async () => { + const fx = makeTriMirror(); + cleanups.push(...fx.dirs); + const boom = new StorageBackendError('primary io error'); + vi.spyOn(fx.primary, 'stat').mockRejectedValueOnce(boom); + + await expect(fx.mirror.stat('nowhere.zip')).rejects.toBe(boom); + }); +}); + +describe('MirrorDriver.backfill polish', () => { + const cleanups: string[] = []; + + afterEach(() => { + vi.restoreAllMocks(); + while (cleanups.length) fs.rmSync(cleanups.pop()!, { recursive: true, force: true }); + }); + + function setup(): { + mirror: MirrorDriver; + primary: LocalDriver; + replica: StorageDriver; + primaryRoot: string; + failures: ReplicaFailure[]; + } { + const p = makeLocal('primary-local'); + const r = makeLocal('replica-local'); + const spool = makeTmpDir(); + const failures: ReplicaFailure[] = []; + const mirror = new MirrorDriver({ + id: 'backup-mirror', + primary: p.driver, + replicas: [r.driver], + tempDir: () => spool, + onReplicaFailure: (f) => failures.push(f), + }); + cleanups.push(p.root, r.root, spool); + return { mirror, primary: p.driver, replica: r.driver, primaryRoot: p.root, failures }; + } + + function hooks(): BackfillHooks { + return { onProgress: () => undefined, isCancelled: () => false }; + } + + it('MIRROR-BF-020 backfill puts carry a contentType derived from the key', async () => { + const fx = setup(); + await fx.primary.put('files/a.pdf', Readable.from('pdf bytes')); + const putSpy = vi.spyOn(fx.replica, 'put'); + + await fx.mirror.backfill(['files/'], hooks()); + + const call = putSpy.mock.calls.find(([key]) => key === 'files/a.pdf')!; + expect(call[2]).toEqual({ contentType: 'application/pdf' }); + }); + + it('MIRROR-BF-021 a replica STAT failure reports op stat, not put', async () => { + const fx = setup(); + await fx.primary.put('files/a.pdf', Readable.from('pdf bytes')); + vi.spyOn(fx.replica, 'stat').mockRejectedValueOnce(new Error('probe down')); + + await fx.mirror.backfill(['files/'], hooks()); + + expect(fx.failures.some((f) => f.op === 'stat')).toBe(true); + }); + + it('MIRROR-BF-022 done never exceeds total, and total settles to done on completion', async () => { + const fx = setup(); + await fx.primary.put('files/a.txt', Readable.from('a')); + await fx.primary.put('files/b.txt', Readable.from('b')); + + const snapshots: Array<{ done: number; total: number }> = []; + const result = await fx.mirror.backfill(['files/'], { + onProgress: (p) => { + snapshots.push({ done: p.done, total: p.total }); + if (snapshots.length === 1) fs.writeFileSync(path.join(fx.primaryRoot, 'files', 'late.txt'), 'x'); + }, + isCancelled: () => false, + }); + + for (const s of snapshots) expect(s.done).toBeLessThanOrEqual(s.total); + expect(result.total).toBe(result.done); + }); +}); diff --git a/server/tests/unit/nest/storage/s3.driver.contract.test.ts b/server/tests/unit/nest/storage/s3.driver.contract.test.ts new file mode 100644 index 000000000..b187058c6 --- /dev/null +++ b/server/tests/unit/nest/storage/s3.driver.contract.test.ts @@ -0,0 +1,95 @@ +import { describe, beforeAll, it, expect } from 'vitest'; +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import awsLite from '@aws-lite/client'; +import { MULTIPART_THRESHOLD, S3Driver } from '../../../../src/nest/storage/drivers/s3.driver'; +import { describeStorageDriver, type DriverHarness } from './storage-driver.contract'; + +/** + * Env-gated: engages only when TREK_TEST_S3_* is set (manual-only, via + * `docker compose -f docker-compose.minio-test.yml up`). Local runs stay + * hermetic. Deviation from local/mirror's bare top-level call: the gate must + * wrap describeStorageDriver because the contract file owns its own describe. + */ +const endpoint = process.env.TREK_TEST_S3_ENDPOINT; +const bucket = process.env.TREK_TEST_S3_BUCKET ?? 'trek-test'; +const accessKeyId = process.env.TREK_TEST_S3_ACCESS_KEY_ID ?? 'trekci'; +const secretAccessKey = process.env.TREK_TEST_S3_SECRET_ACCESS_KEY ?? 'trekci-secret'; + +describe.skipIf(!endpoint)('S3Driver against MinIO', () => { + beforeAll(async () => { + // Idempotent bucket setup — raw client, driver stays bucket-agnostic. + const client = await awsLite({ + accessKeyId, + secretAccessKey, + region: 'us-east-1', + endpoint: endpoint!, + retries: 1, + keepAlive: false, + plugins: [await import('@aws-lite/s3')], + }); + await client.S3.CreateBucket({ Bucket: bucket }).catch((err: { code?: string; statusCode?: number }) => { + if (err.code !== 'BucketAlreadyOwnedByYou' && err.statusCode !== 409) throw err; + }); + }); + + async function makeHarness(): Promise { + // Fresh namespace per test (makeHarness runs in beforeEach) — reruns + // against a shared MinIO never collide; cleanup removes everything. + const keyPrefix = `contract-${randomUUID()}`; + const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'trek-s3-contract-')); + const driver = new S3Driver({ + id: 's3-contract', + endpoint: endpoint!, + region: 'us-east-1', + bucket, + keyPrefix, + accessKeyId, + secretAccessKey, + retries: 1, + timeoutMs: 10000, + keepAlive: false, // vitest forks must exit promptly + }); + let counter = 0; + return { + driver, + async makeTempFile(contents) { + const p = path.join(tmpDir, `src-${counter++}.bin`); + await fs.promises.writeFile(p, contents); + return p; + }, + async cleanup() { + for await (const stat of driver.list('')) await driver.delete(stat.key); + await fs.promises.rm(tmpDir, { recursive: true, force: true }); + }, + }; + } + + describeStorageDriver('S3Driver', makeHarness); + + // The generic contract suite never crosses MULTIPART_THRESHOLD, so the + // aws-lite Upload path only runs HERE against a real server. A contentType + // is load-bearing: upstream Upload leaks it into its CompleteMultipartUpload + // headers, which (unworked-around) turns the parts XML into JSON and draws + // 400 MalformedXML — the exact shape of every real backup/photo put. + it( + 'round-trips a multipart put carrying a contentType (upstream MalformedXML regression)', + async () => { + const harness = await makeHarness(); + try { + const chunk = Buffer.alloc(MULTIPART_THRESHOLD / 2 + 1024 * 1024, 9); + await harness.driver.put('mp/big.zip', Readable.from([chunk, chunk, chunk]), { + contentType: 'application/zip', + }); + const stat = await harness.driver.stat('mp/big.zip'); + expect(stat?.size).toBe(3 * chunk.length); + } finally { + await harness.cleanup(); + } + }, + 60_000, + ); +}); diff --git a/server/tests/unit/nest/storage/s3.driver.test.ts b/server/tests/unit/nest/storage/s3.driver.test.ts new file mode 100644 index 000000000..e5e4c9670 --- /dev/null +++ b/server/tests/unit/nest/storage/s3.driver.test.ts @@ -0,0 +1,500 @@ +import { describe, it, expect, vi } from 'vitest'; +import { Readable } from 'node:stream'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + S3Driver, + MULTIPART_THRESHOLD, + defaultClientFactory, + type S3Api, + type S3DriverOptions, +} from '../../../../src/nest/storage/drivers/s3.driver'; +import { + StorageBackendError, + StorageInvalidKeyError, + StorageNotFoundError, +} from '../../../../src/nest/storage/storage.types'; + +/** Mock client: every method rejects unless a test overrides it. */ +function makeMockApi(overrides: Partial = {}): S3Api { + const unexpected = (name: string) => vi.fn().mockRejectedValue(new Error(`unexpected ${name}`)); + return { + PutObject: unexpected('PutObject'), + Upload: unexpected('Upload'), + GetObject: unexpected('GetObject'), + HeadObject: unexpected('HeadObject'), + DeleteObject: unexpected('DeleteObject'), + ListObjectsV2: unexpected('ListObjectsV2'), + ...overrides, + } as S3Api; +} + +function makeDriver(api: S3Api, opts: Partial = {}): S3Driver { + return new S3Driver({ + id: 's3-test', + endpoint: 'http://127.0.0.1:9000', + region: 'us-east-1', + bucket: 'trek', + keyPrefix: '', + accessKeyId: 'ak', + secretAccessKey: 'sk', + retries: 1, + timeoutMs: 200, + clientFactory: async () => api, + ...opts, + }); +} + +/** aws-lite error shape: real Error + statusCode/code metadata. */ +function awsError(statusCode: number | undefined, code?: string): Error { + const err = new Error(`@aws-lite/client: S3: ${code ?? 'boom'}`) as Error & { + statusCode?: number; + code?: string; + }; + err.statusCode = statusCode; + if (code) err.code = code; + return err; +} + +describe('S3Driver construction', () => { + it('normalizes the key prefix to "" or "segments/" form', async () => { + const api = makeMockApi({ HeadObject: vi.fn().mockResolvedValue({ ContentLength: 1, LastModified: new Date(1000) }) }); + await makeDriver(api, { keyPrefix: '/trek/prod/' }).stat('a/x.bin'); + expect(api.HeadObject).toHaveBeenCalledWith(expect.objectContaining({ Key: 'trek/prod/a/x.bin' })); + }); + it('rejects an invalid key prefix at construction', () => { + expect(() => makeDriver(makeMockApi(), { keyPrefix: '../evil' })).toThrow(StorageInvalidKeyError); + }); + it('makes no client call at construction (boot must not touch the network)', () => { + const factory = vi.fn(); + void new S3Driver({ + id: 's3-test', endpoint: 'http://127.0.0.1:9000', region: 'us-east-1', bucket: 'trek', + keyPrefix: '', accessKeyId: 'ak', secretAccessKey: 'sk', retries: 1, timeoutMs: 200, + clientFactory: factory, + }); + expect(factory).not.toHaveBeenCalled(); + }); +}); + +describe('S3Driver stat/delete', () => { + it('maps HeadObject to ObjectStat', async () => { + const api = makeMockApi({ HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000) }) }); + expect(await makeDriver(api).stat('a/x.bin')).toEqual({ key: 'a/x.bin', size: 42, mtimeMs: 5000 }); + }); + it('surfaces the bucket ETag verbatim (quotes included) so the serving path can use it as a validator', async () => { + const api = makeMockApi({ + HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000), ETag: '"d41d8cd98f00b204"' }), + }); + expect(await makeDriver(api).stat('a/x.bin')).toEqual({ + key: 'a/x.bin', + size: 42, + mtimeMs: 5000, + etag: '"d41d8cd98f00b204"', + }); + }); + it('re-quotes the unquoted ETag aws-lite hands back (a bare tag is not a valid HTTP ETag)', async () => { + // Verified live against AIStor: the SDK strips the surrounding quotes. + const api = makeMockApi({ + HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000), ETag: 'a925576942e9' }), + }); + expect((await makeDriver(api).stat('a/x.bin'))!.etag).toBe('"a925576942e9"'); + }); + it('leaves an already-weak tag alone and an absent/empty ETag undefined', async () => { + const weak = makeMockApi({ + HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000), ETag: 'W/"abc"' }), + }); + expect((await makeDriver(weak).stat('a/x.bin'))!.etag).toBe('W/"abc"'); + + const none = makeMockApi({ HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000) }) }); + expect((await makeDriver(none).stat('a/x.bin'))!.etag).toBeUndefined(); + + const blank = makeMockApi({ + HeadObject: vi.fn().mockResolvedValue({ ContentLength: 42, LastModified: new Date(5000), ETag: '' }), + }); + expect((await makeDriver(blank).stat('a/x.bin'))!.etag).toBeUndefined(); + }); + it('stats a 404/NotFound as null', async () => { + const api = makeMockApi({ HeadObject: vi.fn().mockRejectedValue(awsError(404, 'NotFound')) }); + expect(await makeDriver(api).stat('a/x.bin')).toBeNull(); + }); + it('wraps network errors (no statusCode) in StorageBackendError with cause', async () => { + const cause = awsError(undefined); + const api = makeMockApi({ HeadObject: vi.fn().mockRejectedValue(cause) }); + await expect(makeDriver(api).stat('a/x.bin')).rejects.toMatchObject({ + name: 'StorageBackendError', + cause, + }); + }); + it('deletes idempotently — a 404 resolves', async () => { + const api = makeMockApi({ DeleteObject: vi.fn().mockRejectedValue(awsError(404, 'NoSuchKey')) }); + await expect(makeDriver(api).delete('a/x.bin')).resolves.toBeUndefined(); + }); + it('validates keys before any client call', async () => { + const api = makeMockApi(); + const driver = makeDriver(api); + for (const bad of ['../x', '/abs', 'a\\b', '.tmp/x']) { + await expect(driver.stat(bad)).rejects.toBeInstanceOf(StorageInvalidKeyError); + await expect(driver.delete(bad)).rejects.toBeInstanceOf(StorageInvalidKeyError); + } + expect(api.HeadObject).not.toHaveBeenCalled(); + expect(api.DeleteObject).not.toHaveBeenCalled(); + }); + it('expires a hung call after timeoutMs with StorageBackendError', async () => { + const api = makeMockApi({ HeadObject: vi.fn().mockReturnValue(new Promise(() => {})) }); + await expect(makeDriver(api, { timeoutMs: 50 }).stat('a/x.bin')).rejects.toBeInstanceOf(StorageBackendError); + }); + it('caches the client across calls and retries a failed factory', async () => { + const api = makeMockApi({ HeadObject: vi.fn().mockResolvedValue({ ContentLength: 1, LastModified: new Date(0) }) }); + const factory = vi + .fn() + .mockRejectedValueOnce(new Error('init boom')) + .mockResolvedValue(api); + const driver = makeDriver(api, { clientFactory: factory }); + await expect(driver.stat('a/x.bin')).rejects.toBeInstanceOf(StorageBackendError); + await driver.stat('a/x.bin'); // second call retries the factory + await driver.stat('a/x.bin'); + expect(factory).toHaveBeenCalledTimes(2); + }); +}); + +async function drain(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Buffer)); + return Buffer.concat(chunks); +} + +describe('S3Driver getStream', () => { + it('streams the body with streamResponsePayload and returns the stat', async () => { + const api = makeMockApi({ + GetObject: vi.fn().mockResolvedValue({ + Body: Readable.from('hello'), + ContentLength: 5, + LastModified: new Date(7000), + }), + }); + const { stream, stat } = await makeDriver(api).getStream('a/x.bin'); + expect((await drain(stream)).toString()).toBe('hello'); + expect(stat).toEqual({ key: 'a/x.bin', size: 5, mtimeMs: 7000 }); + expect(api.GetObject).toHaveBeenCalledWith( + expect.objectContaining({ Key: 'a/x.bin', streamResponsePayload: true }), + ); + expect((api.GetObject as ReturnType).mock.calls[0][0]).not.toHaveProperty('Range'); + }); + it('formats closed and open-ended ranges and returns the FULL-object stat from Content-Range', async () => { + const api = makeMockApi({ + GetObject: vi.fn().mockResolvedValue({ + Body: Readable.from('2345'), + ContentLength: 4, // the RANGE length — must not become stat.size + ContentRange: 'bytes 2-5/10', + LastModified: new Date(7000), + }), + }); + const { stat } = await makeDriver(api).getStream('a/x.bin', { start: 2, end: 5 }); + expect(stat.size).toBe(10); + expect(api.GetObject).toHaveBeenCalledWith(expect.objectContaining({ Range: 'bytes=2-5' })); + + await makeDriver(api).getStream('a/x.bin', { start: 7 }); + expect(api.GetObject).toHaveBeenLastCalledWith(expect.objectContaining({ Range: 'bytes=7-' })); + }); + it('surfaces the ETag on a plain get, and the FULL-object ETag on a ranged one', async () => { + const api = makeMockApi({ + GetObject: vi.fn().mockResolvedValue({ + Body: Readable.from('hello'), + ContentLength: 5, + LastModified: new Date(7000), + ETag: '"abc123"', + }), + }); + expect((await makeDriver(api).getStream('a/x.bin')).stat.etag).toBe('"abc123"'); + + const ranged = makeMockApi({ + GetObject: vi.fn().mockResolvedValue({ + Body: Readable.from('23'), + ContentLength: 2, + ContentRange: 'bytes 2-3/10', + LastModified: new Date(7000), + ETag: '"abc123"', + }), + }); + // S3 answers a 206 with the whole object's tag — the validator a resuming + // If-Range client sent us in the first place. + expect((await makeDriver(ranged).getStream('a/x.bin', { start: 2, end: 3 })).stat).toEqual({ + key: 'a/x.bin', + size: 10, + mtimeMs: 7000, + etag: '"abc123"', + }); + }); + it('maps 404/NoSuchKey to StorageNotFoundError', async () => { + const api = makeMockApi({ GetObject: vi.fn().mockRejectedValue(awsError(404, 'NoSuchKey')) }); + await expect(makeDriver(api).getStream('a/x.bin')).rejects.toBeInstanceOf(StorageNotFoundError); + }); + it('rejects a missing Body as StorageBackendError (never a bare undefined stream)', async () => { + const api = makeMockApi({ GetObject: vi.fn().mockResolvedValue({ ContentLength: 5 }) }); + await expect(makeDriver(api).getStream('a/x.bin')).rejects.toBeInstanceOf(StorageBackendError); + }); +}); + +describe('S3Driver list', () => { + function pages(...pageContents: Array>) { + return vi.fn().mockResolvedValue( + (async function* () { + for (const Contents of pageContents) yield { Contents }; + })(), + ); + } + it('paginates with the iterator, strips the keyPrefix, and requests the combined prefix', async () => { + const api = makeMockApi({ + ListObjectsV2: pages( + [{ Key: 'trek/prod/a/one.bin', Size: 2, LastModified: new Date(1000) }], + [{ Key: 'trek/prod/a/sub/two.bin', Size: 3, LastModified: new Date(2000) }], + ), + }); + const out = []; + for await (const stat of makeDriver(api, { keyPrefix: 'trek/prod' }).list('a/')) out.push(stat); + expect(out).toEqual([ + { key: 'a/one.bin', size: 2, mtimeMs: 1000 }, + { key: 'a/sub/two.bin', size: 3, mtimeMs: 2000 }, + ]); + expect(api.ListObjectsV2).toHaveBeenCalledWith( + expect.objectContaining({ Prefix: 'trek/prod/a/', paginate: 'iterator' }), + ); + }); + it('skips keys with dot-segments and keys outside the keyPrefix (out-of-band writes)', async () => { + const api = makeMockApi({ + ListObjectsV2: pages([ + { Key: 'trek/prod/a/ok.bin', Size: 1, LastModified: new Date(0) }, + { Key: 'trek/prod/.tmp/junk', Size: 1, LastModified: new Date(0) }, + { Key: 'trek/prod/a/.hidden', Size: 1, LastModified: new Date(0) }, + { Key: 'elsewhere/x.bin', Size: 1, LastModified: new Date(0) }, + ]), + }); + const out = []; + for await (const stat of makeDriver(api, { keyPrefix: 'trek/prod' }).list('')) out.push(stat); + expect(out.map((s) => s.key)).toEqual(['a/ok.bin']); + }); + it('yields nothing for an empty result and validates the prefix first', async () => { + const api = makeMockApi({ ListObjectsV2: pages([]) }); + const driver = makeDriver(api); + const out = []; + for await (const stat of driver.list('a/')) out.push(stat); + expect(out).toEqual([]); + await expect(async () => { + for await (const s of driver.list('../x')) void s; + }).rejects.toBeInstanceOf(StorageInvalidKeyError); + }); +}); + +describe('defaultClientFactory', () => { + it('builds a real aws-lite client exposing the S3 surface (no network at init)', async () => { + const s3 = await defaultClientFactory({ + endpoint: 'http://127.0.0.1:1', region: 'us-east-1', + accessKeyId: 'ak', secretAccessKey: 'sk', retries: 0, keepAlive: false, + }); + for (const method of ['PutObject', 'Upload', 'GetObject', 'HeadObject', 'DeleteObject', 'ListObjectsV2'] as const) { + expect(typeof s3[method]).toBe('function'); + } + }); +}); + +describe('S3Driver put — bounded peek routing', () => { + it('routes a small stream (ends within threshold) to a single PutObject with buffered Body', async () => { + const api = makeMockApi({ PutObject: vi.fn().mockResolvedValue({}) }); + await makeDriver(api).put('a/x.bin', Readable.from('hello'), { contentType: 'text/plain' }); + const input = (api.PutObject as ReturnType).mock.calls[0][0]; + expect(Buffer.isBuffer(input.Body) && input.Body.toString()).toBe('hello'); + expect(input).toMatchObject({ Key: 'a/x.bin', ContentType: 'text/plain' }); + expect(api.Upload).not.toHaveBeenCalled(); + }); + it('routes a zero-byte stream to PutObject (Upload breaks on empty streams upstream)', async () => { + const api = makeMockApi({ PutObject: vi.fn().mockResolvedValue({}) }); + await makeDriver(api).put('a/empty.bin', Readable.from([])); + const input = (api.PutObject as ReturnType).mock.calls[0][0]; + expect(Buffer.isBuffer(input.Body) && input.Body.length).toBe(0); + // aws4@1.13.2 signing workaround (see zeroByteChecksumWorkaround in + // s3.driver.ts): a zero-byte Body must set ApplyChecksum: true so + // @aws-lite/s3's PutObject never pre-sets a falsy-numeric Content-Length + // that aws4 then duplicates, breaking the signature against real S3. + expect(input.ApplyChecksum).toBe(true); + }); + it('does not apply the zero-byte checksum workaround to a 1-byte stream (boundary is exactly size 0)', async () => { + const api = makeMockApi({ PutObject: vi.fn().mockResolvedValue({}) }); + await makeDriver(api).put('a/one-byte.bin', Readable.from(Buffer.from([7]))); + const input = (api.PutObject as ReturnType).mock.calls[0][0]; + expect(input.Body.length).toBe(1); + expect(input.ApplyChecksum).toBeUndefined(); + }); + it('routes a stream exceeding the threshold to Upload with the peeked prefix re-attached', async () => { + let uploaded: Buffer | null = null; + const api = makeMockApi({ + Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => { + uploaded = await drain(Body); + }), + }); + // Three chunks so the source cannot end during the peek (the peek stops + // after chunk 2 crosses the threshold; chunk 3 is still pending) — the + // Upload path is deterministic, not timing-dependent. + const chunk = Buffer.alloc(6 * 1024 * 1024, 7); + const big = Buffer.concat([chunk, chunk, chunk]); + await makeDriver(api).put('a/big.bin', Readable.from([chunk, chunk, chunk])); + expect(api.PutObject).not.toHaveBeenCalled(); + expect(uploaded!.length).toBe(big.length); + expect(uploaded!.equals(big)).toBe(true); + expect((api.Upload as ReturnType).mock.calls[0][0]).toMatchObject({ + Key: 'a/big.bin', + ChunkSize: MULTIPART_THRESHOLD, + }); + }); + it('never passes ContentType to Upload (it corrupts CompleteMultipartUpload upstream — MalformedXML)', async () => { + const api = makeMockApi({ + Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => { + await drain(Body); + }), + }); + const chunk = Buffer.alloc(6 * 1024 * 1024, 7); + await makeDriver(api).put('a/big.zip', Readable.from([chunk, chunk, chunk]), { + contentType: 'application/zip', + }); + expect(api.Upload).toHaveBeenCalled(); + expect((api.Upload as ReturnType).mock.calls[0][0]).not.toHaveProperty('ContentType'); + }); + it('does not drop bytes buffered ahead of a deferred Upload consumer (Transform interposition, not body.on("data"))', async () => { + // Real aws-lite Upload awaits a full CreateMultipartUpload round trip + // before attaching its own consumer. A `body.on('data', ...)` listener + // added directly to the Body stream calls resume() immediately, so any + // bytes emitted during that window are delivered only to the listener + // and lost forever. Simulate the round trip with two deferred ticks + // before the mock ever reads Body. + let uploaded: Buffer | null = null; + const api = makeMockApi({ + Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => { + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + uploaded = await drain(Body); + }), + }); + const chunk = Buffer.alloc(6 * 1024 * 1024, 7); + const big = Buffer.concat([chunk, chunk, chunk]); + await makeDriver(api).put('a/deferred.bin', Readable.from([chunk, chunk, chunk])); + expect(uploaded!.length).toBe(big.length); + expect(uploaded!.equals(big)).toBe(true); + }); + it('routes a single-chunk oversized stream that ends at the peek boundary to PutObject (readableEnded guard)', async () => { + // One huge chunk: 'end' can fire between the peek resolving and pipe() + // attaching — the driver must detect readableEnded and PutObject the + // already-buffered payload rather than hang a never-ending PassThrough. + const api = makeMockApi({ + PutObject: vi.fn().mockResolvedValue({}), + Upload: vi.fn().mockResolvedValue({}), + }); + const big = Buffer.alloc(MULTIPART_THRESHOLD + 3, 7); + const oneChunk = new Readable({ read() {} }); + oneChunk.push(big); + oneChunk.push(null); + await makeDriver(api).put('a/one-chunk.bin', oneChunk); + // Either path must complete with the full payload; no hang, no data loss. + const putCalls = (api.PutObject as ReturnType).mock.calls; + const uploadCalls = (api.Upload as ReturnType).mock.calls; + expect(putCalls.length + uploadCalls.length).toBe(1); + if (putCalls.length === 1) { + expect((putCalls[0][0].Body as Buffer).length).toBe(big.length); + } + }); + it('surfaces a source-stream error before any client call', async () => { + const api = makeMockApi(); + const boom = new Readable({ + read() { + this.destroy(new Error('source exploded')); + }, + }); + await expect(makeDriver(api).put('a/broken.bin', boom)).rejects.toThrow('source exploded'); + expect(api.PutObject).not.toHaveBeenCalled(); + expect(api.Upload).not.toHaveBeenCalled(); + }); + it('expires a stalled Upload via the inactivity deadline (monitoring Transform destroyed)', async () => { + const api = makeMockApi({ + Upload: vi.fn().mockImplementation( + ({ Body }: { Body: Readable }) => + new Promise((_, reject) => { + Body.on('error', reject); // aws-lite Upload fails when its source errors + }), + ), + }); + const stalled = new Readable({ read() {} }); + stalled.push(Buffer.alloc(MULTIPART_THRESHOLD + 1)); // exceed the peek, then go silent + await expect(makeDriver(api, { timeoutMs: 60 }).put('a/stall.bin', stalled)).rejects.toBeInstanceOf( + StorageBackendError, + ); + }); + it('destroys the original source when Upload rejects for a non-timeout reason (no fd/socket leak)', async () => { + const api = makeMockApi({ + Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => { + await drain(Body); // full data transfer succeeds... + throw awsError(500); // ...but the finalize step (e.g. CompleteMultipartUpload) fails + }), + }); + const chunk = Buffer.alloc(6 * 1024 * 1024, 7); + const source = Readable.from([chunk, chunk, chunk]); + await expect(makeDriver(api).put('a/fail-after-drain.bin', source)).rejects.toBeInstanceOf(StorageBackendError); + expect(source.destroyed).toBe(true); + }); +}); + +describe('S3Driver put — LocalTempFile ownership', () => { + async function makeTmp(bytes: number): Promise { + const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'trek-s3-')); + const p = path.join(dir, 'src.bin'); + await fs.promises.writeFile(p, Buffer.alloc(bytes, 1)); + return p; + } + it('uploads a small temp file via PutObject File and consumes it', async () => { + const api = makeMockApi({ PutObject: vi.fn().mockResolvedValue({}) }); + const tmp = await makeTmp(10); + await makeDriver(api).put('a/t.bin', { tmpPath: tmp }, { contentType: 'image/png' }); + expect(api.PutObject).toHaveBeenCalledWith( + expect.objectContaining({ File: tmp, Key: 'a/t.bin', ContentType: 'image/png' }), + ); + expect(fs.existsSync(tmp)).toBe(false); + }); + it('routes a threshold-sized temp file to Upload', async () => { + const api = makeMockApi({ Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => drain(Body)) }); + const tmp = await makeTmp(MULTIPART_THRESHOLD); + await makeDriver(api).put('a/t.bin', { tmpPath: tmp }); + expect(api.Upload).toHaveBeenCalled(); + expect(fs.existsSync(tmp)).toBe(false); + }); + it('never passes ContentType to a temp-file Upload either (same upstream MalformedXML defect)', async () => { + const api = makeMockApi({ Upload: vi.fn().mockImplementation(async ({ Body }: { Body: Readable }) => drain(Body)) }); + const tmp = await makeTmp(MULTIPART_THRESHOLD); + await makeDriver(api).put('a/t.zip', { tmpPath: tmp }, { contentType: 'application/zip' }); + expect(api.Upload).toHaveBeenCalled(); + expect((api.Upload as ReturnType).mock.calls[0][0]).not.toHaveProperty('ContentType'); + }); + it('rejects a stalled threshold-sized temp-file Upload with StorageBackendError rather than crashing', async () => { + // The Upload mock never reads/drains Body — only listens for its error, + // matching aws-lite's real Upload contract — so the fs.createReadStream + // source AND the interposed Transform must each have an 'error' handler + // attached before expire() destroys them; otherwise an unlistened + // 'error' anywhere in that chain crashes the process instead of + // rejecting this promise. + const api = makeMockApi({ + Upload: vi.fn().mockImplementation( + ({ Body }: { Body: Readable }) => + new Promise((_, reject) => { + Body.on('error', reject); + }), + ), + }); + const tmp = await makeTmp(MULTIPART_THRESHOLD); + await expect( + makeDriver(api, { timeoutMs: 60 }).put('a/stall-tmp.bin', { tmpPath: tmp }), + ).rejects.toBeInstanceOf(StorageBackendError); + }); + it('consumes the temp file on the failure path too (MirrorDriver precedent)', async () => { + const api = makeMockApi({ PutObject: vi.fn().mockRejectedValue(awsError(500)) }); + const tmp = await makeTmp(10); + await expect(makeDriver(api).put('a/t.bin', { tmpPath: tmp })).rejects.toBeInstanceOf(StorageBackendError); + expect(fs.existsSync(tmp)).toBe(false); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-admin.controller.test.ts b/server/tests/unit/nest/storage/storage-admin.controller.test.ts new file mode 100644 index 000000000..cfcea78f2 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-admin.controller.test.ts @@ -0,0 +1,351 @@ +import { describe, it, expect, vi } from 'vitest'; +import { HttpException } from '@nestjs/common'; +import type { Request } from 'express'; +import type { StorageUsage } from '@trek/shared'; +import type { User } from '../../../../src/types'; +import { StorageAdminController } from '../../../../src/nest/storage/storage-admin.controller'; +import type { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; +import type { AuditService } from '../../../../src/nest/audit/audit.service'; +import type { StorageConfigDto, StorageTestRequestDto } from '../../../../src/nest/storage/storage-admin.dto'; +import { StorageModule } from '../../../../src/nest/storage/storage.module'; +import { StorageBackendError, StorageConflictError } from '../../../../src/nest/storage/storage.types'; +import { + BackfillBusyError, + BackfillTargetError, + MigrationRequestError, + MigrationTargetError, +} from '../../../../src/nest/storage/storage-jobs.service'; +import { StatsBusyError } from '../../../../src/nest/storage/storage-stats.service'; +import { expectRegisteredController } from '../../../helpers/module-providers'; + +const user = { id: 1 } as User; +const req = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } as unknown as Request; +const FRESH_STATE = { + backends: [], + categories: {}, + health: { replicaFailures: [] }, + seedFilePresent: false, + migrations: [], +}; + +function makeController(over: Partial> = {}) { + const writeAudit = vi.fn(); + const service = { + state: vi.fn(() => FRESH_STATE), + applyConfig: vi.fn(), + ...over, + } as unknown as StorageAdminService; + const controller = new StorageAdminController(service, { writeAudit } as unknown as AuditService); + return { controller, service, writeAudit }; +} + +const CONFIG = { + backends: [ + { + name: 'off-box', + type: 's3' as const, + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-plain', + region: 'us-east-1', + keyPrefix: '', + retries: 1, + timeoutMs: 30000, + }, + }, + ], + categories: { backups: 'off-box' }, + version: 0, +} as StorageConfigDto; + +describe('StorageAdminController', () => { + it('STORCTL-001 GET returns the service state untouched', () => { + const { controller, service } = makeController(); + expect(controller.get()).toBe(FRESH_STATE); + expect(service.state).toHaveBeenCalledTimes(1); + }); + + it('STORCTL-002 PUT applies, audits with secrets redacted, and answers the fresh state', () => { + const { controller, service, writeAudit } = makeController(); + const result = controller.update(user, CONFIG, req); + expect(service.applyConfig).toHaveBeenCalledWith(CONFIG); + expect(result).toBe(FRESH_STATE); // never echoes the request + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 1, action: 'admin.storage_update' }), + ); + const details = (writeAudit.mock.calls[0]![0] as { details: { backends: Array<{ options: Record }> } }).details; + expect(details.backends[0]!.options.secretAccessKey).toBe('***'); + expect(details.backends[0]!.options.accessKeyId).toBe('ak'); // names/shape survive redaction + }); + + it('STORCTL-003 PUT maps pipeline refusals to a 400 with the message verbatim, no audit', () => { + const { controller, writeAudit } = makeController({ + applyConfig: vi.fn(() => { + throw new StorageBackendError("category 'backups' maps to unknown backend 'nope'"); + }), + }); + try { + controller.update(user, CONFIG, req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(400); + expect((err as HttpException).getResponse()).toEqual({ + error: "category 'backups' maps to unknown backend 'nope'", + }); + } + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-023 PUT maps StorageConflictError to a 409, ahead of the blanket 400, no audit', () => { + const { controller, writeAudit } = makeController({ + applyConfig: vi.fn(() => { + // StorageConflictError IS an Error — this pins that the instanceof + // branch is checked BEFORE the generic catch-all 400 (audit #7). + throw new StorageConflictError(3, 1); + }), + }); + try { + controller.update(user, CONFIG, req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(409); + expect((err as HttpException).getResponse()).toEqual({ + error: "storage settings changed since this form was loaded (current version 3, submitted 1) — reload and reapply your edits", + }); + } + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-004 the controller is registered in StorageModule', () => { + expectRegisteredController(StorageModule, StorageAdminController); + }); + + it('STORCTL-005 POST /test probes, audits names only, answers 200-shaped result', async () => { + const testResult = { ok: true, targets: [{ name: 'cand', ok: true }] }; + const { controller, writeAudit } = makeController({ testBackend: vi.fn(async () => testResult) }); + const result = await controller.test(user, { backend: CONFIG.backends[0]! } as StorageTestRequestDto, req); + expect(result).toBe(testResult); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin.storage_test', + details: { backend: 'off-box', type: 's3', ok: true, targets: [{ name: 'cand', ok: true }] }, + }), + ); + }); + + it('STORCTL-006 POST /test maps thrown refusals to 400 verbatim', async () => { + const { controller } = makeController({ + testBackend: vi.fn(async () => { + throw new StorageBackendError("mirror 'm' references unknown backend 'nope'"); + }), + }); + await expect( + controller.test(user, { backend: CONFIG.backends[0]! } as StorageTestRequestDto, req), + ).rejects.toMatchObject({ status: 400 }); + }); + + it('STORCTL-007 POST backfill starts the sync, audits the backend name, and answers { started: true }', () => { + const { controller, service, writeAudit } = makeController({ startBackfill: vi.fn() }); + const result = controller.backfillStart(user, 'm', req); + expect(service.startBackfill).toHaveBeenCalledWith('m'); + expect(result).toEqual({ started: true }); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 1, action: 'admin.storage_backfill', details: { backend: 'm' } }), + ); + }); + + it('STORCTL-008 POST backfill maps BackfillTargetError to 404, no audit', () => { + const { controller, writeAudit } = makeController({ + startBackfill: vi.fn(() => { + throw new BackfillTargetError("'ghost' is not a mirror routed by any category"); + }), + }); + try { + controller.backfillStart(user, 'ghost', req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(404); + expect((err as HttpException).getResponse()).toEqual({ error: "'ghost' is not a mirror routed by any category" }); + } + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-009 POST backfill maps BackfillBusyError to 409, no audit', () => { + const { controller, writeAudit } = makeController({ + startBackfill: vi.fn(() => { + throw new BackfillBusyError('a sync is already running — one backfill at a time'); + }), + }); + try { + controller.backfillStart(user, 'm', req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(409); + } + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-010 POST backfill rethrows an unrecognized error untouched', () => { + const boom = new Error('unexpected registry failure'); + const { controller, writeAudit } = makeController({ + startBackfill: vi.fn(() => { + throw boom; + }), + }); + expect(() => controller.backfillStart(user, 'm', req)).toThrow(boom); + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-011 DELETE backfill cancels, audits, and answers { cancelled: true } when a sync is active', () => { + const { controller, service, writeAudit } = makeController({ cancelBackfill: vi.fn(() => true) }); + const result = controller.backfillCancel(user, 'm', req); + expect(service.cancelBackfill).toHaveBeenCalledWith('m'); + expect(result).toEqual({ cancelled: true }); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 1, action: 'admin.storage_backfill_cancel', details: { backend: 'm' } }), + ); + }); + + it('STORCTL-012 DELETE backfill answers 404 with no audit when there is no active sync', () => { + const { controller, writeAudit } = makeController({ cancelBackfill: vi.fn(() => false) }); + try { + controller.backfillCancel(user, 'ghost', req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(404); + expect((err as HttpException).getResponse()).toEqual({ error: "no active sync for 'ghost'" }); + } + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-013 POST stats/refresh runs the scan, audits (no details payload), and answers the usage', async () => { + const usage = { computedAt: 1, categories: {}, legacyPhotos: { objects: 0, bytes: 0 } } as unknown as StorageUsage; + const { controller, service, writeAudit } = makeController({ refreshStats: vi.fn(async () => usage) }); + const result = await controller.statsRefresh(user, req); + expect(service.refreshStats).toHaveBeenCalledTimes(1); + expect(result).toBe(usage); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 1, action: 'admin.storage_stats_refresh' }), + ); + }); + + it('STORCTL-014 POST stats/refresh maps StatsBusyError to 409, no audit', async () => { + const { controller, writeAudit } = makeController({ + refreshStats: vi.fn(async () => { + throw new StatsBusyError('a usage scan is already running'); + }), + }); + await expect(controller.statsRefresh(user, req)).rejects.toMatchObject({ status: 409 }); + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-015 POST stats/refresh rethrows an unrecognized error untouched', async () => { + const boom = new Error('disk full'); + const { controller, writeAudit } = makeController({ + refreshStats: vi.fn(async () => { + throw boom; + }), + }); + await expect(controller.statsRefresh(user, req)).rejects.toBe(boom); + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-020 POST migrations starts and audits', () => { + const { controller, service, writeAudit } = makeController({ startMigration: vi.fn() }); + const result = controller.migrationStart(user, { category: 'files', to: 'dest' }, req); + expect(service.startMigration).toHaveBeenCalledWith('files', 'dest'); + expect(result).toEqual({ started: true }); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ + userId: 1, + action: 'admin.storage_migration', + details: { category: 'files', to: 'dest' }, + }), + ); + }); + + it('STORCTL-021 POST maps MigrationRequestError→400, MigrationTargetError→404, BackfillBusyError→409, others rethrow', () => { + const { controller: reqController, writeAudit: reqAudit } = makeController({ + startMigration: vi.fn(() => { + throw new MigrationRequestError("'files' is already on 'dest'"); + }), + }); + try { + reqController.migrationStart(user, { category: 'files', to: 'dest' }, req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(400); + expect((err as HttpException).getResponse()).toEqual({ error: "'files' is already on 'dest'" }); + } + expect(reqAudit).not.toHaveBeenCalled(); + + const { controller: targetController, writeAudit: targetAudit } = makeController({ + startMigration: vi.fn(() => { + throw new MigrationTargetError("no backend named 'ghost'"); + }), + }); + try { + targetController.migrationStart(user, { category: 'files', to: 'ghost' }, req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(404); + expect((err as HttpException).getResponse()).toEqual({ error: "no backend named 'ghost'" }); + } + expect(targetAudit).not.toHaveBeenCalled(); + + const { controller: busyController, writeAudit: busyAudit } = makeController({ + startMigration: vi.fn(() => { + throw new BackfillBusyError('a sync is already running — one storage job at a time'); + }), + }); + try { + busyController.migrationStart(user, { category: 'files', to: 'dest' }, req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(409); + } + expect(busyAudit).not.toHaveBeenCalled(); + + const boom = new Error('unexpected registry failure'); + const { controller: boomController, writeAudit: boomAudit } = makeController({ + startMigration: vi.fn(() => { + throw boom; + }), + }); + expect(() => boomController.migrationStart(user, { category: 'files', to: 'dest' }, req)).toThrow(boom); + expect(boomAudit).not.toHaveBeenCalled(); + }); + + it('STORCTL-022 DELETE migrations/:category cancels + audits; 404 when none running', () => { + const { controller, service, writeAudit } = makeController({ cancelMigration: vi.fn(() => true) }); + const result = controller.migrationCancel(user, 'files', req); + expect(service.cancelMigration).toHaveBeenCalledWith('files'); + expect(result).toEqual({ cancelled: true }); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 1, action: 'admin.storage_migration_cancel', details: { category: 'files' } }), + ); + + const { controller: noneController, writeAudit: noneAudit } = makeController({ + cancelMigration: vi.fn(() => false), + }); + try { + noneController.migrationCancel(user, 'ghost', req); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(HttpException); + expect((err as HttpException).getStatus()).toBe(404); + expect((err as HttpException).getResponse()).toEqual({ error: "no running migration for 'ghost'" }); + } + expect(noneAudit).not.toHaveBeenCalled(); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-admin.service.test.ts b/server/tests/unit/nest/storage/storage-admin.service.test.ts new file mode 100644 index 000000000..d4c7f3fe8 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-admin.service.test.ts @@ -0,0 +1,470 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest'; + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + return { testDb: db, dbMock: { db, closeDb: () => {}, reinitialize: () => {} } }; +}); +vi.mock('../../../../src/db/database', () => dbMock); +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-admin-test-key' })); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Logger } from '@nestjs/common'; +import { MASKED_SETTING_VALUE, type StorageConfig } from '@trek/shared'; +import { createTables } from '../../../../src/db/schema'; +import { runMigrations } from '../../../../src/db/migrations'; +import { DatabaseService } from '../../../../src/nest/database/database.service'; +import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto'; +import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; +import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; +import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; +import { StorageStatsService } from '../../../../src/nest/storage/storage-stats.service'; +import { StorageRegistryService, BACKENDS_KEY, CATEGORIES_KEY } from '../../../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import { StorageConflictError } from '../../../../src/nest/storage/storage.types'; + +const db = new DatabaseService(testDb); + +beforeAll(() => { + createTables(testDb); + runMigrations(testDb); +}); + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-admin-')); + tmpDirs.push(dir); + return dir; +} +function setSetting(key: string, value: string): void { + testDb.prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)').run(key, value); +} +function readRow(key: string): string | undefined { + const row = testDb.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as { value: string } | undefined; + return row?.value; +} + +const S3_OPTIONS = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-plain', + region: 'us-east-1', + keyPrefix: '', + retries: 1, + timeoutMs: 30000, +}; + +/** Real registry + real service over the in-memory DB. */ +function makeService(opts: { uploadsRoot?: string } = {}) { + const uploadsRoot = opts.uploadsRoot ?? makeTmpDir(); + setSetting(BACKENDS_KEY, JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }])); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + const stats = new StorageStatsService(storage, db); + const service = new StorageAdminService(db, registry, storage, jobs, stats); + return { service, registry, uploadsRoot, stats, jobs }; +} + +/** The settings-owned document the service persists (uploads override + extras). */ +function configWith(uploadsRoot: string, extra: Partial = {}): StorageConfig { + return { + backends: [ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + ...(extra.backends ?? []), + ], + categories: extra.categories ?? {}, + }; +} + +/** + * applyConfig now requires the current version (optimistic concurrency, + * audit #7) — most tests just want "the current one", read fresh at call + * time so a second call in the same test picks up the bump from the first. + */ +function put(service: StorageAdminService, config: StorageConfig): void { + service.applyConfig({ ...config, version: service.state().version }); +} + +beforeEach(() => { + testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run(); +}); +afterEach(() => { + vi.restoreAllMocks(); + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +describe('StorageAdminService.state', () => { + it('STORADM-001 renders the effective world: sources, categories-per-backend, flags', () => { + const { service, uploadsRoot } = makeService(); + const state = service.state(); + const uploads = state.backends.find((b) => b.name === 'uploads-local')!; + expect(uploads).toMatchObject({ type: 'local', source: 'settings', options: { root: uploadsRoot } }); + expect(uploads.categories).toContain('files'); + expect(uploads.categories).not.toContain('backups'); + expect(state.backends.find((b) => b.name === 'backups-local')).toMatchObject({ source: 'built-in' }); + expect(state.categories.backups).toEqual({ backend: 'backups-local', source: 'default' }); + expect(state.seedFilePresent).toBe(false); + expect(state.health).toEqual({ replicaFailures: [] }); + }); + + it('STORADM-002 masks exactly the secret fields (accessKeyId stays visible)', () => { + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }], + categories: { backups: 'off-box' }, + })); + const offBox = service.state().backends.find((b) => b.name === 'off-box')!; + expect(offBox.options.secretAccessKey).toBe(MASKED_SETTING_VALUE); + expect(offBox.options.accessKeyId).toBe('ak'); + }); + + it('STORADM-003 surfaces replica failures through StorageService.health()', () => { + const { service, registry } = makeService(); + registry.recordReplicaFailure({ backend: 'nas', key: 'backup-1.zip', op: 'put', error: 'disk full', at: 123 }); + expect(service.state().health.replicaFailures).toEqual([ + { backend: 'nas', key: 'backup-1.zip', op: 'put', error: 'disk full', at: 123 }, + ]); + }); + + it('STORADM-027 configError is null on a clean load, and mirrors registry.lastLoadError() after a bad reload', () => { + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { service, registry } = makeService(); + expect(service.state().configError).toBeNull(); + + setSetting(CATEGORIES_KEY, 'garbage {'); + registry.reload(); + expect(service.state().configError).toBe(registry.lastLoadError()); + expect(service.state().configError).not.toBeNull(); + }); + + it('STORADM-026 state embeds usage (null until computed) and live backfill statuses', async () => { + const { service, stats } = makeService(); + expect(service.state().usage).toBeNull(); + expect(service.state().backfills).toEqual([]); + await stats.scan(); + expect(service.state().usage).not.toBeNull(); + expect(service.state().usage!.computedAt).toBeGreaterThan(0); + }); +}); + +describe('StorageAdminService.applyConfig', () => { + it('STORADM-010 happy path: persists both rows, reloads, new config is live', () => { + const { service, registry, uploadsRoot } = makeService(); + const nasRoot = makeTmpDir(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }], + categories: { backups: 'nas-backups' }, + })); + expect(registry.resolve('backups').backendName).toBe('nas-backups'); // reload() ran + expect(JSON.parse(readRow(CATEGORIES_KEY)!)).toEqual({ backups: 'nas-backups' }); + }); + + it('STORADM-011 encrypts plaintext secrets at rest', () => { + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }], + categories: { backups: 'off-box' }, + })); + const stored = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record }>; + const secret = String(stored.find((b) => b.name === 'off-box')!.options.secretAccessKey); + expect(secret.startsWith('enc:v1:')).toBe(true); + }); + + it('STORADM-012 mask echo preserves the stored ciphertext byte-for-byte', () => { + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }], + categories: { backups: 'off-box' }, + })); + const before = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record }>; + const cipherBefore = before.find((b) => b.name === 'off-box')!.options.secretAccessKey; + + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: MASKED_SETTING_VALUE } }], + categories: { backups: 'off-box' }, + })); + const after = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record }>; + expect(after.find((b) => b.name === 'off-box')!.options.secretAccessKey).toBe(cipherBefore); + }); + + it('STORADM-013 a mask on a renamed/new backend throws the re-enter error, persists nothing', () => { + const { service, uploadsRoot } = makeService(); + const before = readRow(BACKENDS_KEY); + expect(() => + put(service, configWith(uploadsRoot, { + backends: [{ name: 'brand-new', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: MASKED_SETTING_VALUE } }], + categories: {}, + })), + ).toThrow("re-enter the secret 'secretAccessKey' for 'brand-new'"); + expect(readRow(BACKENDS_KEY)).toBe(before); + }); + + it('STORADM-014 a plaintext secret saves without an explicit ENCRYPTION_KEY and is still encrypted at rest', () => { + // No key-presence gate: the implicit key covers encryption when ENCRYPTION_KEY is unset. + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }], + categories: {}, + })); + const stored = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record }>; + const offBox = stored.find((b) => b.name === 'off-box')!; + expect(offBox.options.secretAccessKey.startsWith('enc:v1:')).toBe(true); + }); + + it('STORADM-015 an encrypted (mask-echoed or enc:v1:) secret resaves fine', () => { + // Resaving stored ciphertext must not lock admins out. + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: encrypt_api_key('sk') } }], + categories: { backups: 'off-box' }, + })); + expect(readRow(BACKENDS_KEY)).toBeDefined(); + }); + + it('STORADM-016 preview() refusals surface verbatim and persist nothing', () => { + const { service, uploadsRoot, registry } = makeService(); + const before = readRow(CATEGORIES_KEY); + expect(() => + put(service, configWith(uploadsRoot, { categories: { backups: 'nope' } })), + ).toThrow("category 'backups' maps to unknown backend 'nope'"); + expect(readRow(CATEGORIES_KEY)).toBe(before); + expect(registry.resolve('backups').backendName).toBe('backups-local'); // live state untouched + }); + + it('STORADM-018 a mask sentinel in a NON-secret field throws and persists nothing', () => { + const { service, uploadsRoot } = makeService(); + const before = readRow(BACKENDS_KEY); + expect(() => + put(service, configWith(uploadsRoot, { + backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, accessKeyId: MASKED_SETTING_VALUE } }], + categories: {}, + })), + ).toThrow("backend 'off-box' field 'accessKeyId' is the mask sentinel — a mask can never become a stored value"); + expect(readRow(BACKENDS_KEY)).toBe(before); + }); + + it('STORADM-017 persists both rows in ONE transaction and reloads once', () => { + const { service, registry, uploadsRoot } = makeService(); + const txSpy = vi.spyOn(db, 'transaction'); + const reloadSpy = vi.spyOn(registry, 'reload'); + put(service, configWith(uploadsRoot)); + expect(txSpy).toHaveBeenCalledTimes(1); + expect(reloadSpy).toHaveBeenCalledTimes(1); + }); + + it('STORADM-019 applyConfig invokes the dissolved-job cancellation after reload', () => { + const { service, registry, uploadsRoot, jobs } = makeService(); + const reloadSpy = vi.spyOn(registry, 'reload'); + const cancelSpy = vi.spyOn(jobs, 'cancelJobsForMissingBackends'); + put(service, configWith(uploadsRoot)); + expect(cancelSpy).toHaveBeenCalledTimes(1); + expect(reloadSpy).toHaveBeenCalledTimes(1); + const reloadOrder = reloadSpy.mock.invocationCallOrder[0]!; + const cancelOrder = cancelSpy.mock.invocationCallOrder[0]!; + expect(cancelOrder).toBeGreaterThan(reloadOrder); + }); + + it('STORADM-030 a successful save bumps the version counter by exactly one', () => { + const { service, uploadsRoot } = makeService(); + expect(service.state().version).toBe(0); + put(service, configWith(uploadsRoot)); + expect(service.state().version).toBe(1); + put(service, configWith(uploadsRoot)); + expect(service.state().version).toBe(2); + }); + + it('STORADM-031 a stale submitted version throws StorageConflictError and persists nothing (→ 409, audit #7)', () => { + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot)); // version is now 1 + const beforeBackends = readRow(BACKENDS_KEY); + const beforeCategories = readRow(CATEGORIES_KEY); + expect(() => + service.applyConfig({ ...configWith(uploadsRoot), version: 0 }), // stale — current is 1 + ).toThrow(StorageConflictError); + try { + service.applyConfig({ ...configWith(uploadsRoot), version: 0 }); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(StorageConflictError); + expect((err as StorageConflictError).currentVersion).toBe(1); + expect((err as StorageConflictError).submittedVersion).toBe(0); + } + expect(readRow(BACKENDS_KEY)).toBe(beforeBackends); + expect(readRow(CATEGORIES_KEY)).toBe(beforeCategories); + expect(service.state().version).toBe(1); // unchanged — the conflicting save never wrote + }); + + it('STORADM-032 the version check runs before preview/unmask — a stale submit never reaches those refusals', () => { + // If the version check ran AFTER preview, this would throw the registry's + // "unknown backend" StorageBackendError instead of StorageConflictError. + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot)); // version is now 1 + expect(() => + service.applyConfig({ ...configWith(uploadsRoot, { categories: { backups: 'nope' } }), version: 0 }), + ).toThrow(StorageConflictError); + }); + + it('STORADM-033 regression (audit #7): a migration flip bumps the version; a stale admin PUT built before the flip 409s and the flip survives untouched', () => { + const { service, registry, uploadsRoot } = makeService(); + const destRoot = makeTmpDir(); + // The admin loads the form: draft carries version 0, categories.files → uploads-local. + put(service, configWith(uploadsRoot, { + backends: [{ name: 'dest', type: 'local', options: { root: destRoot } }], + categories: {}, + })); // version is now 1; 'files' still defaults to uploads-local + const staleDraft = configWith(uploadsRoot, { + backends: [{ name: 'dest', type: 'local', options: { root: destRoot } }], + categories: {}, // the admin's draft never touched 'files' + }); + const staleVersion = service.state().version; // 1 — what the admin's form is holding + + // Meanwhile, a category migration flips 'files' to 'dest' — the exact + // write path assignCategory uses, bumping the shared version counter. + registry.assignCategory('files', 'dest'); + expect(registry.snapshot().categories.files.backend).toBe('dest'); + expect(service.state().version).toBe(2); // the flip bumped it past the admin's stale draft + + // The admin's stale PUT (still holding version 1, and no opinion on + // 'files') must 409, NOT silently reassign 'files' back to its old default. + expect(() => service.applyConfig({ ...staleDraft, version: staleVersion })).toThrow(StorageConflictError); + + // The flip survives untouched. + expect(registry.snapshot().categories.files.backend).toBe('dest'); + expect(service.state().version).toBe(2); + }); +}); + +describe('StorageAdminService.testBackend', () => { + it('STORADM-020 probes a healthy local candidate: ok with one green target', async () => { + const { service } = makeService(); + const result = await service.testBackend({ name: 'cand', type: 'local', options: { root: makeTmpDir() } }); + expect(result).toEqual({ ok: true, targets: [{ name: 'cand', ok: true }] }); + }); + + it('STORADM-021 an unreachable s3 candidate fails fast with a per-target error (no registry impact)', async () => { + const { service, registry } = makeService(); + const result = await service.testBackend({ + name: 'cand', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:1', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk', + region: 'us-east-1', + keyPrefix: '', + retries: 0, + timeoutMs: 200, + }, + }); + expect(result.ok).toBe(false); + expect(result.targets[0]!.ok).toBe(false); + expect(result.targets[0]!.error).toBeTruthy(); + expect(registry.resolve('backups').backendName).toBe('backups-local'); // state untouched + }); + + it('STORADM-022 a mirror is probed per target, replica failures reported individually', async () => { + const goodRoot = makeTmpDir(); + // Not yet a file: registry.build() eagerly driver.init()s EVERY local + // backend (used or not) during applyConfig's preview, so a root that's + // already a file at that point would make applyConfig itself throw + // (unrelated to testBackend). Register it as a fresh, empty dir, then + // corrupt it afterward — testBackend's ephemeralDriverFor re-runs + // init() fresh at probe time and hits the same EEXIST there instead. + const badRoot = path.join(makeTmpDir(), 'a-file'); + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [ + { name: 'good-local', type: 'local', options: { root: goodRoot } }, + { name: 'bad-local', type: 'local', options: { root: badRoot } }, + ], + categories: {}, + })); + fs.rmSync(badRoot, { recursive: true, force: true }); + fs.writeFileSync(badRoot, 'not a dir'); + const result = await service.testBackend({ + name: 'cand-mirror', + type: 'mirror', + options: { primary: 'good-local', replicas: ['bad-local'] }, + }); + expect(result.ok).toBe(false); + expect(result.targets).toEqual([ + { name: 'good-local', ok: true }, + expect.objectContaining({ name: 'bad-local', ok: false }), + ]); + }); + + it('STORADM-023 a mirror referencing an unknown or mirror-typed backend throws (→ 400)', async () => { + const { service } = makeService(); + await expect( + service.testBackend({ name: 'm', type: 'mirror', options: { primary: 'nope', replicas: [] } }), + ).rejects.toThrow("mirror 'm' references unknown backend 'nope'"); + }); + + it('STORADM-024 unmasks a stored backend by name before probing (mask echo works on /test)', async () => { + const { service, uploadsRoot } = makeService(); + const root = makeTmpDir(); + put(service, configWith(uploadsRoot, { + backends: [{ name: 'nas', type: 'local', options: { root } }], + categories: {}, + })); + // local has no secrets — the unmask contract is pinned via an s3 mask with no counterpart: + await expect( + service.testBackend({ + name: 'ghost', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:1', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: MASKED_SETTING_VALUE, + region: 'us-east-1', + keyPrefix: '', + retries: 0, + timeoutMs: 200, + }, + }), + ).rejects.toThrow("re-enter the secret 'secretAccessKey' for 'ghost'"); + }); + + it('STORADM-025 a mirror with a stored s3 replica decrypts the enc:v1: secret for the probe', async () => { + const goodRoot = makeTmpDir(); + const { service, uploadsRoot } = makeService(); + put(service, configWith(uploadsRoot, { + backends: [ + { name: 'good-local', type: 'local', options: { root: goodRoot } }, + { + name: 'off-box', + type: 's3', + options: { ...S3_OPTIONS, endpoint: 'http://127.0.0.1:1', retries: 0, timeoutMs: 200 }, + }, + ], + categories: {}, + })); + // applyConfig encrypted the secret at rest — the snapshot the mirror + // expansion reads holds ciphertext, so this probe exercises the full + // snapshot → decryptBackendSecrets → ephemeral S3Driver chain. + expect(readRow(BACKENDS_KEY)).toContain('enc:v1:'); + const result = await service.testBackend({ + name: 'cand-mirror', + type: 'mirror', + options: { primary: 'good-local', replicas: ['off-box'] }, + }); + expect(result.targets).toEqual([ + { name: 'good-local', ok: true }, + expect.objectContaining({ name: 'off-box', ok: false }), + ]); + // Decrypt succeeded: the replica failed on the network. A decrypt failure + // would reject the whole call (decryptBackendSecrets throws), not mark a target. + expect(result.targets[1]!.error).not.toContain('could not decrypt'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-driver.contract.ts b/server/tests/unit/nest/storage/storage-driver.contract.ts new file mode 100644 index 000000000..472cd9123 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-driver.contract.ts @@ -0,0 +1,167 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { Readable } from 'node:stream'; +import fs from 'node:fs'; +import path from 'node:path'; +import { + StorageInvalidKeyError, + StorageNotFoundError, + type ObjectStat, + type StorageDriver, +} from '../../../../src/nest/storage/storage.types'; + +/** + * The parameterized driver contract suite — the groundwork's real product + * (spec, Testing). Every driver must pass it: LocalDriver and MirrorDriver in + * v1; a future S3 driver imports this same suite (against MinIO/localstack) + * and must pass before touching TREK code. + * + * Not a `.test.ts` on purpose: vitest only collects files matching the + * `*.test.ts` include, so this suite runs exclusively through a driver's own + * spec file. + */ + +export interface DriverHarness { + driver: StorageDriver; + /** Create a temp file whose ownership put() may take (LocalTempFile source). */ + makeTempFile(contents: string | Buffer): Promise; + cleanup(): Promise; +} + +const INVALID_KEYS = ['../x', '/abs', 'a\\b', 'a//b', '.tmp/x', 'a/.h', 'a\u0000b']; + +export function describeStorageDriver(name: string, makeHarness: () => Promise): void { + describe(`storage driver contract: ${name}`, () => { + let h: DriverHarness; + + beforeEach(async () => { + h = await makeHarness(); + }); + afterEach(async () => { + await h.cleanup(); + }); + + async function readAll(key: string, range?: { start: number; end?: number }): Promise { + const { stream } = await h.driver.getStream(key, range); + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk as Buffer)); + return Buffer.concat(chunks); + } + + async function listAll(prefix: string): Promise { + const out: ObjectStat[] = []; + for await (const stat of h.driver.list(prefix)) out.push(stat); + return out; + } + + it('roundtrips a stream source and stats its size', async () => { + await h.driver.put('a/stream.bin', Readable.from(Buffer.from('hello storage'))); + expect((await readAll('a/stream.bin')).toString()).toBe('hello storage'); + const stat = await h.driver.stat('a/stream.bin'); + expect(stat).toMatchObject({ key: 'a/stream.bin', size: 13 }); + expect(stat!.mtimeMs).toBeGreaterThan(0); + }); + + it('roundtrips a LocalTempFile source and consumes the temp file', async () => { + const tmp = await h.makeTempFile('from tmp file'); + await h.driver.put('a/tmpfile.bin', { tmpPath: tmp }); + expect((await readAll('a/tmpfile.bin')).toString()).toBe('from tmp file'); + expect(fs.existsSync(tmp)).toBe(false); // ownership transferred + }); + + it('overwrites an existing key', async () => { + await h.driver.put('a/over.bin', Readable.from('first')); + await h.driver.put('a/over.bin', Readable.from('second!')); + expect((await readAll('a/over.bin')).toString()).toBe('second!'); + expect((await h.driver.stat('a/over.bin'))!.size).toBe(7); + }); + + it('serves closed and open-ended byte ranges', async () => { + await h.driver.put('a/range.bin', Readable.from('0123456789')); + expect((await readAll('a/range.bin', { start: 2, end: 5 })).toString()).toBe('2345'); + expect((await readAll('a/range.bin', { start: 7 })).toString()).toBe('789'); + }); + + it('returns the full-object stat on a ranged read', async () => { + // Pins LocalDriver parity for remote drivers: size is the TOTAL object + // size (S3: from the 206 Content-Range total), never the range length. + await h.driver.put('a/rstat.bin', Readable.from('0123456789')); + const { stream, stat } = await h.driver.getStream('a/rstat.bin', { start: 2, end: 5 }); + for await (const _ of stream) void _; // drain so the harness can clean up + expect(stat).toMatchObject({ key: 'a/rstat.bin', size: 10 }); + }); + + it('roundtrips a zero-byte stream source', async () => { + await h.driver.put('a/empty.bin', Readable.from([])); + expect((await readAll('a/empty.bin')).length).toBe(0); + expect((await h.driver.stat('a/empty.bin'))!.size).toBe(0); + await h.driver.delete('a/empty.bin'); + expect(await h.driver.stat('a/empty.bin')).toBeNull(); + }); + + it('roundtrips a zero-byte LocalTempFile source and consumes the temp file', async () => { + const tmp = await h.makeTempFile(''); + await h.driver.put('a/empty-tmpfile.bin', { tmpPath: tmp }); + expect((await readAll('a/empty-tmpfile.bin')).length).toBe(0); + expect((await h.driver.stat('a/empty-tmpfile.bin'))!.size).toBe(0); + expect(fs.existsSync(tmp)).toBe(false); // ownership transferred + await h.driver.delete('a/empty-tmpfile.bin'); + expect(await h.driver.stat('a/empty-tmpfile.bin')).toBeNull(); + }); + + it('rejects getStream on a missing key with StorageNotFoundError', async () => { + await expect(h.driver.getStream('a/nope.bin')).rejects.toBeInstanceOf(StorageNotFoundError); + }); + + it('stats a missing key as null', async () => { + expect(await h.driver.stat('a/nope.bin')).toBeNull(); + }); + + it('deletes idempotently', async () => { + await h.driver.put('a/gone.bin', Readable.from('x')); + await h.driver.delete('a/gone.bin'); + expect(await h.driver.stat('a/gone.bin')).toBeNull(); + await expect(h.driver.delete('a/gone.bin')).resolves.toBeUndefined(); + }); + + it('lists by prefix with full keys and sizes', async () => { + await h.driver.put('a/one.bin', Readable.from('11')); + await h.driver.put('a/sub/two.bin', Readable.from('222')); + await h.driver.put('b/three.bin', Readable.from('3')); + + const underA = await listAll('a/'); + expect(underA.map((s) => s.key).sort()).toEqual(['a/one.bin', 'a/sub/two.bin']); + expect(underA.find((s) => s.key === 'a/sub/two.bin')!.size).toBe(3); + + const everything = await listAll(''); + expect(everything.map((s) => s.key).sort()).toEqual(['a/one.bin', 'a/sub/two.bin', 'b/three.bin']); + }); + + it.each(INVALID_KEYS)('rejects invalid key %j on every method', async (key) => { + await expect(h.driver.put(key, Readable.from('x'))).rejects.toBeInstanceOf(StorageInvalidKeyError); + await expect(h.driver.getStream(key)).rejects.toBeInstanceOf(StorageInvalidKeyError); + await expect(h.driver.stat(key)).rejects.toBeInstanceOf(StorageInvalidKeyError); + await expect(h.driver.delete(key)).rejects.toBeInstanceOf(StorageInvalidKeyError); + }); + + it('leaves no object and no visible partial when the source stream errors', async () => { + const boom = new Readable({ + read() { + this.push('partial bytes'); + this.destroy(new Error('source exploded')); + }, + }); + await expect(h.driver.put('a/broken.bin', boom)).rejects.toThrow(); + expect(await h.driver.stat('a/broken.bin')).toBeNull(); + expect((await listAll('')).map((s) => s.key)).toEqual([]); + }); + + it('honors the getSpoolDir contract', async () => { + const spool = h.driver.getSpoolDir?.() ?? null; + if (spool === null) return; // drivers with no local filesystem + expect(fs.existsSync(spool)).toBe(true); + // Spool contents are invisible to the abstraction by construction. + fs.writeFileSync(path.join(spool, 'leftover.part'), 'junk'); + expect((await listAll('')).map((s) => s.key)).toEqual([]); + }); + }); +} diff --git a/server/tests/unit/nest/storage/storage-jobs.service.test.ts b/server/tests/unit/nest/storage/storage-jobs.service.test.ts new file mode 100644 index 000000000..f4e3b2798 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-jobs.service.test.ts @@ -0,0 +1,609 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest'; + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + return { testDb: db, dbMock: { db, closeDb: () => {}, reinitialize: () => {} } }; +}); +vi.mock('../../../../src/db/database', () => dbMock); +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-jobs-test-key' })); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { Logger } from '@nestjs/common'; +import { createTables } from '../../../../src/db/schema'; +import { runMigrations } from '../../../../src/db/migrations'; +import { DatabaseService } from '../../../../src/nest/database/database.service'; +import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { MirrorDriver } from '../../../../src/nest/storage/drivers/mirror.driver'; +import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; +import { CATEGORIES_KEY, StorageRegistryService } from '../../../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import { + BackfillBusyError, + BackfillTargetError, + MigrationRequestError, + MigrationTargetError, + StorageJobsService, +} from '../../../../src/nest/storage/storage-jobs.service'; + +const db = new DatabaseService(testDb); + +beforeAll(() => { + createTables(testDb); + runMigrations(testDb); +}); + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-jobs-')); + tmpDirs.push(dir); + return dir; +} +function setSetting(key: string, value: string): void { + testDb.prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)').run(key, value); +} +beforeEach(() => { + testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run(); +}); +afterEach(() => { + vi.restoreAllMocks(); + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +/** A registry whose backups category routes through mirror 'm' (nas replica). */ +function makeWorld() { + const uploadsRoot = makeTmpDir(); + const backupsRoot = makeTmpDir(); + const nasRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + { name: 'nas', type: 'local', options: { root: nasRoot } }, + { name: 'm', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'm' })); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + return { registry, storage, jobs, backupsRoot, nasRoot }; +} + +async function waitFor(predicate: () => boolean, ms = 5000, intervalMs = 20): Promise { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > ms) throw new Error('timed out waiting for job'); + await new Promise((r) => setTimeout(r, intervalMs)); + } +} + +/** A registry with 'files' explicitly routed to 'uploads-local' and an unassigned 'dest-local' target. */ +function makeMigrationWorld() { + const uploadsRoot = makeTmpDir(); + const backupsRoot = makeTmpDir(); + const destRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + { name: 'dest-local', type: 'local', options: { root: destRoot } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ files: 'uploads-local' })); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + return { registry, storage, jobs, uploadsRoot, destRoot }; +} + +/** + * A registry with TREK_PLACE_PHOTO_DIR set, so 'photos-google' defaults to + * the bare-key mode-A backend 'place-photos-local' (audit #8's source shape), + * plus an unassigned 'dest-local' target that uses the normal prefixed + * (mode-B) layout for every other backend. + */ +function makePhotosGoogleMigrationWorld() { + const uploadsRoot = makeTmpDir(); + const backupsRoot = makeTmpDir(); + const placePhotoRoot = makeTmpDir(); + const destRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + { name: 'dest-local', type: 'local', options: { root: destRoot } }, + ]), + ); + const env = { env: () => ({ paths: { placePhotoDir: placePhotoRoot } }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + return { registry, storage, jobs, placePhotoRoot, destRoot }; +} + +/** Combines the migration world's 'dest-local' target with the backfill world's routed mirror 'm'. */ +function makeMigrationBackfillWorld() { + const uploadsRoot = makeTmpDir(); + const backupsRoot = makeTmpDir(); + const nasRoot = makeTmpDir(); + const destRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + { name: 'nas', type: 'local', options: { root: nasRoot } }, + { name: 'dest-local', type: 'local', options: { root: destRoot } }, + { name: 'm', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'm', files: 'uploads-local' })); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + return { registry, storage, jobs, uploadsRoot, backupsRoot, nasRoot, destRoot }; +} + +/** Reads the raw 'storage.categories' app_settings row (undefined key ⇒ {}). */ +function registryCategoriesRow(): Record { + const row = testDb.prepare('SELECT value FROM app_settings WHERE key = ?').get(CATEGORIES_KEY) as + | { value: string } + | undefined; + return row ? (JSON.parse(row.value) as Record) : {}; +} + +/** Polls migrationStatuses() for the named category to leave 'running', bounded ~5s. */ +async function waitTerminal(jobs: StorageJobsService, category: string) { + await waitFor(() => jobs.migrationStatuses().some((m) => m.category === category && m.status !== 'running')); + return jobs.migrationStatuses().find((m) => m.category === category)!; +} + +describe('StorageJobsService', () => { + it('JOBS-001 backfills the categories routed through the mirror and lands on done', async () => { + const { storage, jobs, nasRoot } = makeWorld(); + await storage.put('backups', 'old-backup.zip', Readable.from('zipzip')); + fs.rmSync(path.join(nasRoot, 'old-backup.zip'), { force: true }); // simulate pre-mirror object + jobs.startBackfill('m'); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.status === 'done')); + const status = jobs.statuses().find((s) => s.backend === 'm')!; + expect(status).toMatchObject({ status: 'done', total: 1, done: 1, copied: 1, failed: 0, deleted: 0 }); + expect(fs.existsSync(path.join(nasRoot, 'old-backup.zip'))).toBe(true); + }); + + it('JOBS-002 rejects an unrouted or non-mirror name with BackfillTargetError', () => { + const { jobs } = makeWorld(); + expect(() => jobs.startBackfill('nas')).toThrow(BackfillTargetError); + expect(() => jobs.startBackfill('ghost')).toThrow(BackfillTargetError); + }); + + it('JOBS-003 a second start while one runs throws BackfillBusyError (global, either backend)', async () => { + const { storage, jobs } = makeWorld(); + for (let i = 0; i < 20; i++) await storage.put('backups', `b${i}.zip`, Readable.from('x'.repeat(1000))); + jobs.startBackfill('m'); + expect(() => jobs.startBackfill('m')).toThrow(BackfillBusyError); + await waitFor(() => jobs.statuses().some((s) => s.status !== 'running')); + }); + + it('JOBS-004 cancel flips a running job to cancelled; cancelling a finished/unknown one returns false', async () => { + const { storage, jobs } = makeWorld(); + for (let i = 0; i < 50; i++) await storage.put('backups', `c${i}.zip`, Readable.from('y'.repeat(2000))); + jobs.startBackfill('m'); + expect(jobs.cancelBackfill('m')).toBe(true); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.status !== 'running')); + expect(jobs.statuses().find((s) => s.backend === 'm')!.status).toBe('cancelled'); + expect(jobs.cancelBackfill('m')).toBe(false); + expect(jobs.cancelBackfill('ghost')).toBe(false); + }); + + it('JOBS-005 finished statuses expire after the TTL', async () => { + vi.useFakeTimers(); + try { + const { storage, jobs } = makeWorld(); + await storage.put('backups', 'one.zip', Readable.from('z')); + jobs.startBackfill('m'); + // Drain the real async job under fake timers by flushing microtasks. + await vi.waitFor(() => expect(jobs.statuses()[0]?.status).toBe('done')); + vi.advanceTimersByTime(10 * 60_000 + 1000); + expect(jobs.statuses()).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); + + it('JOBS-006 an Error rejection from driver.backfill lands the job on "error" with its message, and is logged', async () => { + const { jobs } = makeWorld(); + const backfillSpy = vi.spyOn(MirrorDriver.prototype, 'backfill').mockRejectedValueOnce(new Error('replica offline')); + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + jobs.startBackfill('m'); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.status === 'error')); + const status = jobs.statuses().find((s) => s.backend === 'm')!; + expect(status.error).toBe('replica offline'); + expect(errorSpy).toHaveBeenCalledWith("backfill 'm' aborted: replica offline"); + backfillSpy.mockRestore(); + }); + + it('JOBS-006b a non-Error rejection is stringified rather than crashing', async () => { + const { jobs } = makeWorld(); + const backfillSpy = vi.spyOn(MirrorDriver.prototype, 'backfill').mockRejectedValueOnce('replica gone'); + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + jobs.startBackfill('m'); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.status === 'error')); + expect(jobs.statuses().find((s) => s.backend === 'm')!.error).toBe('replica gone'); + backfillSpy.mockRestore(); + }); + + it('JOBS-007 withTtl builds a service whose finished jobs expire on the given (short) TTL, not the 10-minute default', async () => { + const { registry, storage } = makeWorld(); + await storage.put('backups', 'q.zip', Readable.from('q')); + const jobs = StorageJobsService.withTtl(registry, 50); + jobs.startBackfill('m'); + await waitFor(() => jobs.statuses().some((s) => s.status === 'done')); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(jobs.statuses()).toEqual([]); + }); + + it('JOBS-008 while a backfill is running, starting an unknown/non-mirror name throws BackfillTargetError, not BackfillBusyError', async () => { + const { storage, jobs } = makeWorld(); + for (let i = 0; i < 20; i++) await storage.put('backups', `d${i}.zip`, Readable.from('x'.repeat(1000))); + jobs.startBackfill('m'); + expect(() => jobs.startBackfill('ghost')).toThrow(BackfillTargetError); + expect(() => jobs.startBackfill('nas')).toThrow(BackfillTargetError); + await waitFor(() => jobs.statuses().some((s) => s.status !== 'running')); + }); +}); + +describe('StorageJobsService migrations', () => { + it('MIG-001 happy path: copies, flips the category, sweeps a raced write, tallies reclaimable', async () => { + const { storage, jobs, uploadsRoot, destRoot } = makeMigrationWorld(); + // Sized up (not slept) so the copy phase has real work left when we poll + // for enumeration-done — a few bytes would finish before the first poll tick. + const big = 'a'.repeat(4_000_000); + await storage.put('files', 'a.txt', Readable.from(big)); + await storage.put('files', 'b.txt', Readable.from(big.replace(/a/g, 'b'))); + + jobs.startMigration('files', 'dest-local'); + // Enumeration done — write the raced third object before the copy phase settles. + // Tight poll interval: the window between "total > 0" and job completion is narrow. + await waitFor( + () => jobs.migrationStatuses().some((m) => m.category === 'files' && m.total > 0), + 5000, + 1, + ); + fs.writeFileSync(path.join(uploadsRoot, 'files', 'c.txt'), 'ccc'); + + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('done'); + // 2 copy-phase + 1 sweep — but the raced write may instead land in the + // copy phase's own (re-run) list(), so assert combined coverage, not + // which phase caught it. + expect(final.copied + final.skipped).toBeGreaterThanOrEqual(3); + expect(registryCategoriesRow().files).toBe('dest-local'); // flip persisted + expect(fs.existsSync(path.join(destRoot, 'files', 'a.txt'))).toBe(true); + expect(fs.existsSync(path.join(destRoot, 'files', 'b.txt'))).toBe(true); + expect(fs.existsSync(path.join(destRoot, 'files', 'c.txt'))).toBe(true); + expect(final.reclaimable).toEqual({ objects: 3, bytes: expect.any(Number) }); // sources kept + }); + + it('MIG-002 a copy failure blocks the flip and ends failed', async () => { + const { storage, jobs, destRoot } = makeMigrationWorld(); + await storage.put('files', 'a.txt', Readable.from('aaa')); + await storage.put('files', 'b.txt', Readable.from('bbb')); + // Pre-create a DIRECTORY at the target key path so the put's rename fails. + fs.mkdirSync(path.join(destRoot, 'files', 'a.txt'), { recursive: true }); + + jobs.startMigration('files', 'dest-local'); + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('failed'); + expect(final.failed).toBeGreaterThan(0); + expect(registryCategoriesRow().files).toBe('uploads-local'); // NOT flipped + }); + + it('MIG-003 cancel before the flip leaves everything untouched', async () => { + const { storage, jobs } = makeMigrationWorld(); + // Many, largish objects (not sleeps) so the copy phase is still running + // when we poll for done >= 1 and issue the cancel. + for (let i = 0; i < 100; i++) await storage.put('files', `f${i}.txt`, Readable.from('x'.repeat(100_000))); + + jobs.startMigration('files', 'dest-local'); + await waitFor(() => jobs.migrationStatuses().some((m) => m.category === 'files' && m.done >= 1), 5000, 1); + expect(jobs.cancelMigration('files')).toBe(true); + + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('cancelled'); + expect(registryCategoriesRow().files).toBe('uploads-local'); + }); + + it('MIG-007 a cancel landing after the copy loop but before the flip ends cancelled, categories row untouched', async () => { + // Deterministic route (no timing race): an EMPTY category never enters + // either copy loop, so the only place a cancel can be observed is the + // guard between the failed-check and the flip. `cancelMigration` is + // called synchronously right after `startMigration` returns — the + // detached runMigration's first `for await` suspends on a microtask + // before doing any work, so this synchronous call always lands the + // cancel flag before the async function resumes. + const { jobs } = makeMigrationWorld(); // 'files' has zero objects + jobs.startMigration('files', 'dest-local'); + expect(jobs.cancelMigration('files')).toBe(true); + + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('cancelled'); + expect(registryCategoriesRow().files).toBe('uploads-local'); // never flipped + }); + + it('MIG-004 validations: 400s, 404, and 409 against a running backfill (and vice versa)', async () => { + const { storage, jobs } = makeMigrationBackfillWorld(); + expect(() => jobs.startMigration('files', 'uploads-local')).toThrow(MigrationRequestError); // to === current + expect(() => jobs.startMigration('nope' as never, 'dest-local')).toThrow(MigrationRequestError); + expect(() => jobs.startMigration('files', 'ghost')).toThrow(MigrationTargetError); + + // A running backfill blocks a migration start. + for (let i = 0; i < 20; i++) await storage.put('backups', `b${i}.zip`, Readable.from('x'.repeat(1000))); + jobs.startBackfill('m'); + expect(() => jobs.startMigration('files', 'dest-local')).toThrow(BackfillBusyError); + await waitFor(() => jobs.statuses().some((s) => s.status !== 'running')); + + // A running migration blocks a backfill start too. + await storage.put('files', 'a.txt', Readable.from('a'.repeat(5000))); + jobs.startMigration('files', 'dest-local'); + expect(() => jobs.startBackfill('m')).toThrow(BackfillBusyError); + await waitFor(() => jobs.migrationStatuses().some((m) => m.status !== 'running')); + }); + + it('MIG-005 an object already on the target with a matching size is skipped in the copy phase itself', async () => { + const { storage, jobs, destRoot } = makeMigrationWorld(); + await storage.put('files', 'a.txt', Readable.from('aaa')); + // Pre-populate the destination with a byte-identical copy — the shape of + // a retried/resumed migration, not just the delta sweep's territory. + fs.mkdirSync(path.join(destRoot, 'files'), { recursive: true }); + fs.writeFileSync(path.join(destRoot, 'files', 'a.txt'), 'aaa'); + + jobs.startMigration('files', 'dest-local'); + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('done'); + expect(final.skipped).toBeGreaterThanOrEqual(1); + expect(final.copied).toBe(0); + expect(registryCategoriesRow().files).toBe('dest-local'); + }); + + it('MIG-006 a sweep-phase copy failure is reported without undoing the already-flipped category', async () => { + const { storage, jobs, registry, uploadsRoot, destRoot } = makeMigrationWorld(); + await storage.put('files', 'a.txt', Readable.from('aaa')); + + // Grab the exact driver instance startMigration will resolve and keep for + // its whole run (the in-flight guarantee), then wrap list() so the THIRD + // call — the delta sweep's own re-list, after phase 1a's total count and + // phase 1b's copy — races a content change into the source right as the + // sweep starts, with the destination path blocked so the sweep's re-copy + // attempt fails without touching the flip that already happened. + const { driver: sourceDriver } = registry.resolve('files'); + const originalList = sourceDriver.list.bind(sourceDriver); + let listCalls = 0; + vi.spyOn(sourceDriver, 'list').mockImplementation((prefix: string) => { + listCalls += 1; + if (listCalls === 3) { + fs.writeFileSync(path.join(uploadsRoot, 'files', 'a.txt'), 'aaaa'); + fs.rmSync(path.join(destRoot, 'files', 'a.txt'), { force: true }); + fs.mkdirSync(path.join(destRoot, 'files', 'a.txt'), { recursive: true }); + } + return originalList(prefix); + }); + + jobs.startMigration('files', 'dest-local'); + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('done'); // sweep failures don't undo a completed flip + expect(final.failed).toBeGreaterThanOrEqual(1); + expect(registryCategoriesRow().files).toBe('dest-local'); + }); + + it('MIG-008 photos-google mode-A (bare "" prefix) migrating to a prefixed backend rewrites destination keys (audit #8)', async () => { + const { storage, jobs, placePhotoRoot, destRoot } = makePhotosGoogleMigrationWorld(); + await storage.put('photos-google', 'abc.jpg', Readable.from('img-a')); + await storage.put('photos-google', 'sub/def.jpg', Readable.from('img-b')); + + jobs.startMigration('photos-google', 'dest-local'); + const final = await waitTerminal(jobs, 'photos-google'); + + expect(final.status).toBe('done'); + expect(final.copied).toBe(2); + expect(final.failed).toBe(0); + // Source stayed bare-keyed (mode A) — untouched. + expect(fs.existsSync(path.join(placePhotoRoot, 'abc.jpg'))).toBe(true); + expect(fs.existsSync(path.join(placePhotoRoot, 'sub', 'def.jpg'))).toBe(true); + // Destination lands under the category's normal prefixed (mode-B) layout — + // NOT at the bare source key, which the registry would never resolve. + expect(fs.existsSync(path.join(destRoot, 'photos', 'google', 'abc.jpg'))).toBe(true); + expect(fs.existsSync(path.join(destRoot, 'photos', 'google', 'sub', 'def.jpg'))).toBe(true); + expect(fs.existsSync(path.join(destRoot, 'abc.jpg'))).toBe(false); // not left at the bare source key + expect(registryCategoriesRow()['photos-google']).toBe('dest-local'); + }); + + it('MIG-009 the skip-check on a rewritten-prefix migration stats the DESTINATION key, not the source key', async () => { + const { storage, jobs, destRoot } = makePhotosGoogleMigrationWorld(); + await storage.put('photos-google', 'abc.jpg', Readable.from('same-bytes')); + // Pre-populate the destination at the REWRITTEN key with matching size — + // if the skip-check mistakenly stat'd the bare source key on the target, + // it would find nothing and copy instead of skipping. + fs.mkdirSync(path.join(destRoot, 'photos', 'google'), { recursive: true }); + fs.writeFileSync(path.join(destRoot, 'photos', 'google', 'abc.jpg'), 'same-bytes'); + + jobs.startMigration('photos-google', 'dest-local'); + const final = await waitTerminal(jobs, 'photos-google'); + + expect(final.status).toBe('done'); + expect(final.skipped).toBeGreaterThanOrEqual(1); + expect(final.copied).toBe(0); + expect(registryCategoriesRow()['photos-google']).toBe('dest-local'); + }); + + it('MIG-010 the delta sweep rewrites a raced object\'s destination key too', async () => { + const { storage, jobs, placePhotoRoot, destRoot } = makePhotosGoogleMigrationWorld(); + const big = 'x'.repeat(2_000_000); + await storage.put('photos-google', 'a.jpg', Readable.from(big)); + + jobs.startMigration('photos-google', 'dest-local'); + await waitFor( + () => jobs.migrationStatuses().some((m) => m.category === 'photos-google' && m.total > 0), + 5000, + 1, + ); + // Race a second bare-keyed object in after enumeration but before the copy + // phase settles — the delta sweep must pick it up and rewrite its key too. + fs.writeFileSync(path.join(placePhotoRoot, 'raced.jpg'), 'raced-bytes'); + + const final = await waitTerminal(jobs, 'photos-google'); + expect(final.status).toBe('done'); + expect(fs.existsSync(path.join(destRoot, 'photos', 'google', 'a.jpg'))).toBe(true); + expect(fs.existsSync(path.join(destRoot, 'photos', 'google', 'raced.jpg'))).toBe(true); + expect(registryCategoriesRow()['photos-google']).toBe('dest-local'); + }); + + it('MIG-011 equal-prefix migrations (files -> dest-local) stay byte-identical: destination key equals the source key', async () => { + const { storage, jobs, destRoot } = makeMigrationWorld(); + await storage.put('files', 'a.txt', Readable.from('aaa')); + + jobs.startMigration('files', 'dest-local'); + const final = await waitTerminal(jobs, 'files'); + + expect(final.status).toBe('done'); + expect(final.copied).toBe(1); + // Both source and destination use the same 'files/' prefix — the key is + // unchanged, exactly today's pre-fix behavior. + expect(fs.existsSync(path.join(destRoot, 'files', 'a.txt'))).toBe(true); + expect(registryCategoriesRow().files).toBe('dest-local'); + }); +}); + +describe('StorageJobsService.cancelJobsForMissingBackends', () => { + it('JOBS-020 cancels a running backfill whose mirror left the config', async () => { + const { storage, jobs, registry } = makeWorld(); + for (let i = 0; i < 50; i++) await storage.put('backups', `c${i}.zip`, Readable.from('y'.repeat(2000))); + jobs.startBackfill('m'); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.done >= 1), 5000, 1); + + // Simulate a config save that drops mirror 'm' entirely — 'backups' reverts + // to the built-in default ('backups-local'), a self-consistent config. + setSetting('storage.backends', JSON.stringify([])); + setSetting('storage.categories', JSON.stringify({})); + registry.reload(); + + jobs.cancelJobsForMissingBackends(); + await waitFor(() => jobs.statuses().some((s) => s.backend === 'm' && s.status !== 'running')); + const status = jobs.statuses().find((s) => s.backend === 'm')!; + expect(status.status).toBe('cancelled'); + }); + + it('JOBS-021 cancels a running migration whose source or target backend is gone', async () => { + // A category migrating away from a non-built-in backend ('nas') to another + // non-built-in backend ('dest-local') — both can genuinely disappear from + // a config save, unlike the always-present 'uploads-local'/'backups-local'. + const uploadsRoot = makeTmpDir(); + const nasRoot = makeTmpDir(); + const destRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'nas', type: 'local', options: { root: nasRoot } }, + { name: 'dest-local', type: 'local', options: { root: destRoot } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ files: 'nas' })); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const jobs = new StorageJobsService(registry); + + for (let i = 0; i < 100; i++) await storage.put('files', `f${i}.txt`, Readable.from('x'.repeat(100_000))); + jobs.startMigration('files', 'dest-local'); + await waitFor(() => jobs.migrationStatuses().some((m) => m.category === 'files' && m.done >= 1), 5000, 1); + + // Simulate a config save: 'files' is rerouted to 'uploads-local' and the + // migration's in-flight FROM backend ('nas') is dropped from the config + // entirely — self-consistent (validateConfig only checks the new + // category map), but the running migration's `from` no longer resolves. + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'dest-local', type: 'local', options: { root: destRoot } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ files: 'uploads-local' })); + registry.reload(); + + jobs.cancelJobsForMissingBackends(); + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('cancelled'); + }); + + it('JOBS-022 cancels a running migration re-routed to a third, still-defined backend (route no longer matches `from`)', async () => { + const { storage, jobs, registry } = makeMigrationWorld(); // 'files' -> 'uploads-local'; 'dest-local' is the target + for (let i = 0; i < 100; i++) await storage.put('files', `f${i}.txt`, Readable.from('x'.repeat(100_000))); + jobs.startMigration('files', 'dest-local'); + await waitFor(() => jobs.migrationStatuses().some((m) => m.category === 'files' && m.done >= 1), 5000, 1); + + // A config save re-routes 'files' to a THIRD backend — both the + // migration's own from ('uploads-local') and to ('dest-local') backends + // still exist (the migration's already-resolved source driver instance + // is unaffected by the roots below — the in-flight guarantee), so the + // existing missing-backend check wouldn't catch this. + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: makeTmpDir() } }, + { name: 'backups-local', type: 'local', options: { root: makeTmpDir() } }, + { name: 'dest-local', type: 'local', options: { root: makeTmpDir() } }, + { name: 'third-local', type: 'local', options: { root: makeTmpDir() } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ files: 'third-local' })); + registry.reload(); + + jobs.cancelJobsForMissingBackends(); + const final = await waitTerminal(jobs, 'files'); + expect(final.status).toBe('cancelled'); + expect(registryCategoriesRow().files).toBe('third-local'); // the save's reroute stands — no flip + }); + + it('JOBS-023 a save that does not touch the migrating category leaves it running', async () => { + const { storage, jobs, registry } = makeMigrationWorld(); + for (let i = 0; i < 100; i++) await storage.put('files', `f${i}.txt`, Readable.from('x'.repeat(100_000))); + jobs.startMigration('files', 'dest-local'); + await waitFor(() => jobs.migrationStatuses().some((m) => m.category === 'files' && m.done >= 1), 5000, 1); + + // A save that touches an unrelated category only — 'files' route is untouched. + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: makeTmpDir() } }, + { name: 'backups-local', type: 'local', options: { root: makeTmpDir() } }, + { name: 'dest-local', type: 'local', options: { root: makeTmpDir() } }, + ]), + ); + // Re-declare the original mapping unchanged (files still on uploads-local). + setSetting('storage.categories', JSON.stringify({ files: 'uploads-local' })); + + jobs.cancelJobsForMissingBackends(); + expect(jobs.migrationStatuses().find((m) => m.category === 'files')!.status).toBe('running'); + + expect(jobs.cancelMigration('files')).toBe(true); // clean up + await waitTerminal(jobs, 'files'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-keys.test.ts b/server/tests/unit/nest/storage/storage-keys.test.ts new file mode 100644 index 000000000..c6d2d6e90 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-keys.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect } from 'vitest'; +import path from 'node:path'; +import { + isValidKey, + assertValidKey, + isValidPrefix, + assertValidPrefix, +} from '../../../../src/nest/storage/storage-keys'; +import { StorageInvalidKeyError } from '../../../../src/nest/storage/storage.types'; +import { + DEFAULT_UPLOADS_ROOT, + DEFAULT_BACKUPS_ROOT, + GLOBAL_TEMP_DIR, +} from '../../../../src/nest/storage/storage-paths'; + +// Central key validation (spec: storage-keys.ts owns the rules; drivers and +// routes never re-implement them). Every rejected shape here replaces one of +// the scattered per-route guards the refactor retires. +describe('storage keys', () => { + const validKeys = [ + 'files/a.pdf', + 'journey/thumbs/ab12.jpg', + 'backup-2026.zip', // backups category uses bare keys (spec rev 3.1) + 'photos/trek/deadbeef.bin', + // legacy photos/ keys are user-named flat files — spaces stay valid + 'photos/my holiday pic.jpg', + ]; + + const invalidKeys = [ + '', // empty + '/abs', // absolute + 'a\\b', // backslash + 'a//b', // empty segment + 'a/', // trailing slash = empty segment + '../x', // traversal + 'a/../b', // embedded traversal + '.', // dot segment + '..', // dot-dot segment + '.tmp/x', // leading-dot segment (spool must be unreachable) + 'a/.hidden', // dotfile segment + 'a\u0000b', // control character + `a/${'x'.repeat(1024)}`, // over length cap + ]; + + it.each(validKeys)('accepts %j', (key) => { + expect(isValidKey(key)).toBe(true); + expect(() => assertValidKey(key)).not.toThrow(); + }); + + it.each(invalidKeys)('rejects %j', (key) => { + expect(isValidKey(key)).toBe(false); + expect(() => assertValidKey(key)).toThrow(StorageInvalidKeyError); + }); + + it('caps keys at 1024 chars (boundary)', () => { + const exactly1024 = `a/${'x'.repeat(1022)}`; + expect(exactly1024.length).toBe(1024); + expect(isValidKey(exactly1024)).toBe(true); + expect(isValidKey(`${exactly1024}x`)).toBe(false); + }); + + describe('prefixes', () => { + it.each(['', 'thumbs/', 'thumbs', 'photos/google/'])('accepts %j', (prefix) => { + expect(isValidPrefix(prefix)).toBe(true); + expect(() => assertValidPrefix(prefix)).not.toThrow(); + }); + + it.each(['../', '/abs', '.tmp/', 'a//b/', 'a\\b'])('rejects %j', (prefix) => { + expect(isValidPrefix(prefix)).toBe(false); + expect(() => assertValidPrefix(prefix)).toThrow(StorageInvalidKeyError); + }); + }); +}); + +// Depth contract for the storage domain's one __dirname anchor (the guard the +// deleted memories/uploads-root.ts pin used to carry): the resolved defaults +// must equal the paths the server has always used, under both src (vitest) and +// dist layouts. +describe('storage default paths', () => { + it('anchors the default uploads root at /uploads', () => { + expect(DEFAULT_UPLOADS_ROOT).toBe(path.resolve(process.cwd(), 'uploads')); + }); + + it('anchors the default backups root at /data/backups', () => { + expect(DEFAULT_BACKUPS_ROOT).toBe(path.resolve(process.cwd(), 'data/backups')); + }); + + it('anchors the global temp dir at /data/tmp', () => { + expect(GLOBAL_TEMP_DIR).toBe(path.resolve(process.cwd(), 'data/tmp')); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-probe.test.ts b/server/tests/unit/nest/storage/storage-probe.test.ts new file mode 100644 index 000000000..be4ae2720 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-probe.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import { S3Driver } from '../../../../src/nest/storage/drivers/s3.driver'; +import { ephemeralDriverFor, probeDriver } from '../../../../src/nest/storage/storage-probe'; +import { StorageBackendError, type StorageDriver } from '../../../../src/nest/storage/storage.types'; + +function makeTmpDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), 'trek-probe-')); +} + +describe('probeDriver', () => { + it('PROBE-001 round-trips put/stat/delete against a healthy local driver and leaves no object', async () => { + const root = makeTmpDir(); + const driver = new LocalDriver({ id: 'probe-t', root }); + driver.init({ ensurePrefixes: [], cleanSpool: false }); + const result = await probeDriver('t', driver); + expect(result).toEqual({ name: 't', ok: true }); + // The probe object itself is gone (an empty trek-probe/ dir may remain — inert). + const leftovers: string[] = []; + for await (const obj of driver.list('')) leftovers.push(obj.key); + expect(leftovers).toEqual([]); + }); + + it('PROBE-002 captures a failing driver as ok:false with the error message', async () => { + const driver = { + id: 'boom', + put: vi.fn().mockRejectedValue(new Error('disk on fire')), + stat: vi.fn(), + delete: vi.fn(), + getStream: vi.fn(), + list: vi.fn(), + } as unknown as StorageDriver; + const result = await probeDriver('boom', driver); + expect(result.ok).toBe(false); + expect(result.error).toContain('disk on fire'); + }); + + it('PROBE-005 appends the wrapped cause chain to the target error message', async () => { + const raw = new Error('connect ECONNREFUSED 127.0.0.1:9000'); + const driver = { + id: 'p', + put: vi.fn().mockRejectedValue(new StorageBackendError("put failed for 'k' on 'probe-s3'", raw)), + stat: vi.fn(), + delete: vi.fn(), + getStream: vi.fn(), + list: vi.fn(), + } as unknown as StorageDriver; + const result = await probeDriver('p', driver); + expect(result.ok).toBe(false); + expect(result.error).toContain("put failed for 'k' on 'probe-s3'"); + expect(result.error).toContain('ECONNREFUSED'); + }); + + it("PROBE-006 appends an S3-style error code that isn't already in the cause's message", async () => { + const raw = Object.assign(new Error('The specified bucket does not exist'), { code: 'NoSuchBucket' }); + const driver = { + id: 'p', + put: vi.fn().mockRejectedValue(new StorageBackendError("put failed for 'k' on 'probe-s3'", raw)), + stat: vi.fn(), + delete: vi.fn(), + getStream: vi.fn(), + list: vi.fn(), + } as unknown as StorageDriver; + const result = await probeDriver('p', driver); + expect(result.ok).toBe(false); + expect(result.error).toContain('The specified bucket does not exist'); + expect(result.error).toContain('NoSuchBucket'); + }); +}); + +describe('ephemeralDriverFor', () => { + it('PROBE-003 builds an initialized LocalDriver for local and an S3Driver for s3 — no registry involved', () => { + const local = ephemeralDriverFor({ name: 'l', type: 'local', options: { root: makeTmpDir() } }); + expect(local).toBeInstanceOf(LocalDriver); + const s3 = ephemeralDriverFor({ + name: 's', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:1', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk', + region: 'us-east-1', + keyPrefix: '', + retries: 0, + timeoutMs: 200, + }, + }); + expect(s3).toBeInstanceOf(S3Driver); + }); + + it('PROBE-004 an uncreatable local root surfaces as a throw (the service turns it into a target error)', () => { + const file = path.join(makeTmpDir(), 'a-file'); + fs.writeFileSync(file, 'not a dir'); + expect(() => ephemeralDriverFor({ name: 'l', type: 'local', options: { root: file } })).toThrow(); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-registry.service.test.ts b/server/tests/unit/nest/storage/storage-registry.service.test.ts new file mode 100644 index 000000000..a6561dd96 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-registry.service.test.ts @@ -0,0 +1,971 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest'; +import { Logger } from '@nestjs/common'; + +// ── DB setup (the permissions.service.test.ts pattern: real in-memory SQLite +// so the app_settings SQL is exercised faithfully) ──────────────────────────── + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + db.exec('PRAGMA busy_timeout = 5000'); + return { testDb: db, dbMock: { db, closeDb: () => {}, reinitialize: () => {} } }; +}); + +vi.mock('../../../../src/db/database', () => dbMock); +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-registry-test-key' })); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { createTables } from '../../../../src/db/schema'; +import { runMigrations } from '../../../../src/db/migrations'; +import { DatabaseService } from '../../../../src/nest/database/database.service'; +import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto'; +import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; +import { StorageRegistryService } from '../../../../src/nest/storage/storage-registry.service'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import { MirrorDriver, type ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver'; +import { S3Driver } from '../../../../src/nest/storage/drivers/s3.driver'; +import { + GLOBAL_TEMP_DIR, + DEFAULT_BACKUPS_ROOT, + DEFAULT_UPLOADS_ROOT, + SEED_CONFIG_PATH, +} from '../../../../src/nest/storage/storage-paths'; +import { STORAGE_CATEGORIES } from '../../../../src/nest/storage/storage.types'; + +const db = new DatabaseService(testDb); + +beforeAll(() => { + createTables(testDb); + runMigrations(testDb); +}); + +// ── helpers ─────────────────────────────────────────────────────────────────── + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-registry-')); + tmpDirs.push(dir); + return dir; +} + +interface EnvPaths { + placePhotoDir?: string; +} + +function makeEnvStub(initial: EnvPaths): { env: RuntimeEnvService } { + const paths = initial; + return { + env: { env: () => ({ paths }) } as unknown as RuntimeEnvService, + }; +} + +function setSetting(key: string, value: string): void { + testDb.prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)').run(key, value); +} + +function uploadsOverride(root: string): unknown { + return { name: 'uploads-local', type: 'local', options: { root } }; +} + +interface RegistryOpts { + uploadsRoot?: string; + placePhotoDir?: string; + boot?: boolean; + /** Extra storage.backends entries, appended after the uploads-local override. */ + backends?: unknown[]; + categories?: Record; +} + +/** + * The uploads-local root is the computed default, relocatable only via a + * settings override row bearing the built-in's name (first-class + * merge-by-name). The helper seeds that override so unit runs never write + * into the real server/uploads. + */ +function makeRegistry(opts: RegistryOpts = {}) { + const uploadsRoot = opts.uploadsRoot ?? makeTmpDir(); + setSetting('storage.backends', JSON.stringify([uploadsOverride(uploadsRoot), ...(opts.backends ?? [])])); + if (opts.categories) setSetting('storage.categories', JSON.stringify(opts.categories)); + const stub = makeEnvStub({ placePhotoDir: opts.placePhotoDir }); + const registry = new StorageRegistryService(db, stub.env, new StorageEventsService()); + if (opts.boot !== false) registry.onModuleInit(); + return { registry, uploadsRoot, setUploadsRoot: (root: string) => rewriteUploadsOverride(root) }; +} + +/** For reload tests: swap the override row's root in place. */ +function rewriteUploadsOverride(root: string): void { + const row = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as + | { value: string } + | undefined; + const entries = row?.value ? (JSON.parse(row.value) as Array<{ name: string; options: { root: string } }>) : []; + const next = entries.map((e) => (e.name === 'uploads-local' ? { ...e, options: { root } } : e)); + setSetting('storage.backends', JSON.stringify(next)); +} + +beforeEach(() => { + testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run(); +}); + +afterEach(() => { + vi.restoreAllMocks(); + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +// ── defaults ────────────────────────────────────────────────────────────────── + +describe('StorageRegistryService defaults', () => { + it('resolves every category with the built-in defaults and no settings rows', () => { + const { registry, uploadsRoot } = makeRegistry(); + + for (const category of STORAGE_CATEGORIES) { + expect(registry.resolve(category)).toBeDefined(); + } + + const files = registry.resolve('files'); + expect(files.backendName).toBe('uploads-local'); + expect(files.keyPrefix).toBe('files/'); + expect(files.driver).toBeInstanceOf(LocalDriver); + expect(files.driver.getLocalPath!('files/x.pdf')).toBe(path.join(fs.realpathSync(uploadsRoot), 'files/x.pdf')); + + // backups: bare keys, root IS the backups dir (spec rev 3.1) + const backups = registry.resolve('backups'); + expect(backups.backendName).toBe('backups-local'); + expect(backups.keyPrefix).toBe(''); + expect(backups.driver.getLocalPath!('backup-1.zip')).toBe(path.join(fs.realpathSync(DEFAULT_BACKUPS_ROOT), 'backup-1.zip')); + + // photos-google without TREK_PLACE_PHOTO_DIR: today's layout under uploads + const googlePhotos = registry.resolve('photos-google'); + expect(googlePhotos.backendName).toBe('uploads-local'); + expect(googlePhotos.keyPrefix).toBe('photos/google/'); + }); + + it('honors TREK_PLACE_PHOTO_DIR via the conditional third backend (mode B: bare keys)', () => { + const photoDir = makeTmpDir(); + const { registry } = makeRegistry({ placePhotoDir: photoDir }); + + const googlePhotos = registry.resolve('photos-google'); + expect(googlePhotos.backendName).toBe('place-photos-local'); + expect(googlePhotos.keyPrefix).toBe(''); + expect(googlePhotos.driver.getLocalPath!('abc.jpg')).toBe(path.join(fs.realpathSync(photoDir), 'abc.jpg')); + + // Everything else stays on uploads-local, unchanged. + expect(registry.resolve('photos-trek').backendName).toBe('uploads-local'); + expect(registry.resolve('photos-trek').keyPrefix).toBe('photos/trek/'); + }); + + it('creates roots, spools, category prefix dirs, and the global temp dir on load', () => { + const { uploadsRoot } = makeRegistry(); + + for (const sub of ['.tmp', 'files', 'journey', 'covers', 'avatars', 'places', 'photos', 'photos/trek', 'photos/google']) { + expect(fs.statSync(path.join(uploadsRoot, sub)).isDirectory()).toBe(true); + } + expect(fs.statSync(GLOBAL_TEMP_DIR).isDirectory()).toBe(true); + expect(fs.statSync(path.join(DEFAULT_BACKUPS_ROOT, '.tmp')).isDirectory()).toBe(true); + }); + + it('roots uploads-local at the computed default when no override row exists', () => { + const { registry } = makeRegistry({ boot: false }); + testDb.prepare("DELETE FROM app_settings WHERE key = 'storage.backends'").run(); + registry.onModuleInit(); + const files = registry.resolve('files'); + expect(files.backendName).toBe('uploads-local'); + expect(files.driver.getLocalPath!('files/x.pdf')).toBe( + path.join(fs.realpathSync(DEFAULT_UPLOADS_ROOT), 'files/x.pdf'), + ); + }); + + it('a settings row bearing a built-in name replaces it (first-class override)', () => { + const relocated = makeTmpDir(); + const { registry } = makeRegistry({ uploadsRoot: relocated }); + expect(registry.resolve('files').driver.getLocalPath!('files/x.pdf')).toBe( + path.join(fs.realpathSync(relocated), 'files/x.pdf'), + ); + }); + + it('serves the legacy photos category from uploads-local without any config (ServedCategory seam)', () => { + const relocated = makeTmpDir(); + const { registry } = makeRegistry({ uploadsRoot: relocated }); + const photos = registry.resolve('photos'); + expect(photos.backendName).toBe('uploads-local'); + expect(photos.keyPrefix).toBe('photos/'); + // Override-following: relocating uploads moves the legacy photos with it. + expect(photos.driver.getLocalPath!('photos/x.jpg')).toBe( + path.join(fs.realpathSync(relocated), 'photos/x.jpg'), + ); + }); + + it('the admin snapshot exposes only the 8 configurable categories — photos is served, not configurable', () => { + const { registry } = makeRegistry(); + const keys = Object.keys(registry.snapshot().categories); + expect(keys).toHaveLength(8); + expect(keys).not.toContain('photos'); + }); +}); + +// ── keyPrefixFor (extracted for the category-migration job) ─────────────────── + +describe('StorageRegistryService keyPrefixFor', () => { + it('returns the plain category prefix regardless of which backend is asked', () => { + const { registry } = makeRegistry(); + expect(registry.keyPrefixFor('files', 'uploads-local')).toBe('files/'); + expect(registry.keyPrefixFor('files', 'some-other-backend')).toBe('files/'); + expect(registry.keyPrefixFor('backups', 'backups-local')).toBe(''); + expect(registry.keyPrefixFor('covers', 'anything')).toBe('covers/'); + }); + + it('photos-google is bare-key (mode A) ONLY on place-photos-local; every other backend gets the prefixed mode-B layout', () => { + const { registry } = makeRegistry(); + expect(registry.keyPrefixFor('photos-google', 'place-photos-local')).toBe(''); + expect(registry.keyPrefixFor('photos-google', 'uploads-local')).toBe('photos/google/'); + expect(registry.keyPrefixFor('photos-google', 'dest-local')).toBe('photos/google/'); + }); + + it('agrees with resolve() for the category\'s CURRENT backend — no drift between the two rules', () => { + const photoDir = makeTmpDir(); + const { registry } = makeRegistry({ placePhotoDir: photoDir }); + const resolved = registry.resolve('photos-google'); + expect(registry.keyPrefixFor('photos-google', resolved.backendName)).toBe(resolved.keyPrefix); + }); +}); + +// ── settings merge + validation ─────────────────────────────────────────────── + +describe('StorageRegistryService settings', () => { + it('merges app_settings backends/categories over the defaults (backup mirror)', () => { + const nasRoot = makeTmpDir(); + const { registry } = makeRegistry({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root: nasRoot } }, + { name: 'backup-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas-backups'] } }, + ], + categories: { backups: 'backup-mirror' }, + }); + const backups = registry.resolve('backups'); + expect(backups.backendName).toBe('backup-mirror'); + expect(backups.keyPrefix).toBe(''); + expect(backups.driver).toBeInstanceOf(MirrorDriver); + // untouched categories keep their defaults + expect(registry.resolve('files').backendName).toBe('uploads-local'); + }); + + it('a mirror may serve any category — the backups-only rule is lifted (replicas-on-primary spec)', () => { + const extraRoot = makeTmpDir(); + const { registry, uploadsRoot } = makeRegistry({ + backends: [ + { name: 'extra-local', type: 'local', options: { root: extraRoot } }, + { name: 'm', type: 'mirror', options: { primary: 'uploads-local', replicas: ['extra-local'] } }, + ], + categories: { covers: 'm' }, + }); + const covers = registry.resolve('covers'); + expect(covers.backendName).toBe('m'); + expect(covers.driver).toBeInstanceOf(MirrorDriver); + expect(covers.keyPrefix).toBe('covers/'); + // Hot-path reads stay free: getLocalPath delegates to the local primary. + expect(covers.driver.getLocalPath!('covers/x.jpg')).toBe( + path.join(fs.realpathSync(uploadsRoot), 'covers/x.jpg'), + ); + }); + + it.each([ + ['unknown backend name in categories', undefined, JSON.stringify({ files: 'nope' })], + [ + 'nested mirror', + JSON.stringify([ + { name: 'extra-local', type: 'local', options: { root: '/tmp/x' } }, + { name: 'm1', type: 'mirror', options: { primary: 'backups-local', replicas: ['extra-local'] } }, + { name: 'm2', type: 'mirror', options: { primary: 'm1', replicas: [] } }, + ]), + JSON.stringify({ backups: 'm2' }), + ], + [ + 'settings s3 backend with invalid options', + JSON.stringify([{ name: 'x', type: 's3', options: {} }]), + undefined, + ], + ['unknown category name', undefined, JSON.stringify({ 'not-a-category': 'uploads-local' })], + ['retired photos category in settings', undefined, JSON.stringify({ photos: 'uploads-local' })], + ['malformed JSON', 'not json at all', undefined], + ])('falls back to built-in defaults at boot on invalid settings: %s', (_label, backendsRow, categoriesRow) => { + const { registry } = makeRegistry({ boot: false }); + if (backendsRow !== undefined) setSetting('storage.backends', backendsRow); + if (categoriesRow !== undefined) setSetting('storage.categories', categoriesRow); + registry.onModuleInit(); + + expect(registry.resolve('files').backendName).toBe('uploads-local'); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + }); + + it('lastLoadError() is null on a clean boot', () => { + const { registry } = makeRegistry(); + expect(registry.lastLoadError()).toBeNull(); + }); + + it.each([ + ['unknown backend name in categories', undefined, JSON.stringify({ files: 'nope' })], + ['malformed JSON', 'not json at all', undefined], + ])( + 'lastLoadError() records the exact failure at boot and stays set (silent-fallback audit minor): %s', + (_label, backendsRow, categoriesRow) => { + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry({ boot: false }); + if (backendsRow !== undefined) setSetting('storage.backends', backendsRow); + if (categoriesRow !== undefined) setSetting('storage.categories', categoriesRow); + registry.onModuleInit(); + + expect(registry.lastLoadError()).not.toBeNull(); + // Still resolves on built-in defaults — the fallback itself is unchanged, + // only the silence around it. + expect(registry.resolve('files').backendName).toBe('uploads-local'); + }, + ); + + it('a malformed row never leaks a secret-looking token into lastLoadError() (admin-facing configError)', () => { + // JSON.parse's own SyntaxError can echo a snippet of the raw input around + // the failure position (Node 24 V8) — an unquoted value like this is + // exactly the shape a corrupted/hand-edited secretAccessKey row takes. + // lastLoadError() must never carry that snippet through to the + // admin-facing configError banner. + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry({ boot: false }); + const secretToken = 'AKIA_SUPER_SECRET_TOKEN_1234567890'; + setSetting('storage.backends', `{"secretAccessKey": ${secretToken}_undefined_broken}`); + registry.onModuleInit(); + + const err = registry.lastLoadError(); + expect(err).not.toBeNull(); + expect(err).not.toContain(secretToken); + expect(err).not.toContain('secretAccessKey'); + expect(err).toContain("'storage.backends' contains malformed JSON"); + }); + + it('lastLoadError() clears on the next successful load/reload', () => { + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry({ boot: false }); + setSetting('storage.categories', 'garbage {'); + registry.onModuleInit(); + expect(registry.lastLoadError()).not.toBeNull(); + + setSetting('storage.categories', '{}'); + registry.reload(); + expect(registry.lastLoadError()).toBeNull(); + }); + + it('keeps the last-good config (not defaults) when a reload() sees invalid settings', () => { + const nasRoot = makeTmpDir(); + const { registry } = makeRegistry({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root: nasRoot } }, + { name: 'backup-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas-backups'] } }, + ], + categories: { backups: 'backup-mirror' }, + }); + expect(registry.resolve('backups').backendName).toBe('backup-mirror'); + + setSetting('storage.categories', 'garbage {'); + registry.reload(); + + // last-good, i.e. the mirror config — NOT the built-in defaults + expect(registry.resolve('backups').backendName).toBe('backup-mirror'); + // ...and the operator can find out about it (audit minor: this used to be silent). + expect(registry.lastLoadError()).toBeTruthy(); + }); +}); + +// ── reload semantics ────────────────────────────────────────────────────────── + +describe('StorageRegistryService reload', () => { + it('swaps the map atomically: new resolves see the new instance, held refs stay usable', async () => { + const { registry, setUploadsRoot } = makeRegistry(); + const before = registry.resolve('files'); + await before.driver.put('files/pre-reload.bin', Readable.from('old root')); + + setUploadsRoot(makeTmpDir()); + registry.reload(); + + const after = registry.resolve('files'); + expect(after.driver).not.toBe(before.driver); + expect(await after.driver.stat('files/pre-reload.bin')).toBeNull(); // new root is empty + + // in-flight semantics: the held old driver still serves reads + const { stream } = await before.driver.getStream('files/pre-reload.bin'); + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Buffer)); + expect(Buffer.concat(chunks).toString()).toBe('old root'); + }); + + it('cleans spool leftovers at boot only, never on reload()', () => { + const uploadsRoot = makeTmpDir(); + const stray = path.join(uploadsRoot, '.tmp', 'in-flight.part'); + + fs.mkdirSync(path.dirname(stray), { recursive: true }); + fs.writeFileSync(stray, 'crash leftover'); + // Aged past the reap gate: the boot sweep spares fresh entries (another + // process may be spooling into the same tree — see LocalDriver.init). + const old = new Date(Date.now() - 2 * 60 * 60 * 1000); + fs.utimesSync(stray, old, old); + const { registry } = makeRegistry({ uploadsRoot }); + expect(fs.existsSync(stray)).toBe(false); // boot reclaimed it + + fs.writeFileSync(stray, 'in-flight upload'); + registry.reload(); + expect(fs.existsSync(stray)).toBe(true); // reload must not touch it + }); +}); + +// ── assignCategory ──────────────────────────────────────────────────────────── + +describe('StorageRegistryService assignCategory', () => { + it('REG-ASSIGN-001 throws on an unknown backend and persists nothing (belt-and-braces alongside the migration job\'s own guard)', () => { + const { registry } = makeRegistry(); + const before = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.categories'").get() as + | { value: string } + | undefined; + + expect(() => registry.assignCategory('files', 'ghost-backend')).toThrow( + "cannot assign 'files' to unknown backend 'ghost-backend'", + ); + + const after = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.categories'").get() as + | { value: string } + | undefined; + expect(after).toEqual(before); // no write happened + expect(registry.snapshot().categories.files.backend).toBe('uploads-local'); // unchanged + }); + + it('REG-ASSIGN-002 assigns and persists when the backend exists', () => { + const { registry } = makeRegistry({ backends: [{ name: 'dest-local', type: 'local', options: { root: makeTmpDir() } }] }); + + registry.assignCategory('files', 'dest-local'); + + expect(registry.snapshot().categories.files.backend).toBe('dest-local'); + const row = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.categories'").get() as { + value: string; + }; + expect((JSON.parse(row.value) as Record).files).toBe('dest-local'); + }); + + it('REG-ASSIGN-003 bumps the shared optimistic-concurrency version counter by exactly one, in the same write (audit #7)', () => { + const { registry } = makeRegistry({ backends: [{ name: 'dest-local', type: 'local', options: { root: makeTmpDir() } }] }); + expect(registry.currentConfigVersion()).toBe(0); + + registry.assignCategory('files', 'dest-local'); + expect(registry.currentConfigVersion()).toBe(1); + + registry.assignCategory('journey', 'dest-local'); + expect(registry.currentConfigVersion()).toBe(2); + + const row = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.config_version'").get() as + | { value: string } + | undefined; + expect(row?.value).toBe('2'); + }); + + it('REG-ASSIGN-004 an unknown-backend refusal bumps nothing — the version write shares the categories transaction', () => { + const { registry } = makeRegistry(); + expect(() => registry.assignCategory('files', 'ghost-backend')).toThrow(); + expect(registry.currentConfigVersion()).toBe(0); + }); + + it('REG-ASSIGN-005 refuses the migration flip onto a backend that is currently a mirror replica (validateConfig, other direction)', () => { + const { registry } = makeRegistry({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root: makeTmpDir() } }, + { name: 'backups-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas-backups'] } }, + ], + categories: { backups: 'backups-mirror' }, + }); + + expect(() => registry.assignCategory('files', 'nas-backups')).toThrow( + /cannot assign 'files' to 'nas-backups' — 'nas-backups' is a mirror replica of 'backups-mirror'/, + ); + expect(registry.snapshot().categories.files.backend).toBe('uploads-local'); // unchanged + expect(registry.currentConfigVersion()).toBe(0); // nothing written + }); + + it('REG-ASSIGN-006 refuses the flip onto a MIRROR whose primary is a replica of another mirror', () => { + const { registry } = makeRegistry({ + backends: [ + { name: 'nas', type: 'local', options: { root: makeTmpDir() } }, + { name: 'extra', type: 'local', options: { root: makeTmpDir() } }, + { name: 'nas-mirror', type: 'mirror', options: { primary: 'nas', replicas: ['extra'] } }, + { name: 'backups-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ], + categories: { backups: 'backups-mirror' }, + }); + + expect(() => registry.assignCategory('files', 'nas-mirror')).toThrow( + /cannot assign 'files' to 'nas-mirror' — 'nas' is a mirror replica of 'backups-mirror'/, + ); + expect(registry.currentConfigVersion()).toBe(0); + }); +}); + +// ── replica-failure health ──────────────────────────────────────────────────── + +describe('StorageRegistryService replica health', () => { + it('keeps a bounded ring of replica failures (last 50)', () => { + const { registry } = makeRegistry(); + for (let i = 0; i < 60; i++) { + registry.recordReplicaFailure({ + backend: 'nas-backups', + key: `backup-${i}.zip`, + op: 'put', + error: 'disk full', + at: i, + } satisfies ReplicaFailure); + } + const failures = registry.replicaFailures(); + expect(failures).toHaveLength(50); + expect(failures[0].key).toBe('backup-10.zip'); // oldest 10 dropped + expect(failures[49].key).toBe('backup-59.zip'); + }); +}); + +// ── settings-declared s3 backends (admin-config slice: acceptance) ─────────── + +describe('settings-declared s3 backends', () => { + const s3Options = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-plain', + region: 'us-east-1', + keyPrefix: '', + retries: 1, + timeoutMs: 30000, + }; + + it('accepts a settings s3 backend and assigns it to a category', () => { + const { registry } = makeRegistry({ + backends: [{ name: 'off-box', type: 's3', options: s3Options }], + categories: { covers: 'off-box' }, + }); + const resolved = registry.resolve('covers'); + expect(resolved.backendName).toBe('off-box'); + expect(resolved.driver).toBeInstanceOf(S3Driver); + expect(resolved.keyPrefix).toBe('covers/'); + }); + + it('decrypts an enc:v1: secretAccessKey at build (decrypt-at-build)', () => { + const { registry } = makeRegistry({ + backends: [{ name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: encrypt_api_key('sk-secret') } }], + categories: { backups: 'off-box' }, + }); + expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver); + }); + + it('applies the shared-schema defaults for omitted optional fields', () => { + const { endpoint, bucket, accessKeyId, secretAccessKey } = s3Options; + const { registry } = makeRegistry({ + backends: [{ name: 'off-box', type: 's3', options: { endpoint, bucket, accessKeyId, secretAccessKey } }], + categories: { backups: 'off-box' }, + }); + expect(registry.resolve('backups').backendName).toBe('off-box'); + }); + + it('serves as a mirror replica (local primary + settings s3 replica on backups)', () => { + const nasRoot = makeTmpDir(); + const { registry } = makeRegistry({ + backends: [ + { name: 'off-box', type: 's3', options: s3Options }, + { name: 'nas-backups', type: 'local', options: { root: nasRoot } }, + { name: 'backup-mirror', type: 'mirror', options: { primary: 'nas-backups', replicas: ['off-box'] } }, + ], + categories: { backups: 'backup-mirror' }, + }); + expect(registry.resolve('backups').backendName).toBe('backup-mirror'); + expect(registry.resolve('backups').driver).toBeInstanceOf(MirrorDriver); + }); + + it('falls back with the pinned message when s3 options fail the shared schema', () => { + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry({ + backends: [{ name: 'off-box', type: 's3', options: { ...s3Options, endpoint: 'not a url' } }], + }); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n'); + expect(logged).toContain("s3 backend 'off-box' has invalid options — endpoint:"); + }); + + it('falls back with the pinned message when the ciphertext cannot be decrypted', () => { + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry({ + backends: [{ name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: 'enc:v1:AAAA' } }], + categories: { backups: 'off-box' }, + }); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n'); + expect(logged).toContain("s3 backend 'off-box': could not decrypt 'secretAccessKey'"); + }); +}); + +// ── preview() — admin writes never take the silent fallback ────────────────── + +describe('preview()', () => { + it('throws the exact registry error for an invalid candidate and mutates nothing', () => { + const { registry } = makeRegistry(); + const before = registry.resolve('backups').driver; + expect(() => + registry.preview({ backends: [], categories: { backups: 'nope' } }), + ).toThrow("category 'backups' maps to unknown backend 'nope'"); + expect(registry.resolve('backups').driver).toBe(before); // same instance — state untouched + }); + + it('accepts a valid candidate without changing resolution', () => { + const nasRoot = makeTmpDir(); + const { registry } = makeRegistry(); + const before = registry.resolve('backups').driver; + registry.preview({ + backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }], + categories: { backups: 'nas-backups' }, + }); + expect(registry.resolve('backups').driver).toBe(before); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + }); +}); + +// ── shared replica backends (audit critical: the backfill deletion sweep) ───── +// +// A mirror's "Sync now" sweep DELETES replica keys under the swept category +// prefixes that the primary doesn't hold. `backups` sweeps prefix '' — the +// replica's ENTIRE root — so any backend that is both a replica and a category +// target would have that category's objects swept away by the first sync. The +// sweep itself is correct; the fix is that no config can express the setup. + +describe('StorageRegistryService shared-replica refusals', () => { + it('REG-SHARED-001 refuses the config that would let a backups sync sweep another category\'s objects (uploads-local as a backups-mirror replica)', () => { + const { registry } = makeRegistry(); + expect(() => + registry.preview({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root: makeTmpDir() } }, + { name: 'backups-mirror', type: 'mirror', options: { primary: 'nas-backups', replicas: ['uploads-local'] } }, + ], + categories: { backups: 'backups-mirror' }, + }), + ).toThrow(/backend 'uploads-local' is a mirror replica of 'backups-mirror' and also serves category/); + }); + + it('REG-SHARED-002 refuses a replica that serves a category as ANOTHER mirror\'s primary', () => { + const { registry } = makeRegistry(); + expect(() => + registry.preview({ + backends: [ + { name: 'nas', type: 'local', options: { root: makeTmpDir() } }, + { name: 'extra', type: 'local', options: { root: makeTmpDir() } }, + { name: 'files-mirror', type: 'mirror', options: { primary: 'nas', replicas: ['extra'] } }, + { name: 'backups-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas'] } }, + ], + categories: { files: 'files-mirror', backups: 'backups-mirror' }, + }), + ).toThrow(/backend 'nas' is a mirror replica of 'backups-mirror' and also serves category 'files'/); + }); + + it('REG-SHARED-003 refuses one backend replicating two mirrors whose swept prefixes overlap (\'\' overlaps everything)', () => { + const { registry } = makeRegistry(); + expect(() => + registry.preview({ + backends: [ + { name: 'nas', type: 'local', options: { root: makeTmpDir() } }, + { name: 'p1', type: 'local', options: { root: makeTmpDir() } }, + { name: 'p2', type: 'local', options: { root: makeTmpDir() } }, + { name: 'm-backups', type: 'mirror', options: { primary: 'p1', replicas: ['nas'] } }, + { name: 'm-files', type: 'mirror', options: { primary: 'p2', replicas: ['nas'] } }, + ], + categories: { backups: 'm-backups', files: 'm-files' }, + }), + ).toThrow(/backend 'nas' replicates both 'm-backups' and 'm-files', whose swept key prefixes overlap/); + }); + + it('REG-SHARED-004 accepts one backend replicating two mirrors with DISJOINT swept prefixes', () => { + const { registry } = makeRegistry(); + expect(() => + registry.preview({ + backends: [ + { name: 'nas', type: 'local', options: { root: makeTmpDir() } }, + { name: 'p1', type: 'local', options: { root: makeTmpDir() } }, + { name: 'p2', type: 'local', options: { root: makeTmpDir() } }, + { name: 'm-files', type: 'mirror', options: { primary: 'p1', replicas: ['nas'] } }, + { name: 'm-covers', type: 'mirror', options: { primary: 'p2', replicas: ['nas'] } }, + ], + categories: { files: 'm-files', covers: 'm-covers' }, + }), + ).not.toThrow(); + }); + + it('REG-SHARED-005 still accepts the ordinary shape: a dedicated, unassigned replica', () => { + const { registry } = makeRegistry(); + expect(() => + registry.preview({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root: makeTmpDir() } }, + { name: 'backups-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas-backups'] } }, + ], + categories: { backups: 'backups-mirror' }, + }), + ).not.toThrow(); + }); +}); + +// ── seed-once boot import ───────────────────────────────────────────────────── + +describe('seed-once storage-config.json import', () => { + const nasRoot = () => makeTmpDir(); + + function writeSeed(content: string): void { + fs.mkdirSync(path.dirname(SEED_CONFIG_PATH), { recursive: true }); + fs.writeFileSync(SEED_CONFIG_PATH, content); + } + + /** A registry with NO storage.* rows (makeRegistry seeds an override row, so build raw). */ + function makeUnseededRegistry() { + const stub = makeEnvStub({}); + return new StorageRegistryService(db, stub.env, new StorageEventsService()); + } + + function readRow(key: string): string | undefined { + const row = testDb.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as + | { value: string } + | undefined; + return row?.value; + } + + afterEach(() => { + fs.rmSync(SEED_CONFIG_PATH, { force: true }); + }); + + it('SEED-001 imports a valid file when no rows exist: encrypted rows, live drivers, pinned log', () => { + const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined); + const root = nasRoot(); + writeSeed( + JSON.stringify({ + backends: [ + { name: 'nas-backups', type: 'local', options: { root } }, + { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-seed', + }, + }, + ], + categories: { backups: 'off-box' }, + }), + ); + const registry = makeUnseededRegistry(); + registry.onModuleInit(); + + expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver); + const storedBackends = JSON.parse(readRow('storage.backends')!) as Array<{ + name: string; + options: Record; + }>; + const offBox = storedBackends.find((b) => b.name === 'off-box')!; + expect(String(offBox.options.secretAccessKey).startsWith('enc:v1:')).toBe(true); // never plaintext at rest + expect(readRow('storage.categories')).toBe(JSON.stringify({ backups: 'off-box' })); + const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(logged).toContain('storage config seeded from'); + expect(logged).toContain('the file is now ignored; manage storage in the admin UI'); + }); + + it('SEED-002 rows exist → file not read, pinned ignore log, rows unchanged', () => { + const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined); + writeSeed('{ this would explode if parsed'); + const { registry } = makeRegistry(); // helper writes the uploads override row + expect(registry.resolve('files').backendName).toBe('uploads-local'); + const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(logged).toContain('storage config rows exist — ignoring'); + }); + + it('SEED-003 no rows, no file → boots on built-in defaults (no seed side effects)', () => { + const registry = makeUnseededRegistry(); + registry.onModuleInit(); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + expect(readRow('storage.backends')).toBeUndefined(); + }); + + it('SEED-004 unparseable JSON aborts boot with the exact error', () => { + writeSeed('not json at all'); + const registry = makeUnseededRegistry(); + expect(() => registry.onModuleInit()).toThrow(/invalid storage seed file .*storage-config\.json: /); + }); + + it('SEED-005 schema violation aborts boot — including the reserved readOnly key', () => { + writeSeed(JSON.stringify({ backends: [], categories: {}, readOnly: true })); + expect(() => makeUnseededRegistry().onModuleInit()).toThrow(/invalid storage seed file/); + }); + + it('SEED-006 semantic violation aborts boot with the registry error verbatim', () => { + writeSeed(JSON.stringify({ backends: [], categories: { backups: 'nope' } })); + expect(() => makeUnseededRegistry().onModuleInit()).toThrow( + /category 'backups' maps to unknown backend 'nope'/, + ); + }); + + it('SEED-007 a plaintext-secret seed imports without an explicit ENCRYPTION_KEY and encrypts at rest', () => { + writeSeed( + JSON.stringify({ + backends: [ + { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-seed', + }, + }, + ], + categories: {}, + }), + ); + const registry = makeUnseededRegistry(); + registry.onModuleInit(); + // No key-presence gate: the implicit key covers encryption when + // ENCRYPTION_KEY is unset, and the plaintext never persists. + const row = readRow('storage.backends')!; + expect(row).not.toContain('sk-seed'); + expect(row).toContain('enc:v1:'); + }); + + it('SEED-008 a mask sentinel in the seed file aborts boot', () => { + writeSeed( + JSON.stringify({ + backends: [ + { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: '••••••••', + }, + }, + ], + categories: {}, + }), + ); + expect(() => makeUnseededRegistry().onModuleInit()).toThrow(/mask/); + }); + + it('SEED-009 a second boot after a successful seed ignores the file (seed-once)', () => { + const root = nasRoot(); + writeSeed(JSON.stringify({ backends: [{ name: 'nas', type: 'local', options: { root } }], categories: { backups: 'nas' } })); + makeUnseededRegistry().onModuleInit(); + const firstRow = readRow('storage.backends'); + + writeSeed(JSON.stringify({ backends: [], categories: { backups: 'other' } })); // would fail preview if read + const second = makeUnseededRegistry(); + second.onModuleInit(); // must not throw — rows exist, file ignored + expect(readRow('storage.backends')).toBe(firstRow); + expect(second.resolve('backups').backendName).toBe('nas'); + }); + + it('SEED-010 an all-encrypted seed imports and keeps its ciphertext byte-for-byte', () => { + const cipher = encrypt_api_key('sk-seed'); + writeSeed( + JSON.stringify({ + backends: [ + { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: cipher, + }, + }, + ], + categories: { backups: 'off-box' }, + }), + ); + const registry = makeUnseededRegistry(); + registry.onModuleInit(); + // enc:v1: values pass through the idempotent encrypt — the ciphertext + // must persist byte-for-byte. + expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver); + expect(readRow('storage.backends')).toContain(cipher); + }); +}); + +// ── snapshot() — effective world with provenance ────────────────────────────── + +describe('snapshot()', () => { + it('reports built-ins, env backends, and settings entries with their sources', () => { + const photoDir = makeTmpDir(); + const nasRoot = makeTmpDir(); + const { registry, uploadsRoot } = makeRegistry({ + placePhotoDir: photoDir, + backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }], + categories: { backups: 'nas-backups' }, + }); + const snap = registry.snapshot(); + const byName = new Map(snap.backends.map((b) => [b.name, b])); + + // uploads-local is overridden by the helper's settings row → source 'settings' + expect(byName.get('uploads-local')).toMatchObject({ type: 'local', source: 'settings', options: { root: uploadsRoot } }); + expect(byName.get('backups-local')).toMatchObject({ source: 'built-in' }); + expect(byName.get('place-photos-local')).toMatchObject({ source: 'env', options: { root: photoDir } }); + expect(byName.get('nas-backups')).toMatchObject({ source: 'settings' }); + + expect(snap.categories.backups).toEqual({ backend: 'nas-backups', source: 'settings' }); + expect(snap.categories.files).toEqual({ backend: 'uploads-local', source: 'default' }); + expect(snap.categories['photos-google']).toEqual({ backend: 'place-photos-local', source: 'default' }); + expect(Object.keys(snap.categories).sort()).toEqual([...STORAGE_CATEGORIES].sort()); + }); + + it('reports pure built-in defaults when no settings rows exist', () => { + const { registry } = makeRegistry({ boot: false }); + testDb.prepare("DELETE FROM app_settings WHERE key = 'storage.backends'").run(); + registry.onModuleInit(); + const snap = registry.snapshot(); + expect(snap.backends.map((b) => [b.name, b.source])).toEqual([ + ['uploads-local', 'built-in'], + ['backups-local', 'built-in'], + ]); + }); + + it('keeps encrypted secrets encrypted in the snapshot (never plaintext)', () => { + const cipher = encrypt_api_key('sk-secret'); + const { registry } = makeRegistry({ + backends: [ + { + name: 'off-box', + type: 's3', + options: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: cipher, + }, + }, + ], + categories: { backups: 'off-box' }, + }); + const offBox = registry.snapshot().backends.find((b) => b.name === 'off-box')!; + expect(offBox.options.secretAccessKey).toBe(cipher); // byte-identical ciphertext + expect(offBox.options).toMatchObject({ region: 'us-east-1', retries: 1 }); // schema defaults visible + }); + + it('shows the last-good world after an invalid reload (snapshot = live state)', () => { + vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const { registry } = makeRegistry(); + const before = registry.snapshot(); + setSetting('storage.categories', 'garbage {'); + registry.reload(); + expect(registry.snapshot()).toEqual(before); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-secrets.test.ts b/server/tests/unit/nest/storage/storage-secrets.test.ts new file mode 100644 index 000000000..044509a0e --- /dev/null +++ b/server/tests/unit/nest/storage/storage-secrets.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, vi } from 'vitest'; + +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-secrets-test-key' })); + +import { MASKED_SETTING_VALUE, type StorageConfig } from '@trek/shared'; +import { + decrypt_api_key, + encrypt_api_key, +} from '../../../../src/nest/common/crypto/apiKeyCrypto'; +import { + assertNoMaskSentinels, + decryptBackendSecrets, + encryptStorageSecrets, + maskBackendOptions, + redactStorageSecrets, + unmaskStorageConfig, +} from '../../../../src/nest/storage/storage-secrets'; + +const S3_OPTIONS = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-plain', + region: 'us-east-1', + keyPrefix: '', + retries: 1, + timeoutMs: 30000, +}; + +function s3Config(secretAccessKey: string): StorageConfig { + return { + backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey } }], + categories: { backups: 'off-box' }, + }; +} +const LOCAL_ONLY: StorageConfig = { + backends: [{ name: 'nas', type: 'local', options: { root: '/mnt/nas' } }], + categories: {}, +}; + + +describe('encryptStorageSecrets', () => { + it('encrypts plaintext secrets and round-trips through decrypt', () => { + const out = encryptStorageSecrets(s3Config('sk-plain')); + const stored = (out.backends[0]!.options as Record).secretAccessKey as string; + expect(stored.startsWith('enc:v1:')).toBe(true); + expect(decrypt_api_key(stored)).toBe('sk-plain'); + }); + + it('is idempotent — an enc:v1: value passes through byte-for-byte', () => { + const cipher = encrypt_api_key('sk'); + const out = encryptStorageSecrets(s3Config(cipher)); + expect((out.backends[0]!.options as Record).secretAccessKey).toBe(cipher); + }); + + it('does not mutate its input', () => { + const input = s3Config('sk-plain'); + void encryptStorageSecrets(input); + expect((input.backends[0]!.options as Record).secretAccessKey).toBe('sk-plain'); + }); +}); + +describe('unmaskStorageConfig', () => { + it('replaces a mask echo with the stored ciphertext byte-for-byte', () => { + const cipher = encrypt_api_key('sk'); + const stored = [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: cipher } }]; + const out = unmaskStorageConfig(s3Config(MASKED_SETTING_VALUE), stored); + expect((out.backends[0]!.options as Record).secretAccessKey).toBe(cipher); + }); + + it('throws for a mask with no stored counterpart (rename or new backend)', () => { + expect(() => unmaskStorageConfig(s3Config(MASKED_SETTING_VALUE), [])).toThrow( + "re-enter the secret 'secretAccessKey' for 'off-box'", + ); + const renamed = [{ name: 'old-name', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: encrypt_api_key('sk') } }]; + expect(() => unmaskStorageConfig(s3Config(MASKED_SETTING_VALUE), renamed)).toThrow( + "re-enter the secret 'secretAccessKey' for 'off-box'", + ); + }); + + it('leaves non-mask values and secretless backends untouched, and tolerates a garbage stored row', () => { + expect(unmaskStorageConfig(s3Config('sk-new'), 'not-an-array')).toEqual(s3Config('sk-new')); + expect(unmaskStorageConfig(LOCAL_ONLY, undefined)).toEqual(LOCAL_ONLY); + }); +}); + +describe('assertNoMaskSentinels', () => { + it('throws when a seed-style config carries the mask sentinel in a secret field', () => { + expect(() => assertNoMaskSentinels(s3Config(MASKED_SETTING_VALUE))).toThrow('mask'); + expect(() => assertNoMaskSentinels(s3Config('sk'))).not.toThrow(); + }); + + it('throws for a mask sentinel in a NON-secret field too — the check is all-field, not secret-only', () => { + const config: StorageConfig = { + backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, accessKeyId: MASKED_SETTING_VALUE } }], + categories: {}, + }; + expect(() => assertNoMaskSentinels(config)).toThrow( + "backend 'off-box' field 'accessKeyId' is the mask sentinel — a mask can never become a stored value", + ); + }); +}); + +describe('decryptBackendSecrets', () => { + it('decrypts enc:v1: secrets, passes plaintext through, throws on tampered ciphertext', () => { + const encrypted = s3Config(encrypt_api_key('sk-secret')).backends[0]!; + expect((decryptBackendSecrets(encrypted).options as Record).secretAccessKey).toBe('sk-secret'); + const plain = s3Config('sk-plain').backends[0]!; + expect((decryptBackendSecrets(plain).options as Record).secretAccessKey).toBe('sk-plain'); + expect(() => decryptBackendSecrets(s3Config('enc:v1:AAAA').backends[0]!)).toThrow( + "could not decrypt 'secretAccessKey'", + ); + }); +}); + +describe('maskBackendOptions / redactStorageSecrets', () => { + it('masks exactly the secret fields on GET-shaped options', () => { + const masked = maskBackendOptions('s3', { ...S3_OPTIONS }); + expect(masked.secretAccessKey).toBe(MASKED_SETTING_VALUE); + expect(masked.accessKeyId).toBe('ak'); // never over-masked + expect(maskBackendOptions('local', { root: '/x' })).toEqual({ root: '/x' }); + }); + + it('redacts secrets to *** for audit details', () => { + const out = redactStorageSecrets(s3Config('sk-plain')); + expect((out.backends[0]!.options as Record).secretAccessKey).toBe('***'); + expect((out.backends[0]!.options as Record).accessKeyId).toBe('ak'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-stats.service.test.ts b/server/tests/unit/nest/storage/storage-stats.service.test.ts new file mode 100644 index 000000000..a79eb3047 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-stats.service.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest'; + +const { testDb, dbMock } = vi.hoisted(() => { + const Database = require('better-sqlite3'); + const db = new Database(':memory:'); + db.exec('PRAGMA journal_mode = WAL'); + db.exec('PRAGMA foreign_keys = ON'); + return { testDb: db, dbMock: { db, closeDb: () => {}, reinitialize: () => {} } }; +}); +vi.mock('../../../../src/db/database', () => dbMock); +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-stats-test-key' })); + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { createTables } from '../../../../src/db/schema'; +import { runMigrations } from '../../../../src/db/migrations'; +import { DatabaseService } from '../../../../src/nest/database/database.service'; +import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; +import { StorageRegistryService } from '../../../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import { StatsBusyError, StorageStatsService } from '../../../../src/nest/storage/storage-stats.service'; + +const db = new DatabaseService(testDb); + +beforeAll(() => { + createTables(testDb); + runMigrations(testDb); +}); + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-stats-')); + tmpDirs.push(dir); + return dir; +} +function setSetting(key: string, value: string): void { + testDb.prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)').run(key, value); +} +beforeEach(() => { + testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run(); +}); +afterEach(() => { + vi.restoreAllMocks(); + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +function makeWorld() { + const uploadsRoot = makeTmpDir(); + const backupsRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + ]), + ); + const env = { env: () => ({ paths: {} }) } as unknown as RuntimeEnvService; + const registry = new StorageRegistryService(db, env, new StorageEventsService()); + registry.onModuleInit(); + const storage = new StorageService(registry); + const stats = new StorageStatsService(storage, db); + return { storage, stats, uploadsRoot }; +} + +describe('StorageStatsService', () => { + it('STATS-001 sums objects and bytes per category, legacy photos separate, and persists with computedAt', async () => { + const { storage, stats } = makeWorld(); + await storage.put('files', 'a.pdf', Readable.from('12345')); // 5 bytes + await storage.put('files', 'b.pdf', Readable.from('123')); // 3 bytes + await storage.put('covers', 'c.jpg', Readable.from('1234567')); // 7 bytes + await storage.put('photos', 'legacy.jpg', Readable.from('12')); // legacy dir + const usage = await stats.scan(); + expect(usage.categories.files).toEqual({ objects: 2, bytes: 8 }); + expect(usage.categories.covers).toEqual({ objects: 1, bytes: 7 }); + expect(usage.categories.backups).toEqual({ objects: 0, bytes: 0 }); + expect(usage.legacyPhotos).toEqual({ objects: 1, bytes: 2 }); + expect(usage.computedAt).toBeGreaterThan(0); + // Persisted round-trip: + expect(stats.readUsage()).toEqual(usage); + const raw = testDb.prepare("SELECT value FROM app_settings WHERE key = 'storage.usage'").get() as { value: string }; + expect(JSON.parse(raw.value)).toEqual(usage); + }); + + it('STATS-002 photos-google/photos-trek nested content is NOT double-counted into legacy photos', async () => { + const { storage, stats } = makeWorld(); + await storage.put('photos-google', 'g.jpg', Readable.from('gggg')); + const usage = await stats.scan(); + expect(usage.categories['photos-google']).toEqual({ objects: 1, bytes: 4 }); + expect(usage.legacyPhotos).toEqual({ objects: 0, bytes: 0 }); + }); + + it('STATS-003 concurrent scans throw StatsBusyError', async () => { + const { storage, stats } = makeWorld(); + for (let i = 0; i < 30; i++) await storage.put('files', `f${i}.bin`, Readable.from('x'.repeat(500))); + const first = stats.scan(); + await expect(stats.scan()).rejects.toThrow(StatsBusyError); + await first; + }); + + it('STATS-004 readUsage returns null on absent or unparseable rows', () => { + const { stats } = makeWorld(); + expect(stats.readUsage()).toBeNull(); + setSetting('storage.usage', 'not json'); + expect(stats.readUsage()).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-upload.factory.test.ts b/server/tests/unit/nest/storage/storage-upload.factory.test.ts new file mode 100644 index 000000000..df08c84eb --- /dev/null +++ b/server/tests/unit/nest/storage/storage-upload.factory.test.ts @@ -0,0 +1,162 @@ +/** + * Unit tests for buildStorageUploadOptions — the multer options factory every + * upload call site consumes via MulterModule.registerAsync. + * + * STORAGE-UPLOAD-001 … 010. + * + * The diskStorage engine exposes its destination/filename callbacks as + * getDestination/getFilename on the engine instance (multer/storage/disk.js), + * so the tests drive them directly instead of streaming a real request. + */ +import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import type { Request } from 'express'; +import { buildStorageUploadOptions } from '../../../../src/nest/storage/storage-upload.factory'; +import type { StorageService } from '../../../../src/nest/storage/storage.service'; +import type { StorageCategory } from '../../../../src/nest/storage/storage.types'; + +interface DiskEngine { + getDestination: (req: Request, file: Express.Multer.File, cb: (err: Error | null, dir: string) => void) => void; + getFilename: (req: Request, file: Express.Multer.File, cb: (err: Error | null, name: string) => void) => void; +} + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +function fakeFile(overrides: Partial = {}): Express.Multer.File { + return { fieldname: 'file', originalname: 'doc.pdf', ...overrides } as Express.Multer.File; +} + +const req = {} as Request; + +function destination(engine: DiskEngine, file: Express.Multer.File): Promise { + return new Promise((resolve, reject) => + engine.getDestination(req, file, (err, dir) => (err ? reject(err) : resolve(dir))), + ); +} + +function filename(engine: DiskEngine, file: Express.Multer.File): Promise { + return new Promise((resolve, reject) => + engine.getFilename(req, file, (err, name) => (err ? reject(err) : resolve(name))), + ); +} + +describe('buildStorageUploadOptions', () => { + let dirA: string; + let dirB: string; + + beforeAll(() => { + dirA = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-upload-factory-a-')); + dirB = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-upload-factory-b-')); + }); + + afterAll(() => { + fs.rmSync(dirA, { recursive: true, force: true }); + fs.rmSync(dirB, { recursive: true, force: true }); + }); + + it('STORAGE-UPLOAD-001 — destination resolves spoolDirFor per request, never cached', async () => { + const spoolDirFor = vi.fn(() => dirA); + const storage = { spoolDirFor } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + expect(await destination(engine, fakeFile())).toBe(dirA); + spoolDirFor.mockReturnValue(dirB); + expect(await destination(engine, fakeFile())).toBe(dirB); + expect(spoolDirFor).toHaveBeenCalledTimes(2); + expect(spoolDirFor).toHaveBeenCalledWith('files'); + }); + + it('STORAGE-UPLOAD-002 — destination creates the spool dir if missing', async () => { + const missing = path.join(dirA, 'not-yet-created'); + const storage = { spoolDirFor: () => missing } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + expect(await destination(engine, fakeFile())).toBe(missing); + expect(fs.existsSync(missing)).toBe(true); + }); + + it('STORAGE-UPLOAD-003 — category resolver receives (req, file) and routes per file', async () => { + const spoolDirFor = vi.fn((cat: StorageCategory) => (cat === 'places' ? dirA : dirB)); + const storage = { spoolDirFor } as unknown as StorageService; + const category = vi.fn((_r: Request, file: Express.Multer.File): StorageCategory => + file.fieldname === 'image' ? 'places' : 'covers', + ); + const opts = buildStorageUploadOptions(storage, { category, maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + expect(await destination(engine, fakeFile({ fieldname: 'image' }))).toBe(dirA); + expect(await destination(engine, fakeFile({ fieldname: 'cover' }))).toBe(dirB); + expect(spoolDirFor).toHaveBeenNthCalledWith(1, 'places'); + expect(spoolDirFor).toHaveBeenNthCalledWith(2, 'covers'); + expect(category).toHaveBeenCalledWith(req, expect.objectContaining({ fieldname: 'cover' })); + }); + + it('STORAGE-UPLOAD-004 — destination surfaces spoolDirFor errors through the callback', async () => { + const boom = new Error('registry not initialized'); + const storage = { spoolDirFor: () => { throw boom; } } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + await expect(destination(engine, fakeFile())).rejects.toBe(boom); + }); + + it('STORAGE-UPLOAD-005 — default filename is a v4 uuid with the case-preserved extension', async () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + const upper = await filename(engine, fakeFile({ originalname: 'Photo.JPG' })); + expect(upper).toMatch(/\.JPG$/); + expect(upper.slice(0, -4)).toMatch(UUID_RE); + }); + + it('STORAGE-UPLOAD-006 — default filename has no fallback extension for extensionless names', async () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + const engine = opts.storage as unknown as DiskEngine; + + expect(await filename(engine, fakeFile({ originalname: 'noext' }))).toMatch(UUID_RE); + }); + + it('STORAGE-UPLOAD-007 — a custom filename hook is used verbatim', async () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { + category: 'journey', + maxSize: 10, + filename: (_r, file) => (file.fieldname === 'poster' ? 'forced.jpg' : 'other.bin'), + }); + const engine = opts.storage as unknown as DiskEngine; + + expect(await filename(engine, fakeFile({ fieldname: 'poster', originalname: 'x.html' }))).toBe('forced.jpg'); + expect(await filename(engine, fakeFile({ fieldname: 'video' }))).toBe('other.bin'); + }); + + it('STORAGE-UPLOAD-008 — maxSize becomes limits.fileSize', () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const opts = buildStorageUploadOptions(storage, { category: 'files', maxSize: 123456 }); + expect(opts.limits).toEqual({ fileSize: 123456 }); + }); + + it('STORAGE-UPLOAD-009 — fileFilter passes through untouched, and is absent when not given', () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const filter = vi.fn(); + const withFilter = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10, fileFilter: filter }); + expect(withFilter.fileFilter).toBe(filter); + + const without = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + expect('fileFilter' in without).toBe(false); + }); + + it('STORAGE-UPLOAD-010 — defParamCharset defaults to utf8 everywhere (fix #3)', () => { + const storage = { spoolDirFor: () => dirA } as unknown as StorageService; + const without = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10 }); + expect((without as { defParamCharset?: string }).defParamCharset).toBe('utf8'); + + const withCharset = buildStorageUploadOptions(storage, { category: 'files', maxSize: 10, defParamCharset: 'utf8' }); + expect((withCharset as { defParamCharset?: string }).defParamCharset).toBe('utf8'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts b/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts new file mode 100644 index 000000000..78df97c3f --- /dev/null +++ b/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts @@ -0,0 +1,67 @@ +/** + * StorageUsageScanJob — the nightly 04:15 cron that drives + * StorageStatsService.scan(). Registration gating follows the + * CronRegistrarService idiom pinned in tests/unit/nest/admin-jobs.test.ts; + * the tick error-containment mirrors VersionCheckJob/DemoResetJob there. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { Logger } from '@nestjs/common'; +import { StorageUsageScanJob } from '../../../../src/nest/storage/storage-usage-scan.job'; +import type { StorageStatsService } from '../../../../src/nest/storage/storage-stats.service'; +import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service'; + +function registrarStub(enabled = true) { + return { + isEnabled: vi.fn(() => enabled), + register: vi.fn((_name: string, _expression: string, _onTick: () => void | Promise, _opts?: { timezone?: 'app' | 'none' }) => enabled), + unregister: vi.fn(), + }; +} + +function statsStub(): StorageStatsService { + return { scan: vi.fn().mockResolvedValue(undefined) } as unknown as StorageStatsService; +} + +beforeEach(() => vi.restoreAllMocks()); + +describe('StorageUsageScanJob', () => { + it('SCAN-001 registers the nightly 04:15 cron when the registrar gate is open', () => { + const registrar = registrarStub(); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub()); + job.onApplicationBootstrap(); + expect(registrar.register).toHaveBeenCalledWith('storage-usage-scan', '15 4 * * *', expect.any(Function)); + }); + + it('SCAN-002 does not register under the test gate (registrar.isEnabled() false)', () => { + const registrar = registrarStub(false); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub()); + job.onApplicationBootstrap(); + expect(registrar.register).not.toHaveBeenCalled(); + }); + + it('SCAN-003 the registered callback drives tick(), which calls stats.scan()', async () => { + const registrar = registrarStub(); + const stats = statsStub(); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, stats); + job.onApplicationBootstrap(); + const onTick = registrar.register.mock.calls[0]![2] as () => void; + onTick(); + await vi.waitFor(() => expect(stats.scan).toHaveBeenCalledTimes(1)); + }); + + it('SCAN-004 tick() resolves and contains an Error rejection to the usage scan log line', async () => { + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const stats = { scan: vi.fn().mockRejectedValue(new Error('disk full')) } as unknown as StorageStatsService; + const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats); + await expect(job.tick()).resolves.toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith('usage scan failed: disk full'); + }); + + it('SCAN-005 tick() stringifies a non-Error rejection rather than crashing', async () => { + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const stats = { scan: vi.fn().mockRejectedValue('backend unreachable') } as unknown as StorageStatsService; + const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats); + await expect(job.tick()).resolves.toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith('usage scan failed: backend unreachable'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage.service.test.ts b/server/tests/unit/nest/storage/storage.service.test.ts new file mode 100644 index 000000000..5b56eb1f6 --- /dev/null +++ b/server/tests/unit/nest/storage/storage.service.test.ts @@ -0,0 +1,946 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { EventEmitter } from 'node:events'; +import { PassThrough, Readable } from 'node:stream'; +import type { Response } from 'express'; +import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; +import type { ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver'; +import type { StorageRegistryService, ResolvedCategory } from '../../../../src/nest/storage/storage-registry.service'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import { + StorageInvalidKeyError, + StorageNotFoundError, + type ByteRange, + type ObjectStat, + type StorageDriver, +} from '../../../../src/nest/storage/storage.types'; + +// Facade over a stub registry (the weather.controller.test.ts casting style) +// backed by a REAL LocalDriver, so key composition, prefix stripping, and both +// sendToResponse/withLocalFile branches are exercised against actual files. + +const tmpDirs: string[] = []; +function makeTmpDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-facade-')); + tmpDirs.push(dir); + return dir; +} + +interface Fixture { + storage: StorageService; + driver: LocalDriver; + root: string; + tempDir: string; + failures: ReplicaFailure[]; +} + +function makeFixture(keyPrefix = 'files/'): Fixture { + const root = makeTmpDir(); + const tempDir = makeTmpDir(); + const driver = new LocalDriver({ id: 'stub-local', root }); + driver.init({ cleanSpool: true }); + const failures: ReplicaFailure[] = []; + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix, backendName: 'stub-local' }), + tempDir: () => tempDir, + replicaFailures: () => failures, + } as unknown as StorageRegistryService; + return { storage: new StorageService(registry), driver, root, tempDir, failures }; +} + +/** A driver with no getLocalPath — the remote-driver branch every helper must handle. */ +function makeStreamOnlyFixture(contents: string): Fixture & { driverCalls: string[] } { + const fx = makeFixture(''); + const driverCalls: string[] = []; + const bytes = Buffer.from(contents); + const stat: ObjectStat = { key: 'remote.bin', size: bytes.length, mtimeMs: 1 }; + const streamOnly: StorageDriver = { + id: 'stub-remote', + put: async () => undefined, + getStream: async (key: string) => { + driverCalls.push(`getStream:${key}`); + return { stream: Readable.from(bytes), stat: { ...stat, key } }; + }, + stat: async (key: string) => ({ ...stat, key }), + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + // no getLocalPath, no getSpoolDir + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver: streamOnly, keyPrefix: '', backendName: 'stub-remote' }), + tempDir: () => fx.tempDir, + replicaFailures: () => fx.failures, + } as unknown as StorageRegistryService; + return { ...fx, storage: new StorageService(registry), driverCalls }; +} + +/** + * A driver whose `getLocalPath` resolves to a path that is NOT on disk, but + * whose `getStream` still succeeds — the façade-level stand-in for a + * MirrorDriver whose primary lost the file (failed delete, disk hiccup, …) + * while a replica still holds it. `getLocalPath` on MirrorDriver only ever + * consults the primary (mirror.driver.ts), so the façade must never trust + * that path as proof of presence — it has to fs-check it and fall through to + * the stream branch (which is what actually reaches the replica) when it's + * gone. + */ +function makeGhostLocalPathFixture(contents: string): Fixture & { driverCalls: string[] } { + const fx = makeFixture('files/'); + const driverCalls: string[] = []; + const bytes = Buffer.from(contents); + const stat: ObjectStat = { key: 'files/ghost.bin', size: bytes.length, mtimeMs: 1 }; + const ghostPath = path.join(fx.root, 'files', 'ghost.bin'); // resolvable, but never written + const driver: StorageDriver = { + id: 'stub-ghost-local', + put: async () => undefined, + getStream: async (key: string) => { + driverCalls.push(`getStream:${key}`); + return { stream: Readable.from(bytes), stat: { ...stat, key } }; + }, + stat: async (key: string) => ({ ...stat, key }), // the composite still resolves the object + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + getLocalPath: () => ghostPath, + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix: 'files/', backendName: 'stub-ghost-local' }), + tempDir: () => fx.tempDir, + replicaFailures: () => fx.failures, + } as unknown as StorageRegistryService; + return { ...fx, storage: new StorageService(registry), driverCalls }; +} + +afterEach(() => { + while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true }); +}); + +describe('StorageService key composition', () => { + it('composes keyPrefix + name on writes and strips it back in list()', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'a.pdf', Readable.from('doc')); + await fx.storage.put('files', 'nested/b.pdf', Readable.from('doc2')); + + // on disk under the driver prefix + expect(fs.existsSync(path.join(fx.root, 'files/a.pdf'))).toBe(true); + + const names: string[] = []; + for await (const stat of fx.storage.list('files')) names.push(stat.key); + expect(names.sort()).toEqual(['a.pdf', 'nested/b.pdf']); // category-relative + + const nested: string[] = []; + for await (const stat of fx.storage.list('files', 'nested/')) nested.push(stat.key); + expect(nested).toEqual(['nested/b.pdf']); + }); + + it('rejects a name that composes into an invalid key', async () => { + const fx = makeFixture('files/'); + await expect(fx.storage.put('files', '../escape.bin', Readable.from('x'))).rejects.toBeInstanceOf( + StorageInvalidKeyError, + ); + await expect(fx.storage.stat('files', '.tmp/spool.part')).rejects.toBeInstanceOf(StorageInvalidKeyError); + }); + + it('delegates stat/exists/delete/getStream with the composed key', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'c.bin', Readable.from('12345')); + + expect((await fx.storage.stat('files', 'c.bin'))!.size).toBe(5); + expect(await fx.storage.exists('files', 'c.bin')).toBe(true); + expect(await fx.storage.exists('files', 'nope.bin')).toBe(false); + + const { stream, stat } = await fx.storage.getStream('files', 'c.bin', { start: 1, end: 3 }); + expect(stat.size).toBe(5); + const chunks: Buffer[] = []; + for await (const c of stream) chunks.push(Buffer.from(c as Buffer)); + expect(Buffer.concat(chunks).toString()).toBe('234'); + + await fx.storage.delete('files', 'c.bin'); + expect(await fx.storage.exists('files', 'c.bin')).toBe(false); + }); +}); + +describe('StorageService spool + temp dirs', () => { + it('spoolDirFor returns the backend spool for local drivers and tempDir() otherwise', () => { + const fx = makeFixture(); + expect(fx.storage.spoolDirFor('files')).toBe(path.join(fs.realpathSync(fx.root), '.tmp')); + expect(fx.storage.tempDir()).toBe(fx.tempDir); + + const remote = makeStreamOnlyFixture('x'); + expect(remote.storage.spoolDirFor('files')).toBe(remote.tempDir); + }); +}); + +describe('StorageService withLocalFile', () => { + it('hands local drivers their real path', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'img.jpg', Readable.from('jpeg bytes')); + + const result = await fx.storage.withLocalFile('files', 'img.jpg', async (absPath) => { + expect(absPath).toBe(path.join(fs.realpathSync(fx.root), 'files/img.jpg')); + return fs.readFileSync(absPath, 'utf8'); + }); + expect(result).toBe('jpeg bytes'); + }); + + it('throws StorageNotFoundError for a local miss', async () => { + const fx = makeFixture('files/'); + await expect(fx.storage.withLocalFile('files', 'ghost.jpg', async () => 'never')).rejects.toBeInstanceOf( + StorageNotFoundError, + ); + }); + + it('downloads to tempDir for path-less drivers and cleans up, even when fn throws', async () => { + const remote = makeStreamOnlyFixture('remote bytes'); + + let seenPath = ''; + const result = await remote.storage.withLocalFile('files', 'remote.bin', async (absPath) => { + seenPath = absPath; + return fs.readFileSync(absPath, 'utf8'); + }); + expect(result).toBe('remote bytes'); + expect(seenPath.startsWith(remote.tempDir + path.sep)).toBe(true); + expect(fs.existsSync(seenPath)).toBe(false); // cleaned up + + await expect( + remote.storage.withLocalFile('files', 'remote.bin', async (absPath) => { + seenPath = absPath; + throw new Error('processing failed'); + }), + ).rejects.toThrow('processing failed'); + expect(fs.existsSync(seenPath)).toBe(false); // cleaned up on throw too + }); + + it('falls through to the stream branch when getLocalPath resolves but the file is not actually on disk', async () => { + // Composite-driver stand-in (task C9-i): the façade must fs-check the + // local path itself rather than trust driver.stat() as "the file is at + // localPath" — driver.stat here still says present (a replica has it). + const fx = makeGhostLocalPathFixture('replica bytes'); + + const result = await fx.storage.withLocalFile('files', 'ghost.bin', async (absPath) => + fs.readFileSync(absPath, 'utf8'), + ); + + expect(result).toBe('replica bytes'); + expect(fx.driverCalls).toEqual(['getStream:files/ghost.bin']); // reached the stream branch + }); +}); + +describe('StorageService getLocalPathOrNull', () => { + it('returns the real path for a local driver whose object exists on disk', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'img.jpg', Readable.from('jpeg bytes')); + + const localPath = await fx.storage.getLocalPathOrNull('files', 'img.jpg'); + expect(localPath).toBe(path.join(fs.realpathSync(fx.root), 'files/img.jpg')); + }); + + it('returns null for a path-less (remote) driver', async () => { + const remote = makeStreamOnlyFixture('remote bytes'); + expect(await remote.storage.getLocalPathOrNull('files', 'remote.bin')).toBeNull(); + // The probe never falls back to streaming itself — it's a pure locality check. + expect(remote.driverCalls).toEqual([]); + }); + + it('returns null for a local miss (no fs.existsSync throw)', async () => { + const fx = makeFixture('files/'); + expect(await fx.storage.getLocalPathOrNull('files', 'ghost.jpg')).toBeNull(); + }); + + it('fail-safe: returns null when the local path exists in the driver API but the file has vanished from disk', async () => { + // Binding controller ruling: "local path available" means the path exists + // on disk. Simulates a file deleted between storage.list() and this call + // (or by another process) — the caller must fall back to streaming rather + // than push a path that will 404/throw when read. + const fx = makeFixture('files/'); + await fx.storage.put('files', 'raced.jpg', Readable.from('bytes')); + fs.rmSync(path.join(fs.realpathSync(fx.root), 'files/raced.jpg')); + + expect(await fx.storage.getLocalPathOrNull('files', 'raced.jpg')).toBeNull(); + }); + + it('rejects a name that composes into an invalid key', async () => { + const fx = makeFixture('files/'); + await expect(fx.storage.getLocalPathOrNull('files', '../escape.bin')).rejects.toBeInstanceOf( + StorageInvalidKeyError, + ); + }); +}); + +interface MockRes { + res: Response & { headersSent: boolean; statusCode: number }; + sendFileCalls: Array<{ name: string; root: string }>; + headers: Record; + body: () => Promise; +} + +/** What the serving path reads off `res.req` — express always populates it. */ +interface ReqInit { + method?: string; + headers?: Record; +} + +// Real headersSent semantics (mirroring http.ServerResponse): flips true on +// the FIRST write/end, before that point it's false — never a static value. +// This matters here specifically: the remote branch's abort-swallow is +// gated on `res.headersSent`, so a mock that fakes it as an inert boolean +// would hide a regression where the gate stops actually gating (see CRITICAL +// finding on task C3 review: the pre-fix code swallowed a pre-header S3 +// source ECONNRESET as if it were a client abort). +function makeRes(req: ReqInit = {}): MockRes { + const sink = new PassThrough(); + const sendFileCalls: Array<{ name: string; root: string }> = []; + const headers: Record = {}; + const collected: Buffer[] = []; + sink.on('data', (c: Buffer) => collected.push(c)); + const res = Object.assign(sink, { + headersSent: false, + statusCode: 200, + // The remote branch reads res.req for method + conditional/Range headers, + // exactly as the local branch's res.sendFile already does. It must be a + // real EventEmitter: node's end-of-stream (which pipeline() drives) treats + // a `.req` on the destination as an http request and cleans up listeners + // on it — a plain object literal there throws + // "stream.req.removeListener is not a function" out of pipeline. + req: Object.assign(new EventEmitter(), { method: req.method ?? 'GET', headers: req.headers ?? {} }), + setHeader: (key: string, value: string) => { + // Node's real ServerResponse.setHeader THROWS ERR_INVALID_CHAR on a + // control character in the value. Reproduced here because it is + // reachable in production: Content-Disposition is built from + // client-supplied filenames (files-download.controller.ts feeds it + // multer's originalname), so a name carrying a CR/LF makes staging + // fail mid-serve — a mock that silently accepts anything would hide + // the resource leak that failure can cause. + if (/[\r\n]/.test(String(value))) { + throw Object.assign(new Error(`Invalid character in header content ["${key}"]`), { + code: 'ERR_INVALID_CHAR', + }); + } + headers[key] = value; + return res; + }, + // Node's getHeader is case-insensitive; the serving path uses it to see + // whether the CALLER already staged a Content-Type (trek-photo-cache + // does, for its extension-less `.bin` objects), so the mock must be too + // — a case-sensitive lookup here would hide a real precedence bug. + getHeader: (key: string): string | undefined => { + const found = Object.keys(headers).find((name) => name.toLowerCase() === key.toLowerCase()); + return found === undefined ? undefined : headers[found]; + }, + // express's res.set — the form every caller actually uses to stage + // headers before handing the response to sendToResponse. + set: (key: string, value: string) => { + headers[key] = value; + return res; + }, + sendFile: (name: string, opts: { root: string }, cb: (err?: Error) => void) => { + sendFileCalls.push({ name, root: opts.root }); + cb(); + }, + }); + const realWrite = sink.write.bind(sink) as (...a: unknown[]) => unknown; + const realEnd = sink.end.bind(sink) as (...a: unknown[]) => unknown; + res.write = ((...args: unknown[]) => { + res.headersSent = true; + return realWrite(...args); + }) as unknown as typeof res.write; + res.end = ((...args: unknown[]) => { + res.headersSent = true; + return realEnd(...args); + }) as unknown as typeof res.end; + return { + res: res as unknown as MockRes['res'], + sendFileCalls, + headers, + // sendToResponse resolves on 'finish', so by the time a test reads the + // body the sink has already flushed everything into `collected`. + body: async () => Buffer.concat(collected).toString(), + }; +} + +describe('StorageService sendToResponse', () => { + it('serves local files via root-relative res.sendFile (the files-download quirk)', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'doc.pdf', Readable.from('%PDF')); + + const mock = makeRes(); + await fx.storage.sendToResponse('files', 'doc.pdf', mock.res, { + contentType: 'application/pdf', + disposition: 'inline', + }); + + expect(mock.sendFileCalls).toEqual([ + { name: 'doc.pdf', root: path.join(fs.realpathSync(fx.root), 'files') }, + ]); + expect(mock.headers['Content-Type']).toBe('application/pdf'); + expect(mock.headers['Content-Disposition']).toBe('inline'); + }); + + it('throws StorageNotFoundError on a miss (callers decide 404/204/next())', async () => { + const fx = makeFixture('files/'); + const mock = makeRes(); + await expect(fx.storage.sendToResponse('files', 'ghost.pdf', mock.res)).rejects.toBeInstanceOf( + StorageNotFoundError, + ); + expect(mock.sendFileCalls).toEqual([]); + }); + + it('stream-pipes with Content-Length for path-less drivers', async () => { + const remote = makeStreamOnlyFixture('streamed body'); + const mock = makeRes(); + + await remote.storage.sendToResponse('files', 'remote.bin', mock.res, { contentType: 'image/jpeg' }); + + expect(mock.sendFileCalls).toEqual([]); // no local fast-path available + expect(mock.headers['Content-Length']).toBe(String('streamed body'.length)); + expect(mock.headers['Content-Type']).toBe('image/jpeg'); + expect(await mock.body()).toBe('streamed body'); + }); + + it('falls through to the stream branch when getLocalPath resolves but the file is not actually on disk', async () => { + // Composite-driver stand-in (task C9-i): sendToResponse must not use + // driver.stat() as "the file is at localPath" — it fs-checks localPath + // itself, and on absence falls through to the branch that actually + // reaches a MirrorDriver's replica. + const fx = makeGhostLocalPathFixture('replica bytes'); + const mock = makeRes(); + + await fx.storage.sendToResponse('files', 'ghost.bin', mock.res, { contentType: 'image/jpeg' }); + + expect(mock.sendFileCalls).toEqual([]); // the local path was never trusted + expect(fx.driverCalls).toEqual(['getStream:files/ghost.bin']); + expect(mock.headers['Content-Type']).toBe('image/jpeg'); + expect(await mock.body()).toBe('replica bytes'); + }); + + // Local (res.sendFile) branch: a client abort mid-download surfaces + // through the sendFile callback's err argument, never a throw. Mirrors the + // exact contract storageStaticHandler (platform.routes.ts) implements + // caller-side for the /uploads/* static mounts. + describe('local branch — sendFile callback abort tolerance', () => { + function makeAbortRes(err: { code?: string; syscall?: string } | undefined): MockRes { + const mock = makeRes(); + mock.res.sendFile = ((_name: string, _opts: unknown, cb: (e?: Error) => void) => { + cb(err as Error | undefined); + }) as unknown as Response['sendFile']; + return mock; + } + + it('resolves (not rejects) when sendFile fails with ECONNABORTED', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'doc.pdf', Readable.from('%PDF')); + const mock = makeAbortRes({ code: 'ECONNABORTED' }); + + await expect(fx.storage.sendToResponse('files', 'doc.pdf', mock.res)).resolves.toBeUndefined(); + }); + + it('resolves (not rejects) when sendFile fails with syscall write (broken pipe)', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'doc.pdf', Readable.from('%PDF')); + const mock = makeAbortRes({ syscall: 'write' }); + + await expect(fx.storage.sendToResponse('files', 'doc.pdf', mock.res)).resolves.toBeUndefined(); + }); + + it('rejects on a real sendFile error unrelated to a client abort', async () => { + const fx = makeFixture('files/'); + await fx.storage.put('files', 'doc.pdf', Readable.from('%PDF')); + const boom = Object.assign(new Error('disk read error'), { code: 'EIO' }); + const mock = makeAbortRes(boom); + + await expect(fx.storage.sendToResponse('files', 'doc.pdf', mock.res)).rejects.toBe(boom); + }); + }); + + // Remote (pipeline) branch: pipeline destroys BOTH ends on any failure — + // the fix for the S3 keep-alive socket leak — while a client abort must + // still resolve the caller's promise rather than reject/500. + describe('remote branch — pipeline abort tolerance and source cleanup', () => { + function makeAbortingStreamFixture(err: { code?: string; syscall?: string; message?: string }) { + const fx = makeFixture(''); + let destroyed = false; + const stream = new PassThrough(); + const origDestroy = stream.destroy.bind(stream); + stream.destroy = ((e?: Error) => { + destroyed = true; + return origDestroy(e); + }) as typeof stream.destroy; + const stat: ObjectStat = { key: 'remote.bin', size: 100, mtimeMs: 1 }; + const streamOnly: StorageDriver = { + id: 'stub-remote-abort', + put: async () => undefined, + getStream: async () => ({ stream, stat }), + stat: async () => stat, + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver: streamOnly, keyPrefix: '', backendName: 'stub-remote-abort' }), + tempDir: () => fx.tempDir, + replicaFailures: () => fx.failures, + } as unknown as StorageRegistryService; + return { + storage: new StorageService(registry), + stream, + isSourceDestroyed: () => destroyed, + fail: () => stream.destroy(Object.assign(new Error(err.message ?? 'aborted'), err)), + }; + } + + it('resolves and destroys the source stream on a POST-header ERR_STREAM_PREMATURE_CLOSE (real mid-download client abort)', async () => { + const fx = makeAbortingStreamFixture({}); + const mock = makeRes(); + const pending = fx.storage.sendToResponse('files', 'remote.bin', mock.res); + // A real "mid-download" abort only happens after at least one chunk + // has already reached res — push one through before aborting, so + // res.headersSent genuinely flips true (the gate this test exists to + // exercise), not just because the mock says so. + fx.stream.write(Buffer.from('partial')); + await new Promise((resolve) => setImmediate(resolve)); + expect(mock.res.headersSent).toBe(true); + // Destroying the destination mid-pipe is what a real client abort does + // to res — pipeline reports it as ERR_STREAM_PREMATURE_CLOSE. + mock.res.destroy(); + + await expect(pending).resolves.toBeUndefined(); + expect(fx.isSourceDestroyed()).toBe(true); // no leaked S3 keep-alive socket + }); + + it('resolves (swallows) a POST-header source ECONNRESET — a genuine mid-download network drop looks identical to a client abort', async () => { + const fx = makeAbortingStreamFixture({ code: 'ECONNRESET' }); + const mock = makeRes(); + const pending = fx.storage.sendToResponse('files', 'remote.bin', mock.res); + fx.stream.write(Buffer.from('partial')); + await new Promise((resolve) => setImmediate(resolve)); + expect(mock.res.headersSent).toBe(true); + fx.fail(); + + await expect(pending).resolves.toBeUndefined(); + }); + + // CRITICAL regression case (task C3 review): the driver's raw source + // stream (e.g. the S3 SDK's HTTP response) can itself throw an + // abort-LOOKING code (ECONNRESET/ERR_STREAM_PREMATURE_CLOSE) for a real + // reason — a network blip — with NO client involved at all, before any + // byte has reached res. The pre-fix code swallowed this unconditionally + // and resolved sendToResponse as a silent empty-body success; the fix + // gates the swallow on res.headersSent so a pre-header failure of ANY + // code — including one that looks abort-like — always rejects, and the + // caller's miss contract (404/204/next()) still runs. + it('rejects a PRE-header source ECONNRESET (a real S3 network blip, not a client abort)', async () => { + const fx = makeAbortingStreamFixture({ code: 'ECONNRESET' }); + const mock = makeRes(); + const pending = fx.storage.sendToResponse('files', 'remote.bin', mock.res); + await new Promise((resolve) => setImmediate(resolve)); + expect(mock.res.headersSent).toBe(false); // no bytes ever reached res + fx.fail(); + + await expect(pending).rejects.toThrow('aborted'); + }); + + it('rejects a PRE-header source error unrelated to any abort code', async () => { + const fx = makeAbortingStreamFixture({ code: 'EPIPE_UNRELATED', message: 'upstream reset' }); + const mock = makeRes(); + const pending = fx.storage.sendToResponse('files', 'remote.bin', mock.res); + await new Promise((resolve) => setImmediate(resolve)); + expect(mock.res.headersSent).toBe(false); + fx.fail(); + + await expect(pending).rejects.toThrow('upstream reset'); + }); + + it('rejects a POST-header source error once headers are sent but the error is not abort-like', async () => { + const fx = makeAbortingStreamFixture({ code: 'EIO', message: 'disk read error' }); + const mock = makeRes(); + const pending = fx.storage.sendToResponse('files', 'remote.bin', mock.res); + fx.stream.write(Buffer.from('partial')); + await new Promise((resolve) => setImmediate(resolve)); + expect(mock.res.headersSent).toBe(true); + fx.fail(); + + await expect(pending).rejects.toThrow('disk read error'); + }); + }); +}); + +// Remote-serving parity (task C4). The local branch inherits express send's +// content-type / ETag / Last-Modified / conditional-GET / Range / HEAD +// machinery from res.sendFile; a path-less backend has none of it, so the +// stream branch has to reproduce the same contract from one pre-flight stat. +// This suite pins the wiring — the state machine itself is proved in +// http-serving.test.ts. +describe('StorageService sendToResponse — remote serving parity', () => { + const MTIME = 5000; + const LAST_MODIFIED = new Date(MTIME).toUTCString(); + const BODY = 'abcdefghij'; // 10 bytes: indices 0-9 + + interface ServingFixture { + storage: StorageService; + getStreamCalls: Array<{ key: string; range?: ByteRange }>; + statCalls: string[]; + /** Whether the most recently opened driver stream was destroyed. */ + streamDestroyed: () => boolean; + } + + function makeServingFixture( + opts: { body?: string; mtimeMs?: number; etag?: string; missing?: boolean; vanishesBeforeGet?: boolean } = {}, + ): ServingFixture { + const tempDir = makeTmpDir(); + const bytes = Buffer.from(opts.body ?? BODY); + const base: ObjectStat = { + key: 'photo.jpg', + size: bytes.length, + mtimeMs: opts.mtimeMs ?? MTIME, + etag: opts.etag, + }; + const getStreamCalls: Array<{ key: string; range?: ByteRange }> = []; + const statCalls: string[] = []; + let destroyed = false; + const driver: StorageDriver = { + id: 'stub-serving', + put: async () => undefined, + getStream: async (key: string, range?: ByteRange) => { + getStreamCalls.push({ key, range }); + // `vanishesBeforeGet` is the stat→get delete race: the pre-flight stat + // resolved, the object was gone by the time the body was opened. + if (opts.missing || opts.vanishesBeforeGet) throw new StorageNotFoundError(key); + const slice = range ? bytes.subarray(range.start, (range.end ?? bytes.length - 1) + 1) : bytes; + // The returned stream stands in for an open S3 socket / fd: whoever + // takes it owns closing it, so destruction has to be observable. + const stream = Readable.from(slice); + const origDestroy = stream.destroy.bind(stream); + stream.destroy = ((err?: Error) => { + destroyed = true; + return origDestroy(err); + }) as typeof stream.destroy; + return { stream, stat: { ...base, key } }; + }, + stat: async (key: string) => { + statCalls.push(key); + return opts.missing ? null : { ...base, key }; + }, + delete: async () => undefined, + list: async function* () { + yield* [] as ObjectStat[]; + }, + // no getLocalPath — the universal fallback branch + }; + const registry = { + resolve: (): ResolvedCategory => ({ driver, keyPrefix: '', backendName: 'stub-serving' }), + tempDir: () => tempDir, + replicaFailures: () => [], + } as unknown as StorageRegistryService; + return { storage: new StorageService(registry), getStreamCalls, statCalls, streamDestroyed: () => destroyed }; + } + + describe('content type', () => { + it('defaults Content-Type from the key extension', async () => { + const fx = makeServingFixture(); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + expect(mock.headers['Content-Type']).toBe('image/jpeg'); + }); + + it('falls back to application/octet-stream for an unknown extension', async () => { + const fx = makeServingFixture(); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'blob.weird', mock.res); + expect(mock.headers['Content-Type']).toBe('application/octet-stream'); + }); + + it('lets an explicit opts.contentType win over the derived one', async () => { + const fx = makeServingFixture(); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res, { + contentType: 'image/webp', + disposition: 'attachment; filename="p.webp"', + }); + expect(mock.headers['Content-Type']).toBe('image/webp'); + expect(mock.headers['Content-Disposition']).toBe('attachment; filename="p.webp"'); + }); + + it('keeps a Content-Type the caller staged on the response itself', async () => { + // trek-photo-cache.service.ts's exact shape: its objects are + // extension-less `.bin` blobs whose real type lives in a meta table, so + // it res.set()s the type and passes no opts. Deriving over that would + // ship every S3-backed cached thumbnail as application/octet-stream. + const fx = makeServingFixture(); + const mock = makeRes(); + mock.res.set('Content-Type', 'image/webp'); + mock.res.set('Cache-Control', 'public, max-age=3600'); + + await fx.storage.sendToResponse('photos-trek', 'cached.bin', mock.res); + + expect(mock.headers['Content-Type']).toBe('image/webp'); + expect(mock.headers['Cache-Control']).toBe('public, max-age=3600'); + expect(await mock.body()).toBe(BODY); + }); + + it('lets opts.contentType override even a staged header', async () => { + const fx = makeServingFixture(); + const mock = makeRes(); + mock.res.set('Content-Type', 'image/webp'); + + await fx.storage.sendToResponse('photos-trek', 'cached.bin', mock.res, { contentType: 'image/avif' }); + + expect(mock.headers['Content-Type']).toBe('image/avif'); + }); + }); + + describe('validators', () => { + it('emits the driver ETag, Last-Modified and Accept-Ranges', async () => { + const fx = makeServingFixture({ etag: '"s3tag"' }); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + expect(mock.headers['ETag']).toBe('"s3tag"'); + expect(mock.headers['Last-Modified']).toBe(LAST_MODIFIED); + expect(mock.headers['Accept-Ranges']).toBe('bytes'); + expect(mock.headers['Content-Length']).toBe('10'); + expect(await mock.body()).toBe(BODY); + }); + + it('derives a weak size-mtime ETag when the driver has none', async () => { + const fx = makeServingFixture(); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + expect(mock.headers['ETag']).toBe(`W/"a-${MTIME.toString(16)}"`); + }); + + it('emits NO ETag and NO Last-Modified when the backend reports mtimeMs 0', async () => { + // S3-compatible backends can answer without a LastModified; a 1970 + // header would be "fresh forever" to any revalidating client. + const fx = makeServingFixture({ mtimeMs: 0 }); + const mock = makeRes(); + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + expect(mock.headers['ETag']).toBeUndefined(); + expect(mock.headers['Last-Modified']).toBeUndefined(); + expect(await mock.body()).toBe(BODY); // still served, just uncached + }); + }); + + describe('conditional GET', () => { + it('answers 304 to a matching If-None-Match with validators, no body and no getStream', async () => { + const fx = makeServingFixture({ etag: '"s3tag"' }); + const mock = makeRes({ headers: { 'if-none-match': '"s3tag"' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(304); + expect(mock.headers['ETag']).toBe('"s3tag"'); + expect(mock.headers['Last-Modified']).toBe(LAST_MODIFIED); + expect(mock.headers['Content-Length']).toBeUndefined(); + expect(mock.headers['Content-Type']).toBeUndefined(); + expect(fx.getStreamCalls).toEqual([]); // the whole point: one HEAD, zero bytes fetched + expect(fx.statCalls).toEqual(['photo.jpg']); + expect(await mock.body()).toBe(''); + }); + + it('answers 304 to an If-Modified-Since at or after the mtime', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { 'if-modified-since': LAST_MODIFIED } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(304); + expect(fx.getStreamCalls).toEqual([]); + }); + + it('serves 200 when the If-None-Match tag is stale', async () => { + const fx = makeServingFixture({ etag: '"s3tag"' }); + const mock = makeRes({ headers: { 'if-none-match': '"old"' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(200); + expect(await mock.body()).toBe(BODY); + }); + }); + + describe('range requests', () => { + it('answers 206 with Content-Range, the adjusted length, and passes the range to the driver', async () => { + const fx = makeServingFixture({ etag: '"s3tag"' }); + const mock = makeRes({ headers: { range: 'bytes=2-5' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(206); + expect(mock.headers['Content-Range']).toBe('bytes 2-5/10'); + expect(mock.headers['Content-Length']).toBe('4'); + expect(mock.headers['Accept-Ranges']).toBe('bytes'); + expect(fx.getStreamCalls).toEqual([{ key: 'photo.jpg', range: { start: 2, end: 5 } }]); + expect(await mock.body()).toBe('cdef'); + }); + + it('resolves an open-ended range against the pre-stat size', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { range: 'bytes=7-' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.headers['Content-Range']).toBe('bytes 7-9/10'); + expect(fx.getStreamCalls[0].range).toEqual({ start: 7, end: 9 }); + expect(await mock.body()).toBe('hij'); + }); + + it('clamps an end past EOF instead of asking the driver for bytes that do not exist', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { range: 'bytes=8-9999' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.headers['Content-Range']).toBe('bytes 8-9/10'); + expect(fx.getStreamCalls[0].range).toEqual({ start: 8, end: 9 }); + }); + + it('resolves a suffix range from the pre-stat size (the reason the path stats first)', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { range: 'bytes=-3' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(206); + expect(mock.headers['Content-Range']).toBe('bytes 7-9/10'); + expect(fx.getStreamCalls[0].range).toEqual({ start: 7, end: 9 }); + expect(await mock.body()).toBe('hij'); + }); + + it('answers 416 with Content-Range bytes */size and never touches the driver body', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { range: 'bytes=50-60' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(416); + expect(mock.headers['Content-Range']).toBe('bytes */10'); + expect(fx.getStreamCalls).toEqual([]); + expect(await mock.body()).toBe(''); + }); + + it('serves the full 200 for a multi-range or malformed Range (no multipart/byteranges)', async () => { + for (const range of ['bytes=0-1,4-5', 'bytes=abc']) { + const fx = makeServingFixture(); + const mock = makeRes({ headers: { range } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(200); + expect(mock.headers['Content-Range']).toBeUndefined(); + expect(mock.headers['Content-Length']).toBe('10'); + expect(fx.getStreamCalls[0].range).toBeUndefined(); + expect(await mock.body()).toBe(BODY); + } + }); + + it('honours a strong If-Range match and ignores the range on a mismatch', async () => { + const hit = makeServingFixture({ etag: '"s3tag"' }); + const hitRes = makeRes({ headers: { range: 'bytes=2-5', 'if-range': '"s3tag"' } }); + await hit.storage.sendToResponse('photos-trek', 'photo.jpg', hitRes.res); + expect(hitRes.res.statusCode).toBe(206); + + const miss = makeServingFixture({ etag: '"s3tag"' }); + const missRes = makeRes({ headers: { range: 'bytes=2-5', 'if-range': '"changed"' } }); + await miss.storage.sendToResponse('photos-trek', 'photo.jpg', missRes.res); + expect(missRes.res.statusCode).toBe(200); + expect(missRes.headers['Content-Length']).toBe('10'); + expect(await missRes.body()).toBe(BODY); + }); + }); + + describe('HEAD', () => { + it('sends the full headers with no body and no getStream call', async () => { + const fx = makeServingFixture({ etag: '"s3tag"' }); + const mock = makeRes({ method: 'HEAD' }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(200); + expect(mock.headers['Content-Length']).toBe('10'); + expect(mock.headers['Content-Type']).toBe('image/jpeg'); + expect(mock.headers['ETag']).toBe('"s3tag"'); + expect(mock.headers['Accept-Ranges']).toBe('bytes'); + expect(fx.getStreamCalls).toEqual([]); + expect(await mock.body()).toBe(''); + }); + + it('answers a ranged HEAD with 206 headers, still without fetching bytes', async () => { + const fx = makeServingFixture(); + const mock = makeRes({ method: 'HEAD', headers: { range: 'bytes=2-5' } }); + + await fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res); + + expect(mock.res.statusCode).toBe(206); + expect(mock.headers['Content-Range']).toBe('bytes 2-5/10'); + expect(mock.headers['Content-Length']).toBe('4'); + expect(fx.getStreamCalls).toEqual([]); + expect(await mock.body()).toBe(''); + }); + }); + + it('destroys the already-opened stream when header staging throws (no leaked S3 socket)', async () => { + // The body opens before headers are staged, so staging is now the last + // thing that can fail with a live stream in hand — and it CAN fail: + // res.setHeader raises ERR_INVALID_CHAR on a control character, and + // files-download.controller.ts builds Content-Disposition out of + // multer's client-supplied originalname. Nothing pipes the stream on + // that path, so nothing else would ever close it. + const fx = makeServingFixture(); + const mock = makeRes(); + + await expect( + fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res, { + disposition: 'attachment; filename="evil\r\nX-Injected: 1"', + }), + ).rejects.toMatchObject({ code: 'ERR_INVALID_CHAR' }); + + expect(fx.getStreamCalls).toHaveLength(1); // the body WAS opened… + expect(fx.streamDestroyed()).toBe(true); // …and closed again before the throw escaped + expect(mock.res.headersSent).toBe(false); + }); + + it('leaves the response pristine when getStream fails after a ranged decision', async () => { + // The stat→get delete race on a Range request. A caller that recovers on + // the SAME response object — trek-photo-cache.service.ts falls back to a + // cache miss on `StorageNotFoundError && !res.headersSent`, and + // photo-resolver.service.ts then re-serves through that very res — would + // otherwise inherit a 206 status and a stale Content-Range from an + // attempt that never produced a byte, even if the retry serves in full. + const fx = makeServingFixture({ vanishesBeforeGet: true }); + const mock = makeRes({ headers: { range: 'bytes=2-5' } }); + + await expect(fx.storage.sendToResponse('photos-trek', 'photo.jpg', mock.res)).rejects.toBeInstanceOf( + StorageNotFoundError, + ); + + expect(mock.res.headersSent).toBe(false); + expect(mock.res.statusCode).toBe(200); // never left on 206 + expect(mock.headers).toEqual({}); // no Content-Range / Content-Length / validator residue + expect(fx.getStreamCalls).toEqual([{ key: 'photo.jpg', range: { start: 2, end: 5 } }]); + }); + + it('throws StorageNotFoundError from the pre-flight stat, before any header is set', async () => { + const fx = makeServingFixture({ missing: true }); + const mock = makeRes(); + + await expect(fx.storage.sendToResponse('photos-trek', 'gone.jpg', mock.res)).rejects.toBeInstanceOf( + StorageNotFoundError, + ); + expect(mock.headers).toEqual({}); + expect(fx.getStreamCalls).toEqual([]); // the miss costs one HEAD, not a GET + }); +}); + +describe('StorageService health', () => { + it('surfaces the registry replica-failure ring', () => { + const fx = makeFixture(); + fx.failures.push({ backend: 'nas', key: 'backup-1.zip', op: 'put', error: 'disk full', at: 1 }); + expect(fx.storage.health().replicaFailures).toHaveLength(1); + expect(fx.storage.health().replicaFailures[0].backend).toBe('nas'); + }); +}); diff --git a/server/tests/unit/nest/thumbnail.service.test.ts b/server/tests/unit/nest/thumbnail.service.test.ts index b5006e204..4cb2aeb6b 100644 --- a/server/tests/unit/nest/thumbnail.service.test.ts +++ b/server/tests/unit/nest/thumbnail.service.test.ts @@ -11,14 +11,23 @@ import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest'; const { isAddonEnabled } = vi.hoisted(() => ({ isAddonEnabled: vi.fn(() => true) })); import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import { Jimp } from 'jimp'; -import { ThumbnailService } from '../../../src/nest/memories/thumbnail.service'; +import { ThumbnailService, journeyThumbName } from '../../../src/nest/memories/thumbnail.service'; import type { AddonsService } from '../../../src/nest/addons/addons.service'; +import Database from 'better-sqlite3'; +import { DatabaseService } from '../../../src/nest/database/database.service'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; -const svc = new ThumbnailService({ isAddonEnabled } as unknown as AddonsService); -const root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-thumbs-')); +// Category-addressed since slice 4: originals + thumbs are ('journey', ) +// objects; the fixture's 'journey/' prefix reproduces the real layout, so the +// on-disk paths below look exactly like the old uploads-root ones. +const fx = makeStorageFixture('journey/'); +// Minimal real DB — the orphan sweep only SELECTs these two columns. +const thumbsDb = new Database(':memory:'); +thumbsDb.exec('CREATE TABLE trek_photos (id INTEGER PRIMARY KEY AUTOINCREMENT, file_path TEXT, thumbnail_path TEXT)'); +const svc = new ThumbnailService({ isAddonEnabled } as unknown as AddonsService, fx.storage, new DatabaseService(thumbsDb as never)); +const root = fx.root; /** A real 1200x900 JPEG — Jimp has to be able to decode it for the happy path. */ async function writeSourceImage(rel: string): Promise { @@ -34,23 +43,28 @@ beforeEach(() => { }); afterAll(() => { - fs.rmSync(root, { recursive: true, force: true }); + thumbsDb.close(); + fx.cleanup(); }); describe('ensureLocalThumbnail', () => { it('THUMB-001: returns null when the journey addon is off, without touching the disk', async () => { isAddonEnabled.mockReturnValue(false); - expect(await svc.ensureLocalThumbnail(root, 'anything.jpg')).toBeNull(); + expect(await svc.ensureLocalThumbnail('anything.jpg')).toBeNull(); }); - it('THUMB-002: returns null when the source file does not exist', async () => { - expect(await svc.ensureLocalThumbnail(root, 'missing/nope.jpg')).toBeNull(); + it('THUMB-002: returns null when the source object does not exist', async () => { + expect(await svc.ensureLocalThumbnail('journey/missing.jpg')).toBeNull(); + }); + + it('THUMB-002b: returns null for a path outside the journey category', async () => { + expect(await svc.ensureLocalThumbnail('missing/nope.jpg')).toBeNull(); }); it('THUMB-003: downscales an oversized image and reports the resulting size', async () => { await writeSourceImage('journey/big.jpg'); - const result = await svc.ensureLocalThumbnail(root, 'journey/big.jpg'); + const result = await svc.ensureLocalThumbnail('journey/big.jpg'); expect(result).not.toBeNull(); expect(result!.thumbnailRelPath).toMatch(/^journey\/thumbs\/[0-9a-f]{16}\.jpg$/); @@ -62,19 +76,19 @@ describe('ensureLocalThumbnail', () => { it('THUMB-004: the path is deterministic, so concurrent requests cannot race on two names', async () => { await writeSourceImage('journey/stable.jpg'); - const first = await svc.ensureLocalThumbnail(root, 'journey/stable.jpg'); - const second = await svc.ensureLocalThumbnail(root, 'journey/stable.jpg'); + const first = await svc.ensureLocalThumbnail('journey/stable.jpg'); + const second = await svc.ensureLocalThumbnail('journey/stable.jpg'); expect(second!.thumbnailRelPath).toBe(first!.thumbnailRelPath); }); it('THUMB-005: reuses an existing thumbnail that is newer than the source', async () => { await writeSourceImage('journey/cached.jpg'); - const first = await svc.ensureLocalThumbnail(root, 'journey/cached.jpg'); + const first = await svc.ensureLocalThumbnail('journey/cached.jpg'); const thumbAbs = path.join(root, first!.thumbnailRelPath); const mtimeBefore = fs.statSync(thumbAbs).mtimeMs; - const second = await svc.ensureLocalThumbnail(root, 'journey/cached.jpg'); + const second = await svc.ensureLocalThumbnail('journey/cached.jpg'); expect(second).toEqual(first); expect(fs.statSync(thumbAbs).mtimeMs).toBe(mtimeBefore); @@ -82,13 +96,13 @@ describe('ensureLocalThumbnail', () => { it('THUMB-006: regenerates when the source is newer than the thumbnail', async () => { await writeSourceImage('journey/changed.jpg'); - const first = await svc.ensureLocalThumbnail(root, 'journey/changed.jpg'); + const first = await svc.ensureLocalThumbnail('journey/changed.jpg'); const thumbAbs = path.join(root, first!.thumbnailRelPath); // Age the thumbnail past the source. const old = new Date(Date.now() - 60_000); fs.utimesSync(thumbAbs, old, old); - const second = await svc.ensureLocalThumbnail(root, 'journey/changed.jpg'); + const second = await svc.ensureLocalThumbnail('journey/changed.jpg'); expect(second!.thumbnailRelPath).toBe(first!.thumbnailRelPath); expect(fs.statSync(thumbAbs).mtimeMs).toBeGreaterThan(old.getTime()); @@ -99,6 +113,62 @@ describe('ensureLocalThumbnail', () => { fs.mkdirSync(path.dirname(abs), { recursive: true }); fs.writeFileSync(abs, 'not an image at all'); - expect(await svc.ensureLocalThumbnail(root, 'journey/corrupt.jpg')).toBeNull(); + expect(await svc.ensureLocalThumbnail('journey/corrupt.jpg')).toBeNull(); + }); +}); + +describe('sweepOrphanThumbs (spec fix #2)', () => { + const thumbsDir = path.join(root, 'journey', 'thumbs'); + const writeThumb = (name: string): string => { + fs.mkdirSync(thumbsDir, { recursive: true }); + const fp = path.join(thumbsDir, name); + fs.writeFileSync(fp, 'jpeg'); + return fp; + }; + + beforeEach(() => { + thumbsDb.prepare('DELETE FROM trek_photos').run(); + fs.rmSync(thumbsDir, { recursive: true, force: true }); + }); + + it('THUMB-SWEEP-001: deletes strays, spares thumbs derivable from live journey rows', async () => { + thumbsDb.prepare('INSERT INTO trek_photos (file_path) VALUES (?)').run('journey/live.jpg'); + const liveName = journeyThumbName('journey/live.jpg'); // 'thumbs/.jpg' + const livePath = writeThumb(path.basename(liveName)); + const strayPath = writeThumb('deadbeefdeadbeef.jpg'); + + const removed = await svc.sweepOrphanThumbs(); + + expect(fs.existsSync(livePath)).toBe(true); + expect(fs.existsSync(strayPath)).toBe(false); + expect(removed).toBe(1); + }); + + it('THUMB-SWEEP-002: spares a recorded thumbnail_path even without a matching file_path hash', async () => { + thumbsDb.prepare('INSERT INTO trek_photos (file_path, thumbnail_path) VALUES (?, ?)') + .run('elsewhere/x.jpg', 'journey/thumbs/recorded00000000.jpg'); + const recorded = writeThumb('recorded00000000.jpg'); + + const removed = await svc.sweepOrphanThumbs(); + + expect(fs.existsSync(recorded)).toBe(true); + expect(removed).toBe(0); + }); + + it('THUMB-SWEEP-003: the addon gate answers 0 without listing or deleting', async () => { + isAddonEnabled.mockReturnValue(false); + const stray = writeThumb('gatedstray000000.jpg'); + + expect(await svc.sweepOrphanThumbs()).toBe(0); + expect(fs.existsSync(stray)).toBe(true); + }); + + it('THUMB-SWEEP-004: non-jpg entries under thumbs/ are never touched', async () => { + fs.mkdirSync(thumbsDir, { recursive: true }); + const note = path.join(thumbsDir, 'note.txt'); + fs.writeFileSync(note, 'not a thumb'); + + expect(await svc.sweepOrphanThumbs()).toBe(0); + expect(fs.existsSync(note)).toBe(true); }); }); diff --git a/server/tests/unit/nest/trek-photo-cache.service.test.ts b/server/tests/unit/nest/trek-photo-cache.service.test.ts index 747bc2fa9..7c7608f59 100644 --- a/server/tests/unit/nest/trek-photo-cache.service.test.ts +++ b/server/tests/unit/nest/trek-photo-cache.service.test.ts @@ -1,11 +1,15 @@ /** - * TrekPhotoCacheService — the disk + metadata cache for provider assets. + * TrekPhotoCacheService — the storage + metadata cache for provider assets. * * It had no suite of its own: it lived in services/memories/, which the * coverage gate does not measure, so a cache that silently stopped evicting or * stopped deduping concurrent fetches would not have failed anything. The * stampede guard in particular is the reason the in-flight map is module-scoped * rather than an instance field. + * + * Slice 4: the cache addresses storage as ('photos-trek', '.bin') over a + * stub-registry fixture rooted in a throwaway dir — the real uploads tree is + * never touched. */ import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest'; @@ -26,17 +30,18 @@ import { createTables } from '../../../src/db/schema'; import { runMigrations } from '../../../src/db/migrations'; import { DatabaseService } from '../../../src/nest/database/database.service'; import { TrekPhotoCacheService, CACHE_TTL } from '../../../src/nest/memories/trek-photo-cache.service'; -import { UPLOADS_ROOT } from '../../../src/nest/memories/uploads-root'; +import { StorageNotFoundError } from '../../../src/nest/storage/storage.types'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; -const svc = new TrekPhotoCacheService(new DatabaseService(testDb)); -const CACHE_DIR = path.join(UPLOADS_ROOT, 'photos/trek'); -const written: string[] = []; +const fx = makeStorageFixture('photos/trek/'); +const svc = new TrekPhotoCacheService(new DatabaseService(testDb), fx.storage); +const CACHE_DIR = path.join(fx.root, 'photos/trek'); +const binPath = (key: string) => path.join(CACHE_DIR, `${key}.bin`); +let counter = 0; -/** Unique per case so parallel files cannot collide on the shared cache dir. */ +/** Unique per case so cases cannot collide on the shared fixture dir. */ function freshKey(label: string): string { - const key = svc.cacheKey('test', `${label}-${process.pid}-${written.length}`, 'thumbnail', 1); - written.push(key); - return key; + return svc.cacheKey('test', `${label}-${counter++}`, 'thumbnail', 1); } beforeAll(() => { @@ -49,10 +54,8 @@ beforeEach(() => { }); afterAll(() => { - for (const key of written) { - try { fs.unlinkSync(path.join(CACHE_DIR, `${key}.bin`)); } catch { /* never written */ } - } testDb.close(); + fx.cleanup(); }); describe('cacheKey', () => { @@ -66,18 +69,18 @@ describe('cacheKey', () => { }); describe('put / getFresh', () => { - it('CACHE-002: a stored entry comes back with its content type and a real file', async () => { + it('CACHE-002: a stored entry comes back with its content type and a real object', async () => { const key = freshKey('hit'); await svc.put(key, Buffer.from('jpegbytes'), 'image/jpeg'); - const entry = svc.getFresh(key); + const entry = await svc.getFresh(key); expect(entry).not.toBeNull(); expect(entry!.contentType).toBe('image/jpeg'); - expect(fs.readFileSync(entry!.filePath).toString()).toBe('jpegbytes'); + expect(fs.readFileSync(binPath(key)).toString()).toBe('jpegbytes'); }); - it('CACHE-003: an unknown key is a miss, not an error', () => { - expect(svc.getFresh('no-such-key')).toBeNull(); + it('CACHE-003: an unknown key is a miss, not an error', async () => { + expect(await svc.getFresh('no-such-key')).toBeNull(); }); it('CACHE-004: an entry past its TTL is a miss and its metadata row is dropped', async () => { @@ -86,16 +89,16 @@ describe('put / getFresh', () => { testDb.prepare('UPDATE trek_photo_cache_meta SET fetched_at = ? WHERE cache_key = ?') .run(Date.now() - CACHE_TTL - 1000, key); - expect(svc.getFresh(key)).toBeNull(); + expect(await svc.getFresh(key)).toBeNull(); expect(testDb.prepare('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?').get(key)).toBeUndefined(); }); - it('CACHE-005: metadata without its file is a miss and the row is dropped', () => { + it('CACHE-005: metadata without its object is a miss and the row is dropped', async () => { const key = 'orphan-meta-row'; testDb.prepare('INSERT INTO trek_photo_cache_meta (cache_key, content_type, fetched_at) VALUES (?, ?, ?)') .run(key, 'image/jpeg', Date.now()); - expect(svc.getFresh(key)).toBeNull(); + expect(await svc.getFresh(key)).toBeNull(); expect(testDb.prepare('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?').get(key)).toBeUndefined(); }); @@ -106,28 +109,43 @@ describe('put / getFresh', () => { const rows = testDb.prepare('SELECT content_type FROM trek_photo_cache_meta WHERE cache_key = ?').all(key); expect(rows).toHaveLength(1); - expect(svc.getFresh(key)!.contentType).toBe('image/png'); - expect(fs.readFileSync(svc.getFresh(key)!.filePath).toString()).toBe('second'); + expect((await svc.getFresh(key))!.contentType).toBe('image/png'); + expect(fs.readFileSync(binPath(key)).toString()).toBe('second'); }); }); describe('serveFresh', () => { - it('CACHE-007: sends the cached file with a one-hour cache header', async () => { + it('CACHE-007: sends the cached bytes through storage with a one-hour cache header', async () => { const key = freshKey('serve'); await svc.put(key, Buffer.from('bytes'), 'image/webp'); - const res = { set: vi.fn(), sendFile: vi.fn() }; + const send = vi.spyOn(fx.storage, 'sendToResponse').mockResolvedValueOnce(undefined); + const res = { set: vi.fn() }; - expect(svc.serveFresh(res as never, key)).toBe(true); + expect(await svc.serveFresh(res as never, key)).toBe(true); expect(res.set).toHaveBeenCalledWith('Content-Type', 'image/webp'); expect(res.set).toHaveBeenCalledWith('Cache-Control', 'public, max-age=3600'); - expect(res.sendFile).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith('photos-trek', `${key}.bin`, res); + send.mockRestore(); }); - it('CACHE-008: reports a miss without touching the response', () => { - const res = { set: vi.fn(), sendFile: vi.fn() }; - expect(svc.serveFresh(res as never, 'nothing-cached')).toBe(false); + it('CACHE-008: reports a miss without touching the response or storage', async () => { + const send = vi.spyOn(fx.storage, 'sendToResponse'); + const res = { set: vi.fn() }; + expect(await svc.serveFresh(res as never, 'nothing-cached')).toBe(false); expect(res.set).not.toHaveBeenCalled(); - expect(res.sendFile).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + send.mockRestore(); + }); + + it('CACHE-013: a getFresh→send delete race reads as a miss, not a crash', async () => { + const key = freshKey('race'); + await svc.put(key, Buffer.from('bytes'), 'image/webp'); + const send = vi.spyOn(fx.storage, 'sendToResponse') + .mockRejectedValueOnce(new StorageNotFoundError(`photos/trek/${key}.bin`)); + const res = { set: vi.fn(), headersSent: false }; + + expect(await svc.serveFresh(res as never, key)).toBe(false); + send.mockRestore(); }); }); @@ -151,7 +169,7 @@ describe('the stampede guard', () => { // Load-bearing: a per-instance map would hand any second instance a private // guard and the dedup would be silently gone. The sweep cron injects the // container singleton now, but the invariant stays pinned. - const other = new TrekPhotoCacheService(new DatabaseService(testDb)); + const other = new TrekPhotoCacheService(new DatabaseService(testDb), fx.storage); const promise = Promise.resolve(null); svc.setInFlight('shared-key', promise); expect(other.getInFlight('shared-key')).toBe(promise); @@ -159,28 +177,65 @@ describe('the stampede guard', () => { }); describe('sweepExpired', () => { - it('CACHE-011: drops rows and files past twice the TTL and leaves fresh ones alone', async () => { + it('CACHE-011: drops rows and objects past twice the TTL and leaves fresh ones alone', async () => { const staleKey = freshKey('sweep-stale'); const freshKeyId = freshKey('sweep-fresh'); await svc.put(staleKey, Buffer.from('old'), 'image/jpeg'); await svc.put(freshKeyId, Buffer.from('new'), 'image/jpeg'); testDb.prepare('UPDATE trek_photo_cache_meta SET fetched_at = ? WHERE cache_key = ?') .run(Date.now() - CACHE_TTL * 2 - 1000, staleKey); - const staleFile = path.join(CACHE_DIR, `${staleKey}.bin`); - svc.sweepExpired(); + await svc.sweepExpired(); expect(testDb.prepare('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?').get(staleKey)).toBeUndefined(); - expect(fs.existsSync(staleFile)).toBe(false); + expect(fs.existsSync(binPath(staleKey))).toBe(false); expect(testDb.prepare('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?').get(freshKeyId)).toBeDefined(); }); - it('CACHE-012: survives a metadata row whose file is already gone', () => { + it('CACHE-012: survives a metadata row whose object is already gone', async () => { const key = 'sweep-orphan'; testDb.prepare('INSERT INTO trek_photo_cache_meta (cache_key, content_type, fetched_at) VALUES (?, ?, ?)') .run(key, 'image/jpeg', Date.now() - CACHE_TTL * 3); - expect(() => svc.sweepExpired()).not.toThrow(); + await expect(svc.sweepExpired()).resolves.toBeUndefined(); expect(testDb.prepare('SELECT 1 FROM trek_photo_cache_meta WHERE cache_key = ?').get(key)).toBeUndefined(); }); + + it('CACHE-014: reclaims a row-less .bin object past the cutoff (fix #4, spec rev 3.2)', async () => { + // getFresh's expiry path deletes the meta row but never the object — the + // list-driven pass 2 is what reclaims those leaks. + const key = freshKey('rowless'); + await svc.put(key, Buffer.from('leaked'), 'image/jpeg'); + testDb.prepare('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?').run(key); + const old = (Date.now() - CACHE_TTL * 3) / 1000; + fs.utimesSync(binPath(key), old, old); + + await svc.sweepExpired(); + expect(fs.existsSync(binPath(key))).toBe(false); + }); + + it('CACHE-015: a row-less .bin younger than the cutoff survives (in-flight put guard)', async () => { + const key = freshKey('rowless-fresh'); + await svc.put(key, Buffer.from('in-flight'), 'image/jpeg'); + testDb.prepare('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?').run(key); + + await svc.sweepExpired(); + expect(fs.existsSync(binPath(key))).toBe(true); + }); + + it('CACHE-016: nested and non-.bin entries are never touched by the sweep', async () => { + const nestedDir = path.join(CACHE_DIR, 'sub'); + fs.mkdirSync(nestedDir, { recursive: true }); + const nested = path.join(nestedDir, 'nested.bin'); + fs.writeFileSync(nested, 'nested'); + const note = path.join(CACHE_DIR, 'note.txt'); + fs.writeFileSync(note, 'not a cache object'); + const old = (Date.now() - CACHE_TTL * 3) / 1000; + fs.utimesSync(nested, old, old); + fs.utimesSync(note, old, old); + + await svc.sweepExpired(); + expect(fs.existsSync(nested)).toBe(true); + expect(fs.existsSync(note)).toBe(true); + }); }); diff --git a/server/tests/unit/nest/trip-read-model.service.test.ts b/server/tests/unit/nest/trip-read-model.service.test.ts index 54ea00bdf..262652900 100644 --- a/server/tests/unit/nest/trip-read-model.service.test.ts +++ b/server/tests/unit/nest/trip-read-model.service.test.ts @@ -72,6 +72,7 @@ import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.se import { UnsplashService } from '../../../src/nest/unsplash/unsplash.service'; import { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.service'; import { TrekPhotosRepository } from '../../../src/nest/photos/trek-photos.repository'; @@ -83,12 +84,13 @@ const budgetSvc = new BudgetService(dbs(), new PermissionsService(dbs()), new Ex const daysSvc = new DaysService(dbs(), new PermissionsService(dbs()), new RealtimeService(), new QueryHelpersService(dbs())); // One shared cache instance (the PlacePhotoCacheService rule): the in-flight dedup in // PlacePhotoCacheService only works while both consumers hold the same object. -const photoCache = new PlacePhotoCacheService(dbs(), new RuntimeEnvService()); +const photoCache = new PlacePhotoCacheService(dbs(), makeStorageFixture('photos/google/').storage); const placesSvc = new PlacesService( dbs(), new PermissionsService(dbs()), new RealtimeService(), new MapsService(dbs(), photoCache), new QueryHelpersService(dbs()), - new UnsplashService(dbs(), new RuntimeEnvService()), photoCache, + new UnsplashService(dbs(), new RuntimeEnvService(), makeStorageFixture('').storage), photoCache, new JourneyDomainService(dbs(), new RealtimeService(), new TrekPhotosRepository(dbs())), + makeStorageFixture('').storage, ); const accommodationsSvc = new AccommodationsService(dbs(), new PermissionsService(dbs()), new RealtimeService()); const membersSvc = new TripMembersService(dbs(), budgetSvc, new UserCleanupService(dbs(), budgetSvc), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()); @@ -98,10 +100,10 @@ const buildReadModel = (database: DatabaseService, roster: TripMembersService = database, roster, daysSvc, accommodationsSvc, budgetSvc, new PackingService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()), new ReservationsService(dbs(), new PermissionsService(dbs()), budgetSvc, new RealtimeService(), notificationsStub(), new ReservationsReadRepository(dbs())), - new CollabService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()), + new CollabService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub(), makeStorageFixture('').storage), placesSvc, new TodoService(dbs(), new PermissionsService(dbs()), new RealtimeService()), - new FilesService(dbs(), new PermissionsService(dbs()), new RealtimeService(), new EphemeralTokenService()), + new FilesService(dbs(), new PermissionsService(dbs()), new RealtimeService(), new EphemeralTokenService(), makeStorageFixture('').storage), ); const svc = buildReadModel(dbs()); diff --git a/server/tests/unit/nest/trips.controller.test.ts b/server/tests/unit/nest/trips.controller.test.ts index 1b09a4fac..a26a8c935 100644 --- a/server/tests/unit/nest/trips.controller.test.ts +++ b/server/tests/unit/nest/trips.controller.test.ts @@ -2,9 +2,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { UnsplashService } from '../../../src/nest/unsplash/unsplash.service'; import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; import { HttpException } from '@nestjs/common'; -import { INTERCEPTORS_METADATA } from '@nestjs/common/constants'; -import fs from 'fs'; -import path from 'path'; import type { Request } from 'express'; vi.mock('../../../src/nest/audit/client-ip', () => ({ getClientIp: vi.fn(() => '1.2.3.4') })); @@ -19,7 +16,7 @@ const unsplashStub = { saveUnsplashCover: vi.fn().mockResolvedValue('mock-cover.jpg'), } as unknown as UnsplashService; -import { TripsController } from '../../../src/nest/trips/trips.controller'; +import { TripsController, MAX_COVER_SIZE, TRIP_COVER_FILE_FILTER } from '../../../src/nest/trips/trips.controller'; import type { AuditService } from '../../../src/nest/audit/audit.service'; import type { TripsService } from '../../../src/nest/trips/trips.service'; import type { CalendarService } from '../../../src/nest/calendar/calendar.service'; @@ -39,8 +36,9 @@ const audit = { writeAudit } as unknown as AuditService; // they were; the two that do pass a stub. // calendar and readModel are optional so the call sites that reach neither the // ICS route nor the bundle stay as they were; the ones that do pass a stub. +const storageStub = { put: vi.fn().mockResolvedValue(undefined) } as unknown as import('../../../src/nest/storage/storage.service').StorageService; const tc = (s: TripsService, calendar?: Partial, readModel?: Partial) => - new TripsController(s, audit, new RuntimeEnvService(), unsplashStub, (calendar ?? {}) as CalendarService, (readModel ?? {}) as TripReadModelService); + new TripsController(s, audit, new RuntimeEnvService(), unsplashStub, (calendar ?? {}) as CalendarService, (readModel ?? {}) as TripReadModelService, storageStub); function svc(o: Partial = {}): TripsService { return { @@ -307,24 +305,25 @@ describe('TripsController (parity with the legacy /api/trips route)', () => { describe('POST /:id/cover', () => { const file = { filename: 'abc.jpg' } as Express.Multer.File; - it('404 without access, 403 without permission, 404 raw trip, 400 no file, else returns url', () => { - expect(thrown(() => tc(svc({ canAccessTrip: vi.fn().mockReturnValue(undefined) })).cover(user, '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); - expect(thrown(() => tc(svc({ can: vi.fn().mockReturnValue(false) })).cover(user, '9', file))).toEqual({ status: 403, body: { error: 'No permission to change the cover image' } }); - expect(thrown(() => tc(svc({ getRaw: vi.fn().mockReturnValue(undefined) } as Partial)).cover(user, '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); - expect(thrown(() => tc(svc({ getRaw: vi.fn().mockReturnValue({ cover_image: null }) } as Partial)).cover(user, '9', undefined))).toEqual({ status: 400, body: { error: 'No image uploaded' } }); + it('404 without access, 403 without permission, 404 raw trip, 400 no file, else commits + returns url', async () => { + expect(await thrownAsync(() => tc(svc({ canAccessTrip: vi.fn().mockReturnValue(undefined) })).cover(user, '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); + expect(await thrownAsync(() => tc(svc({ can: vi.fn().mockReturnValue(false) })).cover(user, '9', file))).toEqual({ status: 403, body: { error: 'No permission to change the cover image' } }); + expect(await thrownAsync(() => tc(svc({ getRaw: vi.fn().mockReturnValue(undefined) } as Partial)).cover(user, '9', file))).toEqual({ status: 404, body: { error: 'Trip not found' } }); + expect(await thrownAsync(() => tc(svc({ getRaw: vi.fn().mockReturnValue({ cover_image: null }) } as Partial)).cover(user, '9', undefined))).toEqual({ status: 400, body: { error: 'No image uploaded' } }); const deleteOldCover = vi.fn(); const updateCoverImage = vi.fn(); const s = svc({ getRaw: vi.fn().mockReturnValue({ cover_image: '/old.jpg' }), deleteOldCover, updateCoverImage } as Partial); - expect(tc(s).cover(user, '9', file)).toEqual({ cover_image: '/uploads/covers/abc.jpg' }); + expect(await tc(s).cover(user, '9', file)).toEqual({ cover_image: '/uploads/covers/abc.jpg' }); + expect(storageStub.put).toHaveBeenCalledWith('covers', 'abc.jpg', { tmpPath: undefined }); expect(deleteOldCover).toHaveBeenCalledWith('/old.jpg'); expect(updateCoverImage).toHaveBeenCalledWith('9', '/uploads/covers/abc.jpg'); }); - it('403 in demo mode for a demo account', () => { + it('403 in demo mode for a demo account', async () => { const prev = process.env.DEMO_MODE; process.env.DEMO_MODE = 'true'; isDemoEmail.mockReturnValueOnce(true); try { - expect(thrown(() => tc(svc()).cover(user, '9', file))).toEqual({ + expect(await thrownAsync(() => tc(svc()).cover(user, '9', file))).toEqual({ status: 403, body: { error: 'Uploads are disabled in demo mode. Self-host TREK for full functionality.' }, }); } finally { @@ -334,69 +333,15 @@ describe('TripsController (parity with the legacy /api/trips route)', () => { }); /** - * The multer options are a module-private const wired into the route through - * @UseInterceptors, so the handler tests above never reach them: a broken - * destination/filename/filter would still let every case in this file pass. - * Pulling the interceptor back off the route and reading the multer instance - * it built (multer keeps storage, limits and fileFilter verbatim) exercises - * the real callbacks instead of asserting an object literal. + * The multer options moved into trips.module.ts's MulterModule factory + * (storage engine + spool destination are covered by the storage-upload + * factory unit suite and TRIP-P01/P02 integration parity); the trips-owned + * piece that remains testable here is the exported cover fileFilter. */ - type StorageCb = (err: Error | null, value: string) => void; - type CoverUpload = { - storage: { - getDestination: (req: unknown, file: Express.Multer.File, cb: StorageCb) => void; - getFilename: (req: unknown, file: Express.Multer.File, cb: StorageCb) => void; - }; - limits: { fileSize: number }; - fileFilter: (req: unknown, file: Express.Multer.File, cb: (err: Error | null, accept: boolean) => void) => void; - }; - function coverUpload(): CoverUpload { - const [Interceptor] = Reflect.getMetadata(INTERCEPTORS_METADATA, TripsController.prototype.cover) as Array { multer: CoverUpload }>; - return new Interceptor().multer; - } - const coversDir = path.join('uploads', 'covers'); - - it('creates uploads/covers on the first upload and reuses it afterwards', () => { - // A fresh container has no uploads/covers, and multer does not create the - // destination itself, so without the mkdir the very first cover upload on - // a new instance fails with ENOENT. - const exists = vi.spyOn(fs, 'existsSync').mockReturnValue(false); - const mkdir = vi.spyOn(fs, 'mkdirSync').mockImplementation(() => undefined); - try { - const { getDestination } = coverUpload().storage; - const cb = vi.fn(); - getDestination(req, file, cb); - expect(mkdir).toHaveBeenCalledWith(expect.stringContaining(coversDir), { recursive: true }); - expect(cb).toHaveBeenCalledWith(null, expect.stringContaining(coversDir)); - exists.mockReturnValue(true); - mkdir.mockClear(); - getDestination(req, file, cb); - expect(mkdir).not.toHaveBeenCalled(); - expect(cb).toHaveBeenLastCalledWith(null, expect.stringContaining(coversDir)); - } finally { - exists.mockRestore(); - mkdir.mockRestore(); - } - }); - - it('gives every stored cover a fresh name and keeps the original extension', () => { - const { getFilename } = coverUpload().storage; - const cb = vi.fn(); - const upload = { originalname: 'sunset.JPG' } as Express.Multer.File; - getFilename(req, upload, cb); - getFilename(req, upload, cb); - const [first, second] = cb.mock.calls.map((c) => c[1] as string); - // Keeping the client's name would let one trip's upload overwrite another's; - // dropping the extension leaves the static handler with no mime to serve. - expect(first).toMatch(/^[0-9a-f-]{36}\.JPG$/); - expect(second).not.toBe(first); - }); - it('accepts the four allowed image types and rejects svg, a faked extension and a non-image mime', () => { - const { fileFilter, limits } = coverUpload(); const verdict = (originalname: string, mimetype: string) => { const cb = vi.fn(); - fileFilter(req, { originalname, mimetype } as Express.Multer.File, cb); + TRIP_COVER_FILE_FILTER!(req, { originalname, mimetype } as Express.Multer.File, cb); return cb.mock.calls[0]; }; expect(verdict('a.jpg', 'image/jpeg')).toEqual([null, true]); @@ -404,11 +349,14 @@ describe('TripsController (parity with the legacy /api/trips route)', () => { expect(verdict('a.WEBP', 'image/webp')).toEqual([null, true]); // svg passes the image/* prefix but runs script when the browser opens the // stored cover, so it is rejected even under an allowed extension. - expect(verdict('logo.png', 'image/svg+xml')).toEqual([expect.any(Error), false]); + expect(verdict('logo.png', 'image/svg+xml')).toEqual([expect.any(Error)]); // A binary renamed to an image mime must still fail on the extension. - expect(verdict('payload.exe', 'image/png')).toEqual([expect.any(Error), false]); - expect(verdict('a.png', 'application/octet-stream')).toEqual([expect.any(Error), false]); - expect(limits.fileSize).toBe(20 * 1024 * 1024); + expect(verdict('payload.exe', 'image/png')).toEqual([expect.any(Error)]); + expect(verdict('a.png', 'application/octet-stream')).toEqual([expect.any(Error)]); + // The quirk stays quirky: the rejection error carries NO statusCode, so + // the route answers 500 (pinned by TRIP-P04), not 400. + expect((verdict('logo.png', 'image/svg+xml')[0] as Error & { statusCode?: number }).statusCode).toBeUndefined(); + expect(MAX_COVER_SIZE).toBe(20 * 1024 * 1024); }); }); diff --git a/server/tests/unit/nest/trips.service.test.ts b/server/tests/unit/nest/trips.service.test.ts index d872b6d4e..61a5a8bb6 100644 --- a/server/tests/unit/nest/trips.service.test.ts +++ b/server/tests/unit/nest/trips.service.test.ts @@ -79,8 +79,10 @@ import { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-pho import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.service'; import { TrekPhotosRepository } from '../../../src/nest/photos/trek-photos.repository'; import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; import { QueryHelpersService } from '../../../src/nest/query-helpers/query-helpers.service'; import fs from 'fs'; +import path from 'path'; import { notificationsStub } from '../../helpers/notifications'; import { EphemeralTokenService } from '../../../src/nest/auth/ephemeral-token.service'; @@ -96,16 +98,18 @@ const daysSvc = new DaysService(dbs(), new PermissionsService(dbs()), new Realti // missing collaborator would look like a pass while swallowing a TypeError. // One PlacePhotoCacheService for both PlacesService and MapsService, matching // production, where the in-flight dedup only works on a shared instance. -const photoCache = new PlacePhotoCacheService(dbs(), new RuntimeEnvService()); +const photoCache = new PlacePhotoCacheService(dbs(), makeStorageFixture('photos/google/').storage); +const coversFx = makeStorageFixture('covers/'); const placesSvc = new PlacesService( dbs(), new PermissionsService(dbs()), new RealtimeService(), new MapsService(dbs(), photoCache), new QueryHelpersService(dbs()), - new UnsplashService(dbs(), new RuntimeEnvService()), + new UnsplashService(dbs(), new RuntimeEnvService(), coversFx.storage), photoCache, new JourneyDomainService(dbs(), new RealtimeService(), new TrekPhotosRepository(dbs())), + makeStorageFixture('').storage, ); const accommodationsSvc = new AccommodationsService(dbs(), new PermissionsService(dbs()), new RealtimeService()); const createAccommodation = accommodationsSvc.createAccommodation.bind(accommodationsSvc); @@ -119,16 +123,17 @@ const svc = new TripsService( new VacayService(dbs(), new RealtimeService(), notificationsStub()), new RealtimeService(), undefined as never, // unsplash — not exercised here + coversFx.storage, ); const membersSvc = new TripMembersService(dbs(), budgetSvc, new UserCleanupService(dbs(), budgetSvc), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()); const readModelSvc = new TripReadModelService( dbs(), membersSvc, daysSvc, accommodationsSvc, budgetSvc, new PackingService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()), new ReservationsService(dbs(), new PermissionsService(dbs()), budgetSvc, new RealtimeService(), notificationsStub(), new ReservationsReadRepository(dbs())), - new CollabService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub()), + new CollabService(dbs(), new PermissionsService(dbs()), new RealtimeService(), notificationsStub(), coversFx.storage), placesSvc, new TodoService(dbs(), new PermissionsService(dbs()), new RealtimeService()), - new FilesService(dbs(), new PermissionsService(dbs()), new RealtimeService(), new EphemeralTokenService()), + new FilesService(dbs(), new PermissionsService(dbs()), new RealtimeService(), new EphemeralTokenService(), coversFx.storage), ); @@ -370,38 +375,39 @@ describe('generateDays', () => { // ── deleteOldCover — path containment ────────────────────────────────────────── describe('deleteOldCover', () => { - it('TRIP-SVC-COVER-001: never unlinks outside uploads/covers for a crafted cover_image', () => { - const existsSpy = vi.spyOn(fs, 'existsSync').mockReturnValue(true); - const unlinkSpy = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => {}); - try { - // Attacker-controlled values aimed at auth-gated sibling upload dirs. - svc.deleteOldCover('/uploads/files/victim.pdf'); - svc.deleteOldCover('/uploads/covers/../files/secret.pdf'); - svc.deleteOldCover('/uploads/avatars/someone.png'); + it('TRIP-SVC-COVER-001: never deletes outside the covers category for a crafted cover_image', async () => { + // Attacker-controlled values aimed at auth-gated sibling upload dirs — the + // basename + category addressing keeps every delete inside covers/. + const filesDir = path.join(coversFx.root, 'files'); + const avatarsDir = path.join(coversFx.root, 'avatars'); + fs.mkdirSync(filesDir, { recursive: true }); + fs.mkdirSync(avatarsDir, { recursive: true }); + const secret = path.join(filesDir, 'secret.pdf'); + const someone = path.join(avatarsDir, 'someone.png'); + fs.writeFileSync(secret, 'pdf'); + fs.writeFileSync(someone, 'png'); - for (const call of unlinkSpy.mock.calls) { - const target = String(call[0]); - expect(target).toMatch(/[\\/]uploads[\\/]covers[\\/]/); // stays in covers - expect(target).not.toMatch(/[\\/]files[\\/]/); - expect(target).not.toMatch(/[\\/]avatars[\\/]/); - } - } finally { - existsSpy.mockRestore(); - unlinkSpy.mockRestore(); - } + await svc.deleteOldCover('/uploads/files/secret.pdf'); + await svc.deleteOldCover('/uploads/covers/../files/secret.pdf'); + await svc.deleteOldCover('/uploads/avatars/someone.png'); + + expect(fs.existsSync(secret)).toBe(true); + expect(fs.existsSync(someone)).toBe(true); }); - it('TRIP-SVC-COVER-002: deletes a legitimate cover file', () => { - const existsSpy = vi.spyOn(fs, 'existsSync').mockReturnValue(true); - const unlinkSpy = vi.spyOn(fs, 'unlinkSync').mockImplementation(() => {}); - try { - svc.deleteOldCover('/uploads/covers/abc123.jpg'); - expect(unlinkSpy).toHaveBeenCalledTimes(1); - expect(String(unlinkSpy.mock.calls[0][0])).toMatch(/[\\/]covers[\\/]abc123\.jpg$/); - } finally { - existsSpy.mockRestore(); - unlinkSpy.mockRestore(); - } + it('TRIP-SVC-COVER-002: deletes a legitimate cover file', async () => { + const coversDir = path.join(coversFx.root, 'covers'); + fs.mkdirSync(coversDir, { recursive: true }); + const cover = path.join(coversDir, 'abc123.jpg'); + fs.writeFileSync(cover, 'jpeg'); + + await svc.deleteOldCover('/uploads/covers/abc123.jpg'); + expect(fs.existsSync(cover)).toBe(false); + }); + + it('TRIP-SVC-COVER-003: an external https cover URL is tolerated (no throw)', async () => { + await expect(svc.deleteOldCover('https://example.com/some/pic.jpg')).resolves.toBeUndefined(); + await expect(svc.deleteOldCover(null)).resolves.toBeUndefined(); }); }); @@ -1015,6 +1021,7 @@ describe('quirk fixes', () => { new VacayService(dbs(), new RealtimeService(), notificationsStub()), new RealtimeService(), undefined as never, + coversFx.storage, ); } diff --git a/server/tests/unit/nest/unsplash/unsplash.service.test.ts b/server/tests/unit/nest/unsplash/unsplash.service.test.ts index 74d6d4ca5..ed6a15fb7 100644 --- a/server/tests/unit/nest/unsplash/unsplash.service.test.ts +++ b/server/tests/unit/nest/unsplash/unsplash.service.test.ts @@ -1,5 +1,4 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; -import os from 'os'; import fs from 'fs'; import path from 'path'; @@ -33,9 +32,11 @@ import { UnsplashService } from '../../../../src/nest/unsplash/unsplash.service' import { DatabaseService } from '../../../../src/nest/database/database.service'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; import { db } from '../../../../src/db/database'; +import { makeStorageFixture } from '../../../helpers/storage-fixture'; // Same four entry points, now methods. The db mock above still feeds them. -const svc = new UnsplashService(new DatabaseService(db), new RuntimeEnvService()); +const coverFx = makeStorageFixture('covers/'); +const svc = new UnsplashService(new DatabaseService(db), new RuntimeEnvService(), coverFx.storage); const searchUnsplashPhotos = svc.searchUnsplashPhotos.bind(svc); const getUnsplashKey = svc.getUnsplashKey.bind(svc); const saveUnsplashCover = svc.saveUnsplashCover.bind(svc); @@ -188,33 +189,37 @@ describe('unsplashService.getUnsplashKey', () => { }); describe('unsplashService.saveUnsplashCover', () => { - const dir = path.join(os.tmpdir(), 'trek-unsplash-cover-test'); - afterEach(() => { try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); + const coversDir = path.join(coverFx.root, 'covers'); + const writtenCovers = () => (fs.existsSync(coversDir) ? fs.readdirSync(coversDir) : []); + afterEach(() => { try { fs.rmSync(coversDir, { recursive: true, force: true }); } catch { /* ignore */ } }); it('UNSPLASH-005: rejects a non-Unsplash host before any fetch', async () => { - await expect(saveUnsplashCover('https://evil.example.com/x.jpg', dir)).rejects.toThrow('Not an Unsplash image URL'); + await expect(saveUnsplashCover('https://evil.example.com/x.jpg')).rejects.toThrow('Not an Unsplash image URL'); expect(safeFetch).not.toHaveBeenCalled(); }); - it('UNSPLASH-006: downloads an Unsplash image and writes it locally', async () => { + it('UNSPLASH-006: downloads an Unsplash image and stores it under covers', async () => { safeFetch.mockResolvedValue(fakeRes({ ok: true, type: 'image/jpeg', bytes: 1234 })); - const filename = await saveUnsplashCover('https://images.unsplash.com/photo-1?w=1080', dir); + const filename = await saveUnsplashCover('https://images.unsplash.com/photo-1?w=1080'); expect(filename).toMatch(/\.jpg$/); - expect(fs.existsSync(path.join(dir, filename))).toBe(true); + expect(fs.existsSync(path.join(coversDir, filename))).toBe(true); }); - it('UNSPLASH-007: rejects an unsupported content type', async () => { + it('UNSPLASH-007: rejects an unsupported content type without writing', async () => { safeFetch.mockResolvedValue(fakeRes({ ok: true, type: 'text/html' })); - await expect(saveUnsplashCover('https://images.unsplash.com/photo-1', dir)).rejects.toThrow(/Unsupported cover image type/); + await expect(saveUnsplashCover('https://images.unsplash.com/photo-1')).rejects.toThrow(/Unsupported cover image type/); + expect(writtenCovers()).toEqual([]); }); - it('UNSPLASH-008: rejects an oversized image', async () => { + it('UNSPLASH-008: rejects an oversized image without writing', async () => { safeFetch.mockResolvedValue(fakeRes({ ok: true, type: 'image/png', bytes: 16 * 1024 * 1024 })); - await expect(saveUnsplashCover('https://images.unsplash.com/photo-1', dir)).rejects.toThrow('Cover image too large'); + await expect(saveUnsplashCover('https://images.unsplash.com/photo-1')).rejects.toThrow('Cover image too large'); + expect(writtenCovers()).toEqual([]); }); it('UNSPLASH-009: throws when the download fails', async () => { safeFetch.mockResolvedValue(fakeRes({ ok: false, status: 404 })); - await expect(saveUnsplashCover('https://images.unsplash.com/photo-1', dir)).rejects.toThrow(/HTTP 404/); + await expect(saveUnsplashCover('https://images.unsplash.com/photo-1')).rejects.toThrow(/HTTP 404/); + expect(writtenCovers()).toEqual([]); }); }); diff --git a/server/tests/unit/nest/uploads-root.test.ts b/server/tests/unit/nest/uploads-root.test.ts deleted file mode 100644 index 769135be1..000000000 --- a/server/tests/unit/nest/uploads-root.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import path from 'node:path'; -import { UPLOADS_ROOT } from '../../../src/nest/memories/uploads-root'; - -/** - * The anchor three separate `__dirname` joins used to duplicate. A wrong depth - * does not throw — multer and mkdirSync would happily create the wrong tree and - * every stored photo would land somewhere nobody serves from. - */ -describe('UPLOADS_ROOT', () => { - it('UPLOADS-ROOT-001: resolves to /uploads, whatever the process cwd is', () => { - // tests run from server/, so the repo-relative answer is unambiguous - expect(UPLOADS_ROOT).toBe(path.resolve(process.cwd(), 'uploads')); - expect(path.basename(UPLOADS_ROOT)).toBe('uploads'); - expect(path.basename(path.dirname(UPLOADS_ROOT))).toBe('server'); - }); -}); diff --git a/server/tests/unit/nest/user-profile.service.test.ts b/server/tests/unit/nest/user-profile.service.test.ts index 7025fca8d..a8152f0eb 100644 --- a/server/tests/unit/nest/user-profile.service.test.ts +++ b/server/tests/unit/nest/user-profile.service.test.ts @@ -38,11 +38,15 @@ import { createTables } from '../../../src/db/schema'; import { runMigrations } from '../../../src/db/migrations'; import { resetTestDb } from '../../helpers/test-db'; import { createUser, createAdmin } from '../../helpers/factories'; +import fs from 'node:fs'; +import path from 'node:path'; import { UserProfileService } from '../../../src/nest/auth/user-profile.service'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; import { DatabaseService } from '../../../src/nest/database/database.service'; import { SEARCH_TEXT_FIELD_MASK } from '../../../src/nest/maps/maps.helpers'; -const profile = new UserProfileService(new DatabaseService(testDb)); +const avatarsFx = makeStorageFixture('avatars/'); +const profile = new UserProfileService(new DatabaseService(testDb), avatarsFx.storage); beforeAll(() => { createTables(testDb); @@ -326,6 +330,40 @@ describe('updateMapsKey / avatar', () => { const row = testDb.prepare('SELECT avatar FROM users WHERE id = ?').get(user.id) as { avatar: string | null }; expect(row.avatar).toBeNull(); }); + + function writeAvatar(name: string): string { + const dir = path.join(avatarsFx.root, 'avatars'); + fs.mkdirSync(dir, { recursive: true }); + const fp = path.join(dir, name); + fs.writeFileSync(fp, 'png-bytes'); + return fp; + } + + it('AUTH-DB-069b: saveAvatar reclaims the previous uploaded avatar file', async () => { + const { user } = createUser(testDb); + const old = writeAvatar('old.png'); + testDb.prepare('UPDATE users SET avatar = ? WHERE id = ?').run('old.png', user.id); + + await profile.saveAvatar(user.id, 'new.png'); + expect(fs.existsSync(old)).toBe(false); + }); + + it('AUTH-DB-070b: deleteAvatar removes the uploaded avatar file', async () => { + const { user } = createUser(testDb); + const fp = writeAvatar('mine.png'); + testDb.prepare('UPDATE users SET avatar = ? WHERE id = ?').run('mine.png', user.id); + + expect(await profile.deleteAvatar(user.id)).toEqual({ success: true }); + expect(fs.existsSync(fp)).toBe(false); + }); + + it('AUTH-DB-070c: a hostile stored avatar value is swallowed, never thrown', async () => { + const { user } = createUser(testDb); + testDb.prepare('UPDATE users SET avatar = ? WHERE id = ?').run('../../etc/passwd', user.id); + // Central key validation rejects the value; the delete swallows it exactly + // like the old rm().catch did — the DB update still wins. + expect(await profile.deleteAvatar(user.id)).toEqual({ success: true }); + }); }); describe('profile quirk fixes', () => { diff --git a/server/tests/unit/places/places.rpc.test.ts b/server/tests/unit/places/places.rpc.test.ts index 95ec337c5..08d54ad32 100644 --- a/server/tests/unit/places/places.rpc.test.ts +++ b/server/tests/unit/places/places.rpc.test.ts @@ -32,11 +32,15 @@ const req = (method: string, params: Record = {}): RpcRequest = * signature in front of the mock, and the cases below read .mock off these methods. */ function build(opts: { canEdit?: boolean; journeyThrows?: boolean } = {}) { + // update/remove are async in production (they may delete a storage object), so the + // doubles return promises: an un-awaited call in the RPC handler would compare a + // Promise against null/false instead of the resolved value, and these mocks pin + // that regression down rather than papering over it with a synchronous return. const places = { create: vi.fn((_t: string, i: Record) => ({ id: 10, ...i })), - update: vi.fn((_t: string, id: string) => (id === '7' ? { id: 7, name: 'updated' } : null)), + update: vi.fn((_t: string, id: string) => Promise.resolve(id === '7' ? { id: 7, name: 'updated' } : null)), get: vi.fn((_t: string, id: string) => (id === '7' ? { id: 7 } : undefined)), - remove: vi.fn((_t: string, id: string) => id === '7'), + remove: vi.fn((_t: string, id: string) => Promise.resolve(id === '7')), linkedExpenseIds: vi.fn(() => []), }; const journey = { @@ -147,4 +151,27 @@ describe('PlacesRpc', () => { it('PLACES-RPC-010 the class is listed in its module providers', () => { expectRegisteredProvider(PlacesModule, PlacesRpc); }); + + it('PLACES-RPC-011 an update broadcasts the resolved place, not the pending promise', async () => { + const f = build(); + const host = f.host('db:write:places'); + const res = await host.dispatch(req('places.update', { tripId: 1, placeId: 7, input: { name: 'x' } }), 42); + expect(res.ok).toBe(true); + expect((res as { result: unknown }).result).toEqual({ id: 7, name: 'updated' }); + // Not a Promise instance and not [object Promise] — the broadcast payload is the + // awaited place, exactly what a client-side listener would expect to render. + expect(f.realtime.broadcast).toHaveBeenCalledWith(1, 'place:updated', { place: { id: 7, name: 'updated' } }); + }); + + it('PLACES-RPC-012 a delete that the service refuses is ForbiddenResource, not {deleted: true}', async () => { + const f = build(); + // place.get finds it (it exists), but the actual delete comes back false — the + // race the null/false-vs-Promise bug used to paper over by always being truthy. + f.places.remove.mockImplementationOnce(() => Promise.resolve(false)); + const res = (await f.host('db:write:places').dispatch(req('places.delete', { tripId: 1, placeId: 7 }), 42)) as RpcError; + expect(res.ok).toBe(false); + expect(res.error.code).toBe('RESOURCE_FORBIDDEN'); + expect(res.error.message).toBe('no place 7 on trip 1'); + expect(f.realtime.broadcast).not.toHaveBeenCalled(); + }); }); diff --git a/server/tests/unit/plugins/addon-domains-errors.rpc.test.ts b/server/tests/unit/plugins/addon-domains-errors.rpc.test.ts index b2fb6d97d..9b11a22ba 100644 --- a/server/tests/unit/plugins/addon-domains-errors.rpc.test.ts +++ b/server/tests/unit/plugins/addon-domains-errors.rpc.test.ts @@ -52,7 +52,11 @@ function build(overrides: { new AtlasRpc({ listVisitedCountries: vi.fn(() => []), listManuallyVisitedRegions: vi.fn(() => []), bucketList: vi.fn(() => []), markCountry: vi.fn(), unmarkCountry: vi.fn(), markRegion: vi.fn(), unmarkRegion: vi.fn(), createBucketItem: vi.fn(() => ({})), deleteBucketItem: vi.fn(() => true), ...overrides.atlas } as never, guards), new VacayRpc({ getPlanData: vi.fn(() => ({})), getActivePlanId: vi.fn(() => 1), toggleEntry: vi.fn(() => ({})), toggleCompanyHoliday: vi.fn(() => ({})), ...overrides.vacay } as never, guards), new JournalRpc({ listJourneys: vi.fn(() => []), listEntries: vi.fn(() => []), createEntry: vi.fn(() => ({})), updateEntry: vi.fn(() => ({})), deleteEntry: vi.fn(() => true), createJourney: vi.fn(() => ({})), deleteJourney: vi.fn(() => true), ...overrides.journey } as never, guards), - new CollectionsRpc({ listCollections: vi.fn(() => []), getCollection: vi.fn(() => ({})), createCollection: vi.fn(() => ({})), updateCollection: vi.fn(() => ({})), savePlace: vi.fn(() => ({})), copyToTrip: vi.fn(() => ({})), deletePlace: vi.fn(() => undefined), ...overrides.collections } as never, guards), + // deletePlace is async in production, so its double returns a promise: an + // un-awaited call in the RPC handler would let a rejection sail past + // mapCollectionError as an unhandled rejection while the handler still replied + // {deleted: true}. + new CollectionsRpc({ listCollections: vi.fn(() => []), getCollection: vi.fn(() => ({})), createCollection: vi.fn(() => ({})), updateCollection: vi.fn(() => ({})), savePlace: vi.fn(() => ({})), copyToTrip: vi.fn(() => ({})), deletePlace: vi.fn(() => Promise.resolve(undefined)), ...overrides.collections } as never, guards), ]); const granted = new Set([ 'db:read:collab', 'db:write:collab', 'db:read:journal', 'db:write:journal', @@ -194,6 +198,16 @@ describe('addon-gated domains validate their input', () => { expect(err(await host.dispatch(req('collections.deletePlace', { placeId: 1 }), 42))).toMatchObject({ code: 'HOST_ERROR', message: 'disk on fire' }); }); + it('ADDONERR-009b a refused collections delete is ForbiddenResource, not {deleted: true}', async () => { + // deletePlace is async, so a status-tagged rejection has to be awaited before the + // handler decides what to reply — an un-awaited call would let this rejection + // become an unhandled promise rejection while the RPC still answered {deleted: true}. + const host = build({ collections: { deletePlace: vi.fn(() => Promise.reject(statusError(403, 'not your collection'))) } }); + const res = await host.dispatch(req('collections.deletePlace', { placeId: 1 }), 42); + expect(res.ok).toBe(false); + expect(err(res)).toMatchObject({ code: 'RESOURCE_FORBIDDEN', message: 'not your collection' }); + }); + it('ADDONERR-010 collections reject a payload the shared schema refuses', async () => { const host = build(); expect(err(await host.dispatch(req('collections.create', { input: {} }), 42)).code).toBe('BAD_PARAMS'); diff --git a/server/tests/unit/plugins/addon-gates.rpc.test.ts b/server/tests/unit/plugins/addon-gates.rpc.test.ts index 136af4791..403405758 100644 --- a/server/tests/unit/plugins/addon-gates.rpc.test.ts +++ b/server/tests/unit/plugins/addon-gates.rpc.test.ts @@ -39,6 +39,11 @@ function spyService(calls: string[], name: string) { if (prop === 'votePoll' || prop === 'createMessage') return { error: null, poll: {}, message: {} }; if (prop === 'listEntries') return []; if (prop === 'getActivePlanId') return 1; + // collections.deletePlace is async in production (it deletes a storage + // object); a promise-returning double here pins the fix that awaits it — + // an un-awaited call would leave an unhandled rejection unnoticed by every + // assertion below. + if (name === 'collections' && prop === 'deletePlace') return Promise.resolve(undefined); if (String(prop).startsWith('delete')) return true; if (String(prop).startsWith('list')) return []; return { id: 1 }; diff --git a/server/tests/unit/plugins/rpc-fallbacks.test.ts b/server/tests/unit/plugins/rpc-fallbacks.test.ts index 19bc9ebfa..6caa8503d 100644 --- a/server/tests/unit/plugins/rpc-fallbacks.test.ts +++ b/server/tests/unit/plugins/rpc-fallbacks.test.ts @@ -80,7 +80,7 @@ describe('optional fields fall back rather than reaching the service as undefine const { guards, db } = guardsFor(); const createFileLink = vi.fn(() => []); const files = { getFileById: vi.fn(() => ({})), findForeignLinkTarget: vi.fn(() => null), createFileLink } as never; - const host = new PluginRpcHost('p', ALL, makeDeps(), createTestPluginRegistry([new FilesRpc(files, realtime(), db, guards)])); + const host = new PluginRpcHost('p', ALL, makeDeps(), createTestPluginRegistry([new FilesRpc(files, realtime(), db, guards, {} as never)])); await host.dispatch(req('files.createLink', { tripId: 1, fileId: 2, opts: {} }), 42); expect(createFileLink).toHaveBeenCalledWith(2, { reservation_id: null, assignment_id: null, place_id: null }); }); @@ -89,7 +89,7 @@ describe('optional fields fall back rather than reaching the service as undefine const { guards, db } = guardsFor(); const updateFile = vi.fn((id: number) => ({ id })); const files = { getFileById: vi.fn(() => ({})), findForeignLinkTarget: vi.fn(() => null), updateFile } as never; - const host = new PluginRpcHost('p', ALL, makeDeps(), createTestPluginRegistry([new FilesRpc(files, realtime(), db, guards)])); + const host = new PluginRpcHost('p', ALL, makeDeps(), createTestPluginRegistry([new FilesRpc(files, realtime(), db, guards, {} as never)])); await host.dispatch(req('files.update', { tripId: 1, fileId: 2, input: { reservation_id: null, place_id: 7 } }), 42); expect(updateFile).toHaveBeenLastCalledWith(2, expect.anything(), expect.objectContaining({ reservation_id: null, place_id: '7' })); }); @@ -107,7 +107,10 @@ describe('optional fields fall back rather than reaching the service as undefine describe('a service that says no becomes a refusal, not a crash', () => { it('FALLBACK-006 a place that will not delete is refused', async () => { const { guards } = guardsFor(); - const places = { get: vi.fn(() => ({ id: 7 })), remove: vi.fn(() => false), create: vi.fn(), update: vi.fn(), linkedExpenseIds: vi.fn(() => []) } as never; + // remove is async in production; a Promise-returning double pins the fix that + // awaits it — an un-awaited call compares a Promise (always truthy) against + // falsy and never refuses. + const places = { get: vi.fn(() => ({ id: 7 })), remove: vi.fn(() => Promise.resolve(false)), create: vi.fn(), update: vi.fn(), linkedExpenseIds: vi.fn(() => []) } as never; const journey = { onPlaceDeleted: vi.fn(), onPlaceCreated: vi.fn(), onPlaceUpdated: vi.fn() } as never; const host = new PluginRpcHost('p', ALL, makeDeps(), createTestPluginRegistry([new PlacesRpc(places, journey, realtime(), guards)])); expect(err(await host.dispatch(req('places.delete', { tripId: 1, placeId: 7 }), 42)).message).toBe('no place 7 on trip 1'); diff --git a/server/tests/unit/services/apiKeyCrypto.test.ts b/server/tests/unit/services/apiKeyCrypto.test.ts index 8434ffb57..0435f772f 100644 --- a/server/tests/unit/services/apiKeyCrypto.test.ts +++ b/server/tests/unit/services/apiKeyCrypto.test.ts @@ -7,7 +7,12 @@ vi.mock('../../../src/config', () => ({ updateJwtSecret: () => {}, })); -import { encrypt_api_key, decrypt_api_key, maybe_encrypt_api_key } from '../../../src/nest/common/crypto/apiKeyCrypto'; +import { + encrypt_api_key, + decrypt_api_key, + maybe_encrypt_api_key, + is_encrypted_api_key, +} from '../../../src/nest/common/crypto/apiKeyCrypto'; describe('apiKeyCrypto', () => { const PLAINTEXT_KEY = 'my-secret-api-key-12345'; @@ -76,4 +81,15 @@ describe('apiKeyCrypto', () => { expect(result).toBe(encrypted); }); }); + + describe('is_encrypted_api_key', () => { + it('is true exactly for enc:v1: envelopes', () => { + expect(is_encrypted_api_key(encrypt_api_key('s'))).toBe(true); + expect(is_encrypted_api_key('enc:v1:AAAA')).toBe(true); + expect(is_encrypted_api_key('plaintext')).toBe(false); + expect(is_encrypted_api_key('')).toBe(false); + expect(is_encrypted_api_key(null)).toBe(false); + expect(is_encrypted_api_key(42)).toBe(false); + }); + }); }); diff --git a/server/tests/unit/services/conflictUpdate.test.ts b/server/tests/unit/services/conflictUpdate.test.ts index c318c8975..d3589e311 100644 --- a/server/tests/unit/services/conflictUpdate.test.ts +++ b/server/tests/unit/services/conflictUpdate.test.ts @@ -52,13 +52,14 @@ import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.s import { TrekPhotosRepository } from '../../../src/nest/photos/trek-photos.repository'; import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service'; import { notificationsStub } from '../../helpers/notifications'; +import { makeStorageFixture } from '../../helpers/storage-fixture'; const dbs = new DatabaseService(testDb); const realtime = new RealtimeService(); const runtimeEnv = new RuntimeEnvService(); // One cache instance shared by maps and places, the way the container wires it: // the service's stampede guard only works if there is exactly one of them. -const photoCache = new PlacePhotoCacheService(dbs, runtimeEnv); +const photoCache = new PlacePhotoCacheService(dbs, makeStorageFixture('photos/google/').storage); const packing = new PackingService(dbs, new PermissionsService(dbs), realtime, notificationsStub()); const places = new PlacesService( @@ -67,9 +68,10 @@ const places = new PlacesService( realtime, new MapsService(dbs, photoCache), new QueryHelpersService(dbs), - new UnsplashService(dbs, runtimeEnv), + new UnsplashService(dbs, runtimeEnv, makeStorageFixture('').storage), photoCache, new JourneyDomainService(dbs, realtime, new TrekPhotosRepository(dbs)), + makeStorageFixture('').storage, ); beforeAll(() => { @@ -91,32 +93,32 @@ function freshPlace(tripId: number) { } describe('PlacesService.update — optimistic concurrency', () => { - it('updates normally when no If-Match token is sent (back-compat)', () => { + it('updates normally when no If-Match token is sent (back-compat)', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = freshPlace(trip.id); - const result = places.update(String(trip.id), String(place.id), { name: 'Edited' }); + const result = await places.update(String(trip.id), String(place.id), { name: 'Edited' }); expect(isUpdateConflict(result)).toBe(false); expect((result as { name: string }).name).toBe('Edited'); }); - it('updates when the If-Match token matches the current updated_at', () => { + it('updates when the If-Match token matches the current updated_at', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = freshPlace(trip.id); - const result = places.update(String(trip.id), String(place.id), { name: 'Edited' }, place.updated_at); + const result = await places.update(String(trip.id), String(place.id), { name: 'Edited' }, place.updated_at); expect(isUpdateConflict(result)).toBe(false); expect((result as { name: string }).name).toBe('Edited'); }); - it('returns a conflict (with the server row) when the token is stale', () => { + it('returns a conflict (with the server row) when the token is stale', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); const place = freshPlace(trip.id); - const result = places.update(String(trip.id), String(place.id), { name: 'Mine' }, '1999-01-01 00:00:00'); + const result = await places.update(String(trip.id), String(place.id), { name: 'Mine' }, '1999-01-01 00:00:00'); expect(isUpdateConflict(result)).toBe(true); if (isUpdateConflict(result)) { expect((result.server as { name: string }).name).toBe('Original'); @@ -126,10 +128,10 @@ describe('PlacesService.update — optimistic concurrency', () => { expect(row.name).toBe('Original'); }); - it('returns null for a place that does not exist', () => { + it('returns null for a place that does not exist', async () => { const { user } = createUser(testDb); const trip = createTrip(testDb, user.id); - expect(places.update(String(trip.id), '999999', { name: 'x' }, 'whatever')).toBeNull(); + expect(await places.update(String(trip.id), '999999', { name: 'x' }, 'whatever')).toBeNull(); }); }); diff --git a/server/tests/unit/uploads-dirs.test.ts b/server/tests/unit/uploads-dirs.test.ts index 691b36b2a..1f5beab16 100644 --- a/server/tests/unit/uploads-dirs.test.ts +++ b/server/tests/unit/uploads-dirs.test.ts @@ -1,37 +1,46 @@ import { describe, it, expect } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; +import { CATEGORY_PREFIXES } from '../../src/nest/storage/storage-registry.service'; -// The set of uploads subdirectories is hand-mirrored between the image build and -// the boot-time mkdir in server/src/index.ts. Drift is invisible until a user on -// a host whose bind-mounted uploads dir isn't writable by node hits an EACCES on -// first upload to the missing dir (#1762), so pin the two copies together. +// Driver-owned roots are created by LocalDriver.init on every storage-registry +// load now (boot and reload()) — index.ts no longer mkdirs the uploads tree. +// The Dockerfile still pre-creates the subdirs (and chowns them) at image +// build, so its `mkdir -p` list must stay in sync with the category prefixes +// rooted under uploads/. Drift is invisible until a user on a host whose +// bind-mounted uploads dir isn't writable by node hits an EACCES on first +// upload to the missing dir (#1762), so pin the two lists together. const repoRoot = path.resolve(__dirname, '../../..'); -const readSubdirs = (file: string, pattern: RegExp): Set => { - const source = fs.readFileSync(path.join(repoRoot, file), 'utf8'); +const readDockerfileSubdirs = (): Set => { + const source = fs.readFileSync(path.join(repoRoot, 'Dockerfile'), 'utf8'); const found = new Set(); - for (const m of source.matchAll(pattern)) found.add(m[1]); + for (const m of source.matchAll(/\/app\/uploads\/([A-Za-z0-9_-]+)/g)) found.add(m[1]); return found; }; describe('uploads subdirectory parity', () => { - const fromDockerfile = readSubdirs('Dockerfile', /\/app\/uploads\/([A-Za-z0-9_-]+)/g); - const fromBoot = readSubdirs('server/src/index.ts', /path\.join\(uploadsDir, '([A-Za-z0-9_-]+)'\)/g); + const fromDockerfile = readDockerfileSubdirs(); + // The registry's real values: every non-empty category prefix lives under + // uploads/ (backups' empty prefix is rooted in data/, not uploads/), and the + // nested cache prefixes (photos/google, photos/trek) collapse into their + // top-level dir. + const fromRegistry = new Set( + Object.values(CATEGORY_PREFIXES) + .filter((p) => p !== '') + .map((p) => p.split('/')[0]), + ); - it('finds both lists (guards against the regexes silently matching nothing)', () => { + it('finds the Dockerfile list (guards against the regex silently matching nothing)', () => { expect(fromDockerfile.size).toBeGreaterThan(0); - expect(fromBoot.size).toBeGreaterThan(0); }); - it('creates the same uploads subdirs in the Dockerfile and at boot', () => { - expect([...fromBoot].sort()).toEqual([...fromDockerfile].sort()); + it('creates the same uploads subdirs in the Dockerfile and at registry init', () => { + expect([...fromRegistry].sort()).toEqual([...fromDockerfile].sort()); }); it('covers every uploads subdir the server writes to', () => { - // journey/ and places/ are written by journey.controller.ts and - // services/placeImage.ts; the rest are the long-standing upload stores. - expect([...fromBoot].sort()).toEqual([ + expect([...fromRegistry].sort()).toEqual([ 'avatars', 'covers', 'files', diff --git a/server/vitest.config.ts b/server/vitest.config.ts index 3c1f3e246..40fc602ee 100644 --- a/server/vitest.config.ts +++ b/server/vitest.config.ts @@ -70,13 +70,13 @@ export default defineConfig({ 'src/nest/atlas/**/*.ts': { statements: 92, branches: 82, functions: 96, lines: 94 }, 'src/nest/audit/**/*.ts': { statements: 93, branches: 80, functions: 99, lines: 96 }, 'src/nest/auth/**/*.ts': { statements: 93, branches: 85, functions: 95, lines: 96 }, - 'src/nest/backup/**/*.ts': { statements: 96, branches: 92, functions: 99, lines: 98 }, + 'src/nest/backup/**/*.ts': { statements: 97, branches: 93, functions: 99, lines: 98 }, 'src/nest/booking-import/**/*.ts': { statements: 50, branches: 28, functions: 64, lines: 53 }, 'src/nest/budget/**/*.ts': { statements: 91, branches: 77, functions: 97, lines: 95 }, 'src/nest/calendar/**/*.ts': { statements: 96, branches: 93, functions: 99, lines: 99 }, 'src/nest/categories/**/*.ts': { statements: 99, branches: 80, functions: 99, lines: 99 }, 'src/nest/collab/**/*.ts': { statements: 92, branches: 85, functions: 95, lines: 97 }, - 'src/nest/collections/**/*.ts': { statements: 83, branches: 70, functions: 92, lines: 92 }, + 'src/nest/collections/**/*.ts': { statements: 86, branches: 75, functions: 95, lines: 95 }, 'src/nest/common/**/*.ts': { statements: 95, branches: 89, functions: 99, lines: 97 }, 'src/nest/config/**/*.ts': { statements: 99, branches: 100, functions: 99, lines: 99 }, 'src/nest/database/**/*.ts': { statements: 99, branches: 100, functions: 99, lines: 99 }, @@ -91,19 +91,19 @@ export default defineConfig({ 'src/nest/journey/**/*.ts': { statements: 91, branches: 84, functions: 88, lines: 93 }, 'src/nest/llm-parse/**/*.ts': { statements: 91, branches: 85, functions: 85, lines: 94 }, 'src/nest/maps/**/*.ts': { statements: 93, branches: 86, functions: 97, lines: 96 }, - 'src/nest/memories/**/*.ts': { statements: 88, branches: 81, functions: 97, lines: 90 }, + 'src/nest/memories/**/*.ts': { statements: 92, branches: 83, functions: 97, lines: 94 }, 'src/nest/notifications/**/*.ts': { statements: 83, branches: 72, functions: 85, lines: 87 }, 'src/nest/oauth/**/*.ts': { statements: 96, branches: 95, functions: 97, lines: 97 }, 'src/nest/oidc/**/*.ts': { statements: 88, branches: 83, functions: 91, lines: 92 }, 'src/nest/packing/**/*.ts': { statements: 92, branches: 83, functions: 99, lines: 97 }, 'src/nest/permissions/**/*.ts': { statements: 97, branches: 92, functions: 99, lines: 97 }, - 'src/nest/photos/**/*.ts': { statements: 95, branches: 92, functions: 89, lines: 95 }, + 'src/nest/photos/**/*.ts': { statements: 96, branches: 92, functions: 92, lines: 96 }, // Landed on dev without an entry, so it sat on the 80 catch-all while // measuring 99.5/99.0/97.4/100 — the gap the per-domain ratchet exists to // close. Set with a few points of slack against the Linux/Windows drift. 'src/nest/place-enrichment/**/*.ts': { statements: 96, branches: 95, functions: 94, lines: 97 }, - 'src/nest/place-photos/**/*.ts': { statements: 79, branches: 70, functions: 72, lines: 80 }, - 'src/nest/places/**/*.ts': { statements: 90, branches: 80, functions: 96, lines: 94 }, + 'src/nest/place-photos/**/*.ts': { statements: 87, branches: 79, functions: 72, lines: 89 }, + 'src/nest/places/**/*.ts': { statements: 91, branches: 82, functions: 96, lines: 94 }, 'src/nest/platform/**/*.ts': { statements: 99, branches: 99, functions: 99, lines: 99 }, 'src/nest/plugins/**/*.ts': { statements: 86, branches: 81, functions: 78, lines: 89 }, 'src/nest/query-helpers/**/*.ts': { statements: 90, branches: 75, functions: 99, lines: 92 }, @@ -112,6 +112,7 @@ export default defineConfig({ 'src/nest/reservations/**/*.ts': { statements: 92, branches: 83, functions: 96, lines: 96 }, 'src/nest/settings/**/*.ts': { statements: 87, branches: 71, functions: 99, lines: 88 }, 'src/nest/share/**/*.ts': { statements: 97, branches: 87, functions: 99, lines: 99 }, + 'src/nest/storage/**/*.ts': { statements: 94, branches: 84, functions: 97, lines: 94 }, 'src/nest/system-notices/**/*.ts': { statements: 99, branches: 99, functions: 99, lines: 99 }, 'src/nest/tags/**/*.ts': { statements: 97, branches: 89, functions: 99, lines: 99 }, 'src/nest/todo/**/*.ts': { statements: 90, branches: 82, functions: 99, lines: 99 }, @@ -120,7 +121,7 @@ export default defineConfig({ 'src/nest/trip-membership/**/*.ts': { statements: 99, branches: 86, functions: 99, lines: 99 }, 'src/nest/trip-members/**/*.ts': { statements: 95, branches: 91, functions: 99, lines: 95 }, 'src/nest/trip-read-model/**/*.ts': { statements: 97, branches: 96, functions: 99, lines: 97 }, - 'src/nest/trips/**/*.ts': { statements: 92, branches: 82, functions: 91, lines: 94 }, + 'src/nest/trips/**/*.ts': { statements: 95, branches: 86, functions: 95, lines: 96 }, 'src/nest/unsplash/**/*.ts': { statements: 99, branches: 88, functions: 99, lines: 99 }, 'src/nest/vacay/**/*.ts': { statements: 82, branches: 66, functions: 90, lines: 86 }, 'src/nest/weather/**/*.ts': { statements: 93, branches: 78, functions: 91, lines: 97 }, @@ -136,7 +137,7 @@ export default defineConfig({ // // src/demo/** is deliberately absent: it measures 0%, and a floor of // zero asserts nothing. It needs tests before it needs a threshold. - 'src/app-config/**/*.ts': { statements: 99, branches: 93, functions: 99, lines: 99 }, + 'src/app-config/**/*.ts': { statements: 99, branches: 95, functions: 99, lines: 99 }, 'src/db/**/*.ts': { statements: 73, branches: 38, functions: 59, lines: 80 }, 'src/mcp/**/*.ts': { statements: 58, branches: 43, functions: 63, lines: 60 }, 'src/middleware/**/*.ts': { statements: 91, branches: 89, functions: 87, lines: 94 }, diff --git a/shared/src/i18n/ar/admin.ts b/shared/src/i18n/ar/admin.ts index 93633d932..08e5b34cc 100644 --- a/shared/src/i18n/ar/admin.ts +++ b/shared/src/i18n/ar/admin.ts @@ -185,6 +185,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'فشل الحفظ', 'admin.tabs.addons': 'الإضافات', 'admin.tabs.plugins': 'الإضافات الخارجية', + 'admin.tabs.storage': 'التخزين', 'admin.plugins.rescan': 'إعادة الفحص', 'admin.plugins.rescanned': 'تمت إعادة فحص مجلد الإضافات', 'admin.plugins.upload': 'رفع إضافة', diff --git a/shared/src/i18n/ar/externalNotifications.ts b/shared/src/i18n/ar/externalNotifications.ts index 52a22aa4e..44b1e091b 100644 --- a/shared/src/i18n/ar/externalNotifications.ts +++ b/shared/src/i18n/ar/externalNotifications.ts @@ -52,6 +52,12 @@ const ar: NotificationLocale = { title: 'إصدار TREK جديد متاح', body: `TREK ${p.version} متاح الآن. تفضل بزيارة لوحة الإدارة للتحديث.`, }), + replica_failure: (p) => ({ + title: 'فشل النسخة المتماثلة للتخزين', + body: + `فشلت الكتابة على النسخة المتماثلة '${p.backend}': ${p.op} لـ ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` تم تجاهل ${p.suppressed} فشل إضافي منذ آخر إشعار.` : ''), + }), synology_session_cleared: () => ({ title: 'تمت إعادة تعيين جلسة Synology', body: 'تغيّر حسابك أو رابط Synology. تم تسجيل خروجك من Synology Photos.', diff --git a/shared/src/i18n/ar/index.ts b/shared/src/i18n/ar/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/ar/index.ts +++ b/shared/src/i18n/ar/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/ar/notif.ts b/shared/src/i18n/ar/notif.ts index eef17db83..bb101fbe5 100644 --- a/shared/src/i18n/ar/notif.ts +++ b/shared/src/i18n/ar/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} عيّنك في {category} في {trip}', 'notif.version_available.title': 'إصدار جديد متاح', 'notif.version_available.text': 'TREK {version} متاح الآن', + 'notif.replica_failure.title': 'فشل النسخة المتماثلة للتخزين', + 'notif.replica_failure.text': "فشلت الكتابة على النسخة المتماثلة '{backend}': {op} لـ {key} — {error}", + 'notif.replica_failure.textSuppressed': "فشلت الكتابة على النسخة المتماثلة '{backend}': {op} لـ {key} — {error}. تم تجاهل {suppressed} فشل إضافي منذ آخر إشعار.", 'notif.action.view_trip': 'عرض الرحلة', 'notif.action.view_collab': 'عرض الرسائل', 'notif.action.view_packing': 'عرض التعبئة', diff --git a/shared/src/i18n/ar/storage.ts b/shared/src/i18n/ar/storage.ts new file mode 100644 index 000000000..0922a84c4 --- /dev/null +++ b/shared/src/i18n/ar/storage.ts @@ -0,0 +1,137 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'الدليل الجذري', + 'storage.help.root': 'المسار المطلق على الخادم حيث يخزّن هذا الواجهة الخلفية كائناته.', + 'storage.field.endpoint': 'عنوان URL لنقطة النهاية', + 'storage.help.endpoint': + 'عنوان URL الأساسي للخدمة المتوافقة مع S3، مثل https://s3.example.com أو http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'معرّف مفتاح الوصول', + 'storage.field.secretAccessKey': 'مفتاح الوصول السري', + 'storage.field.region': 'المنطقة', + 'storage.help.region': 'احتفظ بالقيمة الافتراضية ما لم يتطلب مزوّد الخدمة لديك منطقة معينة.', + 'storage.field.keyPrefix': 'بادئة المفتاح', + 'storage.help.keyPrefix': 'بادئة اختيارية تُضاف إلى كل مفتاح كائن، مثل trek/prod.', + 'storage.field.retries': 'عدد المحاولات', + 'storage.field.timeoutMs': 'مهلة الانتظار (بالمللي ثانية)', + 'storage.field.primary': 'الواجهة الخلفية الأساسية', + 'storage.field.replicas': 'النسخ المتماثلة', + 'storage.title': 'التخزين', + 'storage.description': 'المكان الذي يحفظ فيه TREK الملفات والصور والنسخ الاحتياطية المرفوعة. لا يتغيّر شيء حتى تحفظ.', + 'storage.loading': 'جارٍ التحميل…', + 'storage.saved': 'تم حفظ إعدادات التخزين', + 'storage.save': 'حفظ التغييرات', + 'storage.unsaved': 'تغييرات غير محفوظة', + 'storage.saveConflict': + 'تغيّرت إعدادات التخزين منذ تحميلها، لذلك لم يتم حفظ تغييراتك. تجاهلها وأعد تحميل الإعدادات المحفوظة للبدء من جديد.', + 'storage.discardAndReload': 'تجاهل تغييراتي وأعد التحميل', + 'storage.configError.banner': 'فشل تحميل إعدادات التخزين المخزنة — سيؤدي الحفظ إلى استبدالها: {error}', + 'storage.backends.title': 'الواجهات الخلفية', + 'storage.backends.add': 'إضافة واجهة خلفية', + 'storage.backends.usedBy': 'مستخدَمة بواسطة: {categories}', + 'storage.backends.unused': 'غير مخصّصة لأي فئة', + 'storage.backends.envReadOnly': 'محدَّدة بواسطة متغيّر بيئة — للقراءة فقط', + 'storage.source.built-in': 'مدمجة', + 'storage.source.env': 'البيئة', + 'storage.source.settings': 'الإعدادات', + 'storage.type.local': 'محلي', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'مرآة', + 'storage.actions.test': 'اختبار', + 'storage.actions.edit': 'تعديل', + 'storage.actions.remove': 'إزالة', + 'storage.test.running': 'جارٍ الاختبار…', + 'storage.test.ok': 'الاتصال ناجح', + 'storage.test.failed': 'فشل الاختبار', + 'storage.remove.title': 'إزالة الواجهة الخلفية', + 'storage.remove.body': 'هل تريد إزالة {name} من الإعدادات؟ يرفض الخادم الحفظ إذا كان لا يزال هناك ما يعتمد عليها.', + 'storage.remove.stillAssigned': 'لا تزال مخصّصة لـ: {categories}', + 'storage.form.addTitle': 'إضافة واجهة خلفية', + 'storage.form.editTitle': 'تعديل الواجهة الخلفية', + 'storage.form.name': 'الاسم', + 'storage.form.type': 'النوع', + 'storage.form.apply': 'تطبيق', + 'storage.form.cancel': 'إلغاء', + 'storage.form.duplicateName': 'توجد بالفعل واجهة خلفية باسم {name}', + 'storage.categories.title': 'الفئات', + 'storage.categories.default': 'الافتراضية', + 'storage.categories.reassignWarning': + 'لا يتم نقل الكائنات الحالية: تذهب الكائنات الجديدة إلى الواجهة الخلفية المخصّصة حديثًا، وتبقى القديمة في مكانها.', + 'storage.category.files': 'مستندات الرحلة', + 'storage.category.journey': 'صور Journey', + 'storage.category.covers': 'صور الغلاف', + 'storage.category.avatars': 'الصور الشخصية', + 'storage.category.places': 'صور الأماكن', + 'storage.category.photos-google': 'ذاكرة تخزين مؤقت لصور Google', + 'storage.category.photos-trek': 'ذاكرة تخزين مؤقت لصور TREK', + 'storage.category.backups': 'النسخ الاحتياطية', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'مرفقات الملفات التي تُرفع إلى الرحلات — التذاكر وملفات PDF وتأكيدات الحجز والملفات المُشارَكة في محادثة الرحلة.', + 'storage.categoryDesc.journey': 'الصور والصور المصغّرة المرفقة بمدخلات Journey.', + 'storage.categoryDesc.covers': 'صور غلاف الرحلات والمجموعات، بما في ذلك الأغلفة المجلوبة من Unsplash.', + 'storage.categoryDesc.avatars': 'الصور الشخصية لحسابات المستخدمين.', + 'storage.categoryDesc.places': 'الصور المرفقة بالأماكن وأماكن المجموعات — سواء تم رفعها أو استيرادها.', + 'storage.categoryDesc.photos-google': + 'نسخ مخزّنة مؤقتًا من صور Google Places — يمكن إعادة جلبها، ويُمكن فقدانها بأمان.', + 'storage.categoryDesc.photos-trek': + 'صور مخزّنة مؤقتًا من خدمة صور TREK التي تستخدمها ميزة الصور (Memories) — يمكن إعادة جلبها، ويُمكن فقدانها بأمان.', + 'storage.categoryDesc.backups': 'أرشيفات النسخ الاحتياطي للخادم التي تُنشئها لوحة النسخ الاحتياطي أو الجدولة.', + 'storage.health.title': 'الحالة', + 'storage.health.allClear': 'لم يتم تسجيل أي أعطال في النسخ المتماثلة.', + 'storage.health.seedFile': + 'يوجد ملف بذرة storage-config.json لكن يتم تجاهله — صفوف الإعدادات موجودة بالفعل. أدر التخزين من هنا.', + 'storage.health.failureLine': 'فشلت عملية {op} لـ {key} على {backend}: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'أهداف المرآة', + 'storage.mirror.targetsHelp': 'كل عملية كتابة على هذه الواجهة الخلفية تُنسخ أيضًا إلى كل هدف محدد.', + 'storage.mirror.latencyNote': + 'تُكتب النسخ المتماثلة واحدة تلو الأخرى أثناء كل عملية رفع — أي هدف بطيء أو يتعذّر الوصول إليه يُبطئ كل عملية رفع لكل فئة على هذه الواجهة الخلفية.', + 'storage.mirror.mirroredTo': 'معكوسة إلى: {targets}', + 'storage.mirror.replicaOf': 'نسخة من: {primaries}', + 'storage.mirror.cacheWarning': + 'غير مستحسن: تحتوي هذه الفئة على محتوى قابل لإعادة الجلب — نسخه عادةً ما يكون مضيعة للموارد.', + 'storage.mirror.degenerate.duplicate-mirror': + 'مرآة ثانية تُغلّف {primary} — تُدير اللوحة الأولى فقط؛ أزل هذه لإدارة المرآة من {primary}.', + 'storage.mirror.degenerate.env-primary': 'تُغلّف واجهة خلفية معرّفة بواسطة متغيّر بيئة — غير قابلة للتعديل هنا.', + 'storage.mirror.degenerate.missing-primary': 'تشير إلى واجهة خلفية لم تعد موجودة.', + 'storage.remove.usedAsReplicaBy': 'مستخدَمة كنسخة متماثلة بواسطة: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'مزامنة الآن', + 'storage.sync.running': 'جارٍ المزامنة… {done}/{total}', + 'storage.sync.counts': '{copied} منسوخة · {skipped} متخطاة · {failed} فاشلة', + 'storage.sync.cancel': 'إلغاء المزامنة', + 'storage.sync.done': 'انتهت المزامنة: تم نسخ {copied}، وحُذف {deleted}، وفشل {failed}', + 'storage.sync.cancelled': 'تم إلغاء المزامنة', + 'storage.sync.error': 'فشلت المزامنة: {error}', + 'storage.sync.prompt': 'الكائنات الحالية لم تُنسخ بعد — هل تريد المزامنة الآن؟', + 'storage.sync.dismiss': 'تجاهل', + 'storage.usage.line': '{objects} كائن · {size}', + 'storage.usage.computed': 'تم حساب الاستخدام {age}', + 'storage.usage.never': 'لم يُحسب الاستخدام بعد', + 'storage.usage.refresh': 'تحديث', + 'storage.usage.compute': 'احسب الآن', + 'storage.usage.legacyNote': 'يشمل مكتبة الصور القديمة', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'هل تريد نقل الكائنات الحالية إلى الواجهة الخلفية الجديدة؟', + 'storage.migrate.promptLine': '{category}: {objects} كائن ({size}) من {from} إلى {to}', + 'storage.migrate.promptLineUnknown': '{category}: حجم غير معروف (لم يُجرَ فحص استخدام بعد) من {from} إلى {to}', + 'storage.migrate.move': 'نقل الكائنات الحالية', + 'storage.migrate.routeOnly': 'توجيه الكتابات الجديدة فقط', + 'storage.migrate.running': 'جارٍ نقل {category}… {done}/{total}', + 'storage.migrate.done': 'اكتمل النقل: تم نسخ {copied}، وتخطي {skipped}', + 'storage.migrate.doneFailures': '{failed} فشل — لم يتم نسخ تلك الكائنات إلى الواجهة الخلفية الجديدة', + 'storage.migrate.failed': 'فشل النقل: {error} — لم يتم تبديل الفئة', + 'storage.migrate.cancelled': 'تم إلغاء النقل — لم يتم تبديل أي شيء', + 'storage.migrate.reclaimable': '{objects} كائن ({size}) لا تزال على {from} — استرجعها يدويًا', + 'storage.migrate.cancel': 'إلغاء النقل', + 'storage.migrate.promptCancel': 'إلغاء', + 'storage.migrate.queued': 'في قائمة الانتظار: {categories}', + 'storage.migrate.queueDropped': 'تعذّر بدء عملية النقل التالية — تم مسح قائمة الانتظار المتبقية: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/br/admin.ts b/shared/src/i18n/br/admin.ts index 942c05998..1213da371 100644 --- a/shared/src/i18n/br/admin.ts +++ b/shared/src/i18n/br/admin.ts @@ -164,6 +164,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Falha ao salvar', 'admin.tabs.addons': 'Complementos', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Armazenamento', 'admin.plugins.rescan': 'Reescanear', 'admin.plugins.rescanned': 'Pasta de plugins reescaneada', 'admin.plugins.upload': 'Enviar plugin', diff --git a/shared/src/i18n/br/externalNotifications.ts b/shared/src/i18n/br/externalNotifications.ts index 91fd6b4e0..2efad4196 100644 --- a/shared/src/i18n/br/externalNotifications.ts +++ b/shared/src/i18n/br/externalNotifications.ts @@ -52,6 +52,12 @@ const br: NotificationLocale = { title: 'Nova versão do TREK disponível', body: `O TREK ${p.version} está disponível. Acesse o painel de administração para atualizar.`, }), + replica_failure: (p) => ({ + title: 'Falha na réplica de armazenamento', + body: + `Falha ao gravar na réplica '${p.backend}': ${p.op} de ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Mais ${p.suppressed} falha(s) foram suprimidas desde a última notificação.` : ''), + }), synology_session_cleared: () => ({ title: 'Sessão Synology encerrada', body: 'Sua conta ou URL do Synology foi alterada. Você foi desconectado do Synology Photos.', diff --git a/shared/src/i18n/br/index.ts b/shared/src/i18n/br/index.ts index 71788f536..f7465a624 100644 --- a/shared/src/i18n/br/index.ts +++ b/shared/src/i18n/br/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/br/notif.ts b/shared/src/i18n/br/notif.ts index 2f0711f2c..d67524c95 100644 --- a/shared/src/i18n/br/notif.ts +++ b/shared/src/i18n/br/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} atribuiu você a {category} em {trip}', 'notif.version_available.title': 'Nova versão disponível', 'notif.version_available.text': 'TREK {version} está disponível', + 'notif.replica_failure.title': 'Falha na réplica de armazenamento', + 'notif.replica_failure.text': "Falha ao gravar na réplica '{backend}': {op} de {key} — {error}", + 'notif.replica_failure.textSuppressed': "Falha ao gravar na réplica '{backend}': {op} de {key} — {error}. Mais {suppressed} falha(s) foram suprimidas desde a última notificação.", 'notif.action.view_trip': 'Ver viagem', 'notif.action.view_collab': 'Ver mensagens', 'notif.action.view_packing': 'Ver bagagem', diff --git a/shared/src/i18n/br/storage.ts b/shared/src/i18n/br/storage.ts new file mode 100644 index 000000000..a046ccbbf --- /dev/null +++ b/shared/src/i18n/br/storage.ts @@ -0,0 +1,138 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Diretório raiz', + 'storage.help.root': 'Caminho absoluto no servidor onde este backend armazena seus objetos.', + 'storage.field.endpoint': 'URL do endpoint', + 'storage.help.endpoint': + 'URL base do serviço compatível com S3, ex.: https://s3.example.com ou http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID da chave de acesso', + 'storage.field.secretAccessKey': 'Chave de acesso secreta', + 'storage.field.region': 'Região', + 'storage.help.region': 'Mantenha o padrão, a menos que seu provedor exija uma região específica.', + 'storage.field.keyPrefix': 'Prefixo de chave', + 'storage.help.keyPrefix': 'Prefixo opcional adicionado a cada chave de objeto, ex.: trek/prod.', + 'storage.field.retries': 'Tentativas', + 'storage.field.timeoutMs': 'Tempo limite (ms)', + 'storage.field.primary': 'Backend primário', + 'storage.field.replicas': 'Réplicas', + 'storage.title': 'Armazenamento', + 'storage.description': 'Onde o TREK guarda arquivos, fotos e backups enviados. Nada muda até você salvar.', + 'storage.loading': 'Carregando…', + 'storage.saved': 'Configuração de armazenamento salva', + 'storage.save': 'Salvar alterações', + 'storage.unsaved': 'Alterações não salvas', + 'storage.saveConflict': + 'A configuração de armazenamento mudou desde que foi carregada, por isso suas alterações não foram salvas. Descarte-as e recarregue a configuração salva para recomeçar.', + 'storage.discardAndReload': 'Descartar minhas alterações e recarregar', + 'storage.configError.banner': + 'Falha ao carregar as configurações de armazenamento salvas — salvar irá substituí-las: {error}', + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Adicionar backend', + 'storage.backends.usedBy': 'Usado por: {categories}', + 'storage.backends.unused': 'Não atribuído a nenhuma categoria', + 'storage.backends.envReadOnly': 'Definido por uma variável de ambiente — somente leitura', + 'storage.source.built-in': 'Integrado', + 'storage.source.env': 'Ambiente', + 'storage.source.settings': 'Configurações', + 'storage.type.local': 'Local', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Espelho', + 'storage.actions.test': 'Testar', + 'storage.actions.edit': 'Editar', + 'storage.actions.remove': 'Remover', + 'storage.test.running': 'Testando…', + 'storage.test.ok': 'Conexão OK', + 'storage.test.failed': 'Teste falhou', + 'storage.remove.title': 'Remover backend', + 'storage.remove.body': 'Remover {name} da configuração? O servidor rejeita o salvamento se algo ainda depender dele.', + 'storage.remove.stillAssigned': 'Ainda atribuído a: {categories}', + 'storage.form.addTitle': 'Adicionar backend', + 'storage.form.editTitle': 'Editar backend', + 'storage.form.name': 'Nome', + 'storage.form.type': 'Tipo', + 'storage.form.apply': 'Aplicar', + 'storage.form.cancel': 'Cancelar', + 'storage.form.duplicateName': 'Já existe um backend chamado {name}', + 'storage.categories.title': 'Categorias', + 'storage.categories.default': 'padrão', + 'storage.categories.reassignWarning': + 'Objetos existentes não são movidos: novos objetos vão para o backend recém-atribuído, os antigos permanecem onde estão.', + 'storage.category.files': 'Documentos da viagem', + 'storage.category.journey': 'Fotos da jornada', + 'storage.category.covers': 'Imagens de capa', + 'storage.category.avatars': 'Fotos de perfil', + 'storage.category.places': 'Imagens de locais', + 'storage.category.photos-google': 'Cache de fotos do Google', + 'storage.category.photos-trek': 'Cache de fotos do TREK', + 'storage.category.backups': 'Backups', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Anexos enviados para viagens — passagens, PDFs, confirmações de reserva e arquivos compartilhados no chat da viagem.', + 'storage.categoryDesc.journey': 'Fotos e miniaturas anexadas às entradas da jornada.', + 'storage.categoryDesc.covers': 'Imagens de capa de viagens e coleções, incluindo capas obtidas do Unsplash.', + 'storage.categoryDesc.avatars': 'Fotos de perfil das contas de usuário.', + 'storage.categoryDesc.places': 'Imagens anexadas a locais e locais de coleções — enviadas ou importadas.', + 'storage.categoryDesc.photos-google': + 'Cópias em cache de fotos do Google Places — podem ser buscadas novamente, seguro perdê-las.', + 'storage.categoryDesc.photos-trek': + 'Fotos em cache do serviço de fotos do TREK usado pelas Fotos (Memórias) — podem ser buscadas novamente, seguro perdê-las.', + 'storage.categoryDesc.backups': 'Arquivos de backup do servidor criados pelo painel de Backup ou pela agenda.', + 'storage.health.title': 'Integridade', + 'storage.health.allClear': 'Nenhuma falha de réplica registrada.', + 'storage.health.seedFile': + 'Um arquivo semente storage-config.json está presente, mas é ignorado — já existem linhas de configuração. Gerencie o armazenamento aqui.', + 'storage.health.failureLine': '{op} de {key} em {backend} falhou: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Destinos do espelho', + 'storage.mirror.targetsHelp': 'Cada gravação neste backend também é copiada para cada destino selecionado.', + 'storage.mirror.latencyNote': + 'As réplicas são gravadas uma após a outra durante cada envio — um destino lento ou inacessível torna mais lento todo envio de toda categoria neste backend.', + 'storage.mirror.mirroredTo': 'Espelhado para: {targets}', + 'storage.mirror.replicaOf': 'Réplica de: {primaries}', + 'storage.mirror.cacheWarning': + 'Não recomendado: esta categoria contém conteúdo que pode ser buscado novamente — replicá-lo geralmente é um desperdício.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Um segundo espelho envolve {primary} — o painel gerencia apenas o primeiro; remova este para gerenciar o espelhamento a partir de {primary}.', + 'storage.mirror.degenerate.env-primary': 'Envolve um backend definido por variável de ambiente — não editável aqui.', + 'storage.mirror.degenerate.missing-primary': 'Referencia um backend que não existe mais.', + 'storage.remove.usedAsReplicaBy': 'Usado como réplica por: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sincronizar agora', + 'storage.sync.running': 'Sincronizando… {done}/{total}', + 'storage.sync.counts': '{copied} copiados · {skipped} ignorados · {failed} falharam', + 'storage.sync.cancel': 'Cancelar sincronização', + 'storage.sync.done': 'Sincronização concluída: {copied} copiados, {deleted} excluídos, {failed} falharam', + 'storage.sync.cancelled': 'Sincronização cancelada', + 'storage.sync.error': 'Falha na sincronização: {error}', + 'storage.sync.prompt': 'Os objetos existentes ainda não foram replicados — sincronizar agora?', + 'storage.sync.dismiss': 'Dispensar', + 'storage.usage.line': '{objects} objetos · {size}', + 'storage.usage.computed': 'Uso calculado {age}', + 'storage.usage.never': 'Uso ainda não calculado', + 'storage.usage.refresh': 'Atualizar', + 'storage.usage.compute': 'Calcular agora', + 'storage.usage.legacyNote': 'inclui a biblioteca de fotos legada', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Mover objetos existentes para o novo backend?', + 'storage.migrate.promptLine': '{category}: {objects} objetos ({size}) de {from} para {to}', + 'storage.migrate.promptLineUnknown': '{category}: tamanho desconhecido (uso ainda não calculado) de {from} para {to}', + 'storage.migrate.move': 'Mover objetos existentes', + 'storage.migrate.routeOnly': 'Apenas rotear novas gravações', + 'storage.migrate.running': 'Movendo {category}… {done}/{total}', + 'storage.migrate.done': 'Movimentação concluída: {copied} copiados, {skipped} ignorados', + 'storage.migrate.doneFailures': '{failed} falharam — esses objetos não foram copiados para o novo backend', + 'storage.migrate.failed': 'Falha ao mover: {error} — a categoria não foi alterada', + 'storage.migrate.cancelled': 'Movimentação cancelada — nada foi alterado', + 'storage.migrate.reclaimable': '{objects} objetos ({size}) permanecem em {from} — recupere manualmente', + 'storage.migrate.cancel': 'Cancelar movimentação', + 'storage.migrate.promptCancel': 'Cancelar', + 'storage.migrate.queued': 'Na fila: {categories}', + 'storage.migrate.queueDropped': 'Não foi possível iniciar a próxima migração — a fila restante foi limpa: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/ca/admin.ts b/shared/src/i18n/ca/admin.ts index e2bd47c33..509d5b387 100644 --- a/shared/src/i18n/ca/admin.ts +++ b/shared/src/i18n/ca/admin.ts @@ -377,6 +377,7 @@ const admin: TranslationStrings = { 'admin.unsplashKey': "Clau API d'Unsplash", 'admin.unsplashKeyHint': "Per a la cerca d'imatges. Gratuït a unsplash.com/developers", 'admin.tabs.plugins': 'Connectors', + 'admin.tabs.storage': 'Emmagatzematge', 'admin.plugins.rescan': 'Torna a escanejar', 'admin.plugins.rescanned': "S'ha tornat a escanejar la carpeta de connectors", 'admin.plugins.upload': 'Puja el connector', diff --git a/shared/src/i18n/ca/externalNotifications.ts b/shared/src/i18n/ca/externalNotifications.ts index f33138356..45a631720 100644 --- a/shared/src/i18n/ca/externalNotifications.ts +++ b/shared/src/i18n/ca/externalNotifications.ts @@ -48,6 +48,12 @@ const ca: NotificationLocale = { title: 'Versió nova de TREK disponible', body: `TREK ${p.version} ja està disponible. Visita el panell d'administració per actualitzar.`, }), + replica_failure: (p) => ({ + title: "Error de rèplica d'emmagatzematge", + body: + `L'escriptura a la rèplica '${p.backend}' ha fallat: ${p.op} de ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` S'han suprimit ${p.suppressed} errors més des de l'última notificació.` : ''), + }), synology_session_cleared: () => ({ title: 'Sessió de Synology tancada', body: 'El teu compte o URL de Synology ha canviat. Has tancat la sessió a Synology Photos.', diff --git a/shared/src/i18n/ca/index.ts b/shared/src/i18n/ca/index.ts index b7844e226..5b95ae022 100644 --- a/shared/src/i18n/ca/index.ts +++ b/shared/src/i18n/ca/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/ca/notif.ts b/shared/src/i18n/ca/notif.ts index 28fa58849..8a9e6c786 100644 --- a/shared/src/i18n/ca/notif.ts +++ b/shared/src/i18n/ca/notif.ts @@ -25,6 +25,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} et va assignar a {category} a {trip}', 'notif.version_available.title': 'Versió nova disponible', 'notif.version_available.text': 'TREK {version} ja està disponible', + 'notif.replica_failure.title': "Error de rèplica d'emmagatzematge", + 'notif.replica_failure.text': "L'escriptura a la rèplica '{backend}' ha fallat: {op} de {key} — {error}", + 'notif.replica_failure.textSuppressed': "L'escriptura a la rèplica '{backend}' ha fallat: {op} de {key} — {error}. S'han suprimit {suppressed} errors més des de l'última notificació.", 'notif.action.view_trip': 'Mostra el viatge', 'notif.action.view_collab': 'Mostra els missatges', 'notif.action.view_packing': "Mostra l'equipatge", diff --git a/shared/src/i18n/ca/storage.ts b/shared/src/i18n/ca/storage.ts new file mode 100644 index 000000000..2f3297570 --- /dev/null +++ b/shared/src/i18n/ca/storage.ts @@ -0,0 +1,142 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Directori arrel', + 'storage.help.root': 'Camí absolut al servidor on aquest backend emmagatzema els seus objectes.', + 'storage.field.endpoint': "URL de l'endpoint", + 'storage.help.endpoint': + 'URL base del servei compatible amb S3, p. ex. https://s3.example.com o http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': "ID de clau d'accés", + 'storage.field.secretAccessKey': "Clau d'accés secreta", + 'storage.field.region': 'Regió', + 'storage.help.region': 'Mantén el valor predeterminat llevat que el teu proveïdor requereixi una regió específica.', + 'storage.field.keyPrefix': 'Prefix de clau', + 'storage.help.keyPrefix': "Prefix opcional afegit a cada clau d'objecte, p. ex. trek/prod.", + 'storage.field.retries': 'Reintents', + 'storage.field.timeoutMs': "Temps d'espera (ms)", + 'storage.field.primary': 'Backend principal', + 'storage.field.replicas': 'Rèpliques', + 'storage.title': 'Emmagatzematge', + 'storage.description': 'On TREK desa els fitxers, fotos i còpies de seguretat pujats. Res no canvia fins que desis.', + 'storage.loading': 'Carregant…', + 'storage.saved': "Configuració d'emmagatzematge desada", + 'storage.save': 'Desar els canvis', + 'storage.unsaved': 'Canvis sense desar', + 'storage.saveConflict': + "La configuració d'emmagatzematge ha canviat des que es va carregar, així que els teus canvis no s'han desat. Descarta'ls i torna a carregar la configuració desada per començar de nou.", + 'storage.discardAndReload': 'Descarta els meus canvis i torna a carregar', + 'storage.configError.banner': + "No s'han pogut carregar els paràmetres d'emmagatzematge desats — desar-los els substituirà: {error}", + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Afegir backend', + 'storage.backends.usedBy': 'Utilitzat per: {categories}', + 'storage.backends.unused': 'No assignat a cap categoria', + 'storage.backends.envReadOnly': "Definit per una variable d'entorn — només lectura", + 'storage.source.built-in': 'Integrat', + 'storage.source.env': 'Entorn', + 'storage.source.settings': 'Configuració', + 'storage.type.local': 'Local', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Mirall', + 'storage.actions.test': 'Provar', + 'storage.actions.edit': 'Editar', + 'storage.actions.remove': 'Suprimir', + 'storage.test.running': 'Provant…', + 'storage.test.ok': 'Connexió correcta', + 'storage.test.failed': 'Prova fallida', + 'storage.remove.title': 'Suprimir backend', + 'storage.remove.body': + 'Voleu suprimir {name} de la configuració? El servidor rebutja el desament si encara hi ha alguna cosa que en depengui.', + 'storage.remove.stillAssigned': 'Encara assignat a: {categories}', + 'storage.form.addTitle': 'Afegir backend', + 'storage.form.editTitle': 'Editar backend', + 'storage.form.name': 'Nom', + 'storage.form.type': 'Tipus', + 'storage.form.apply': 'Aplicar', + 'storage.form.cancel': 'Cancel·lar', + 'storage.form.duplicateName': 'Ja existeix un backend anomenat {name}', + 'storage.categories.title': 'Categories', + 'storage.categories.default': 'predeterminada', + 'storage.categories.reassignWarning': + "Els objectes existents no es mouen: els objectes nous van al backend acabat d'assignar, els antics es queden on són.", + 'storage.category.files': 'Documents del viatge', + 'storage.category.journey': 'Fotos de la travesia', + 'storage.category.covers': 'Imatges de portada', + 'storage.category.avatars': 'Fotos de perfil', + 'storage.category.places': 'Imatges de llocs', + 'storage.category.photos-google': 'Memòria cau de fotos de Google', + 'storage.category.photos-trek': 'Memòria cau de fotos de TREK', + 'storage.category.backups': 'Còpies de seguretat', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Fitxers adjunts pujats als viatges — bitllets, PDF, confirmacions de reserva i fitxers compartits al xat del viatge.', + 'storage.categoryDesc.journey': 'Fotos i miniatures adjuntes a les entrades de la travesia.', + 'storage.categoryDesc.covers': + "Imatges de portada de viatges i col·leccions, incloses les portades obtingudes d'Unsplash.", + 'storage.categoryDesc.avatars': "Fotos de perfil dels comptes d'usuari.", + 'storage.categoryDesc.places': 'Imatges adjuntes a llocs i llocs de col·leccions — pujades o importades.', + 'storage.categoryDesc.photos-google': + 'Còpies en memòria cau de fotos de Google Places — es poden tornar a obtenir, es poden perdre sense problemes.', + 'storage.categoryDesc.photos-trek': + 'Fotos en memòria cau del servei de fotos de TREK utilitzat per les Fotos (Memòries) — es poden tornar a obtenir, es poden perdre sense problemes.', + 'storage.categoryDesc.backups': + 'Arxius de còpia de seguretat del servidor creats pel panell de còpia de seguretat o la planificació.', + 'storage.health.title': 'Estat', + 'storage.health.allClear': "No s'ha registrat cap fallada de rèplica.", + 'storage.health.seedFile': + "Hi ha un fitxer llavor storage-config.json present però s'ignora — ja existeixen files de configuració. Gestiona l'emmagatzematge aquí.", + 'storage.health.failureLine': '{op} de {key} a {backend} ha fallat: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Objectius del mirall', + 'storage.mirror.targetsHelp': 'Cada escriptura en aquest backend també es copia a cada objectiu seleccionat.', + 'storage.mirror.latencyNote': + "Les rèpliques s'escriuen una després de l'altra durant cada pujada — un objectiu lent o inabastable alenteix cada pujada de cada categoria en aquest backend.", + 'storage.mirror.mirroredTo': 'Reflectit a: {targets}', + 'storage.mirror.replicaOf': 'Rèplica de: {primaries}', + 'storage.mirror.cacheWarning': + 'No recomanat: aquesta categoria conté contingut que es pot tornar a obtenir — replicar-lo sol ser un malbaratament.', + 'storage.mirror.degenerate.duplicate-mirror': + "Un segon mirall embolcalla {primary} — el panell només gestiona el primer; elimina'l per gestionar el mirall des de {primary}.", + 'storage.mirror.degenerate.env-primary': + "Embolcalla un backend definit per una variable d'entorn — no editable aquí.", + 'storage.mirror.degenerate.missing-primary': 'Fa referència a un backend que ja no existeix.', + 'storage.remove.usedAsReplicaBy': 'Utilitzat com a rèplica per: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sincronitza ara', + 'storage.sync.running': 'Sincronitzant… {done}/{total}', + 'storage.sync.counts': '{copied} copiats · {skipped} omesos · {failed} fallits', + 'storage.sync.cancel': 'Cancel·la la sincronització', + 'storage.sync.done': 'Sincronització finalitzada: {copied} copiats, {deleted} suprimits, {failed} fallits', + 'storage.sync.cancelled': 'Sincronització cancel·lada', + 'storage.sync.error': 'Ha fallat la sincronització: {error}', + 'storage.sync.prompt': "Els objectes existents encara no s'han replicat — vols sincronitzar ara?", + 'storage.sync.dismiss': 'Descarta', + 'storage.usage.line': '{objects} objectes · {size}', + 'storage.usage.computed': 'Ús calculat {age}', + 'storage.usage.never': 'Ús encara no calculat', + 'storage.usage.refresh': 'Actualitza', + 'storage.usage.compute': 'Calcula ara', + 'storage.usage.legacyNote': 'inclou la biblioteca de fotos antiga', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Vols moure els objectes existents al nou backend?', + 'storage.migrate.promptLine': '{category}: {objects} objectes ({size}) de {from} a {to}', + 'storage.migrate.promptLineUnknown': "{category}: mida desconeguda (encara no s'ha calculat l'ús) de {from} a {to}", + 'storage.migrate.move': 'Mou els objectes existents', + 'storage.migrate.routeOnly': 'Només enruta les escriptures noves', + 'storage.migrate.running': 'Movent {category}… {done}/{total}', + 'storage.migrate.done': 'Trasllat finalitzat: {copied} copiats, {skipped} omesos', + 'storage.migrate.doneFailures': "{failed} han fallat — aquests objectes no s'han copiat al nou backend", + 'storage.migrate.failed': "Ha fallat el trasllat: {error} — la categoria no s'ha canviat", + 'storage.migrate.cancelled': "Trasllat cancel·lat — no s'ha canviat res", + 'storage.migrate.reclaimable': "{objects} objectes ({size}) romanen a {from} — recupera'ls manualment", + 'storage.migrate.cancel': 'Cancel·la el trasllat', + 'storage.migrate.promptCancel': 'Cancel·la', + 'storage.migrate.queued': 'En cua: {categories}', + 'storage.migrate.queueDropped': "No s'ha pogut iniciar el trasllat següent — s'ha buidat la cua restant: {categories}", +}; +export default storage; diff --git a/shared/src/i18n/cs/admin.ts b/shared/src/i18n/cs/admin.ts index bef076aa9..2ac78d20a 100644 --- a/shared/src/i18n/cs/admin.ts +++ b/shared/src/i18n/cs/admin.ts @@ -165,6 +165,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Uložení se nezdařilo', 'admin.tabs.addons': 'Doplňky', 'admin.tabs.plugins': 'Pluginy', + 'admin.tabs.storage': 'Úložiště', 'admin.plugins.rescan': 'Znovu prohledat', 'admin.plugins.rescanned': 'Složka pluginů byla znovu prohledána', 'admin.plugins.upload': 'Nahrát plugin', diff --git a/shared/src/i18n/cs/externalNotifications.ts b/shared/src/i18n/cs/externalNotifications.ts index 10cbc237c..036d79c81 100644 --- a/shared/src/i18n/cs/externalNotifications.ts +++ b/shared/src/i18n/cs/externalNotifications.ts @@ -52,6 +52,12 @@ const cs: NotificationLocale = { title: 'Nová verze TREK dostupná', body: `TREK ${p.version} je nyní dostupný. Navštivte administrátorský panel pro aktualizaci.`, }), + replica_failure: (p) => ({ + title: 'Selhání repliky úložiště', + body: + `Zápis do repliky '${p.backend}' selhal: ${p.op} u ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Od poslední notifikace bylo potlačeno ${p.suppressed} dalších selhání.` : ''), + }), synology_session_cleared: () => ({ title: 'Relace Synology byla zrušena', body: 'Váš účet nebo URL Synology se změnil. Byli jste odhlášeni ze Synology Photos.', diff --git a/shared/src/i18n/cs/index.ts b/shared/src/i18n/cs/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/cs/index.ts +++ b/shared/src/i18n/cs/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/cs/notif.ts b/shared/src/i18n/cs/notif.ts index f7d8255b3..6bef10d38 100644 --- a/shared/src/i18n/cs/notif.ts +++ b/shared/src/i18n/cs/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} vás přiřadil k {category} v {trip}', 'notif.version_available.title': 'Nová verze dostupná', 'notif.version_available.text': 'TREK {version} je nyní dostupný', + 'notif.replica_failure.title': 'Selhání repliky úložiště', + 'notif.replica_failure.text': "Zápis do repliky '{backend}' selhal: {op} u {key} — {error}", + 'notif.replica_failure.textSuppressed': "Zápis do repliky '{backend}' selhal: {op} u {key} — {error}. Od poslední notifikace bylo potlačeno {suppressed} dalších selhání.", 'notif.action.view_trip': 'Zobrazit výlet', 'notif.action.view_collab': 'Zobrazit zprávy', 'notif.action.view_packing': 'Zobrazit balení', diff --git a/shared/src/i18n/cs/storage.ts b/shared/src/i18n/cs/storage.ts new file mode 100644 index 000000000..810183533 --- /dev/null +++ b/shared/src/i18n/cs/storage.ts @@ -0,0 +1,137 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Kořenový adresář', + 'storage.help.root': 'Absolutní cesta na serveru, kam tento backend ukládá své objekty.', + 'storage.field.endpoint': 'URL koncového bodu', + 'storage.help.endpoint': + 'Základní URL služby kompatibilní s S3, např. https://s3.example.com nebo http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID přístupového klíče', + 'storage.field.secretAccessKey': 'Tajný přístupový klíč', + 'storage.field.region': 'Region', + 'storage.help.region': 'Ponechte výchozí hodnotu, pokud váš poskytovatel nevyžaduje konkrétní region.', + 'storage.field.keyPrefix': 'Prefix klíče', + 'storage.help.keyPrefix': 'Volitelný prefix přidaný ke každému klíči objektu, např. trek/prod.', + 'storage.field.retries': 'Počet pokusů', + 'storage.field.timeoutMs': 'Časový limit (ms)', + 'storage.field.primary': 'Primární backend', + 'storage.field.replicas': 'Repliky', + 'storage.title': 'Úložiště', + 'storage.description': 'Kam TREK ukládá nahrané soubory, fotky a zálohy. Dokud neuložíte, nic se nezmění.', + 'storage.loading': 'Načítání…', + 'storage.saved': 'Konfigurace úložiště uložena', + 'storage.save': 'Uložit změny', + 'storage.unsaved': 'Neuložené změny', + 'storage.saveConflict': + 'Konfigurace úložiště se od načtení změnila, takže vaše změny nebyly uloženy. Zahoďte je a načtěte uložené nastavení znovu, abyste mohli začít od začátku.', + 'storage.discardAndReload': 'Zahodit mé změny a načíst znovu', + 'storage.configError.banner': 'Nepodařilo se načíst uložené nastavení úložiště — uložením je nahradíte: {error}', + 'storage.backends.title': 'Backendy', + 'storage.backends.add': 'Přidat backend', + 'storage.backends.usedBy': 'Používá: {categories}', + 'storage.backends.unused': 'Nepřiřazeno k žádné kategorii', + 'storage.backends.envReadOnly': 'Definováno proměnnou prostředí — pouze pro čtení', + 'storage.source.built-in': 'Vestavěné', + 'storage.source.env': 'Prostředí', + 'storage.source.settings': 'Nastavení', + 'storage.type.local': 'Místní', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Zrcadlo', + 'storage.actions.test': 'Test', + 'storage.actions.edit': 'Upravit', + 'storage.actions.remove': 'Odebrat', + 'storage.test.running': 'Testuje se…', + 'storage.test.ok': 'Připojení v pořádku', + 'storage.test.failed': 'Test selhal', + 'storage.remove.title': 'Odebrat backend', + 'storage.remove.body': 'Odebrat {name} z konfigurace? Server odmítne uložení, pokud na něm ještě něco závisí.', + 'storage.remove.stillAssigned': 'Stále přiřazeno k: {categories}', + 'storage.form.addTitle': 'Přidat backend', + 'storage.form.editTitle': 'Upravit backend', + 'storage.form.name': 'Název', + 'storage.form.type': 'Typ', + 'storage.form.apply': 'Použít', + 'storage.form.cancel': 'Zrušit', + 'storage.form.duplicateName': 'Backend s názvem {name} již existuje', + 'storage.categories.title': 'Kategorie', + 'storage.categories.default': 'výchozí', + 'storage.categories.reassignWarning': + 'Existující objekty se nepřesouvají: nové objekty jdou do nově přiřazeného backendu, staré zůstávají tam, kde jsou.', + 'storage.category.files': 'Dokumenty cesty', + 'storage.category.journey': 'Fotky z cestovního deníku', + 'storage.category.covers': 'Titulní obrázky', + 'storage.category.avatars': 'Profilové fotky', + 'storage.category.places': 'Obrázky míst', + 'storage.category.photos-google': 'Mezipaměť fotek Google', + 'storage.category.photos-trek': 'Mezipaměť fotek TREK', + 'storage.category.backups': 'Zálohy', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Soubory nahrané k cestám — jízdenky, PDF, potvrzení rezervací a soubory sdílené v chatu cesty.', + 'storage.categoryDesc.journey': 'Fotky a náhledy připojené k záznamům cestovního deníku.', + 'storage.categoryDesc.covers': 'Titulní obrázky cest a sbírek, včetně obrázků načtených z Unsplash.', + 'storage.categoryDesc.avatars': 'Profilové fotky uživatelských účtů.', + 'storage.categoryDesc.places': 'Obrázky připojené k místům a místům ve sbírkách — nahrané nebo importované.', + 'storage.categoryDesc.photos-google': + 'Mezipaměťové kopie fotek z Google Places — lze znovu načíst, jejich ztráta je bezpečná.', + 'storage.categoryDesc.photos-trek': + 'Fotky uložené v mezipaměti ze služby fotek TREK používané funkcí Fotky (Memories) — lze znovu načíst, jejich ztráta je bezpečná.', + 'storage.categoryDesc.backups': 'Archivy záloh serveru vytvořené panelem Záloha nebo plánovačem.', + 'storage.health.title': 'Stav', + 'storage.health.allClear': 'Nebyly zaznamenány žádné chyby replik.', + 'storage.health.seedFile': + 'Je přítomen seed soubor storage-config.json, ale je ignorován — konfigurační řádky již existují. Spravujte úložiště zde.', + 'storage.health.failureLine': '{op} u {key} na {backend} selhalo: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Cíle zrcadla', + 'storage.mirror.targetsHelp': 'Každý zápis do tohoto backendu se také zkopíruje do každého vybraného cíle.', + 'storage.mirror.latencyNote': + 'Repliky se zapisují jedna po druhé během každého nahrávání — pomalý nebo nedostupný cíl zpomalí každé nahrávání každé kategorie na tomto backendu.', + 'storage.mirror.mirroredTo': 'Zrcadleno do: {targets}', + 'storage.mirror.replicaOf': 'Replika: {primaries}', + 'storage.mirror.cacheWarning': + 'Nedoporučeno: tato kategorie obsahuje znovu získatelný obsah — jeho replikace je obvykle zbytečná.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Druhé zrcadlo obaluje {primary} — panel spravuje pouze první; odeberte toto pro správu zrcadlení z {primary}.', + 'storage.mirror.degenerate.env-primary': 'Obaluje backend definovaný proměnnou prostředí — zde needitovatelné.', + 'storage.mirror.degenerate.missing-primary': 'Odkazuje na backend, který již neexistuje.', + 'storage.remove.usedAsReplicaBy': 'Používáno jako replika: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Synchronizovat nyní', + 'storage.sync.running': 'Synchronizace… {done}/{total}', + 'storage.sync.counts': '{copied} zkopírováno · {skipped} přeskočeno · {failed} selhalo', + 'storage.sync.cancel': 'Zrušit synchronizaci', + 'storage.sync.done': 'Synchronizace dokončena: {copied} zkopírováno, {deleted} odstraněno, {failed} selhalo', + 'storage.sync.cancelled': 'Synchronizace zrušena', + 'storage.sync.error': 'Synchronizace selhala: {error}', + 'storage.sync.prompt': 'Existující objekty ještě nejsou replikovány — synchronizovat nyní?', + 'storage.sync.dismiss': 'Zavřít', + 'storage.usage.line': '{objects} objektů · {size}', + 'storage.usage.computed': 'Využití vypočteno {age}', + 'storage.usage.never': 'Využití zatím nebylo vypočteno', + 'storage.usage.refresh': 'Obnovit', + 'storage.usage.compute': 'Vypočítat nyní', + 'storage.usage.legacyNote': 'zahrnuje starší fotoknihovnu', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Přesunout stávající objekty do nového backendu?', + 'storage.migrate.promptLine': '{category}: {objects} objektů ({size}) z {from} do {to}', + 'storage.migrate.promptLineUnknown': '{category}: neznámá velikost (využití ještě nebylo vypočteno) z {from} do {to}', + 'storage.migrate.move': 'Přesunout stávající objekty', + 'storage.migrate.routeOnly': 'Pouze směrovat nové zápisy', + 'storage.migrate.running': 'Přesouvání {category}… {done}/{total}', + 'storage.migrate.done': 'Přesun dokončen: {copied} zkopírováno, {skipped} přeskočeno', + 'storage.migrate.doneFailures': '{failed} se nezdařilo — tyto objekty nebyly zkopírovány do nového backendu', + 'storage.migrate.failed': 'Přesun selhal: {error} — kategorie nebyla přepnuta', + 'storage.migrate.cancelled': 'Přesun zrušen — nic nebylo přepnuto', + 'storage.migrate.reclaimable': '{objects} objektů ({size}) zůstává na {from} — uvolněte ručně', + 'storage.migrate.cancel': 'Zrušit přesun', + 'storage.migrate.promptCancel': 'Zrušit', + 'storage.migrate.queued': 'Ve frontě: {categories}', + 'storage.migrate.queueDropped': 'Nepodařilo se spustit další přesun — zbývající fronta byla vymazána: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/de/admin.ts b/shared/src/i18n/de/admin.ts index a4a887ae3..405d01b96 100644 --- a/shared/src/i18n/de/admin.ts +++ b/shared/src/i18n/de/admin.ts @@ -167,6 +167,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Fehler beim Speichern', 'admin.tabs.addons': 'Addons', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Speicher', 'admin.plugins.rescan': 'Neu scannen', 'admin.plugins.rescanned': 'Plugin-Ordner neu gescannt', 'admin.plugins.upload': 'Plugin hochladen', diff --git a/shared/src/i18n/de/externalNotifications.ts b/shared/src/i18n/de/externalNotifications.ts index 5509451f7..985f86073 100644 --- a/shared/src/i18n/de/externalNotifications.ts +++ b/shared/src/i18n/de/externalNotifications.ts @@ -52,6 +52,14 @@ const de: NotificationLocale = { title: 'Neue TREK-Version verfügbar', body: `TREK ${p.version} ist jetzt verfügbar. Besuche das Admin-Panel zum Aktualisieren.`, }), + replica_failure: (p) => ({ + title: 'Speicher-Replik-Fehler', + body: + `Schreibvorgang auf Replik '${p.backend}' fehlgeschlagen: ${p.op} von ${p.key} — ${p.error}.` + + (p.suppressed !== '0' + ? ` Seit der letzten Benachrichtigung wurden ${p.suppressed} weitere Fehler unterdrückt.` + : ''), + }), synology_session_cleared: () => ({ title: 'Synology-Sitzung beendet', body: 'Dein Synology-Konto oder die URL hat sich geändert. Du wurdest von Synology Photos abgemeldet.', diff --git a/shared/src/i18n/de/index.ts b/shared/src/i18n/de/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/de/index.ts +++ b/shared/src/i18n/de/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/de/notif.ts b/shared/src/i18n/de/notif.ts index be71805b1..c126a8609 100644 --- a/shared/src/i18n/de/notif.ts +++ b/shared/src/i18n/de/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} hat dich zu {category} in {trip} zugewiesen', 'notif.version_available.title': 'Neue Version verfügbar', 'notif.version_available.text': 'TREK {version} ist jetzt verfügbar', + 'notif.replica_failure.title': 'Speicher-Replik-Fehler', + 'notif.replica_failure.text': "Schreibvorgang auf Replik '{backend}' fehlgeschlagen: {op} von {key} — {error}", + 'notif.replica_failure.textSuppressed': "Schreibvorgang auf Replik '{backend}' fehlgeschlagen: {op} von {key} — {error}. Seit der letzten Benachrichtigung wurden {suppressed} weitere Fehler unterdrückt.", 'notif.action.view_trip': 'Reise ansehen', 'notif.action.view_collab': 'Nachrichten ansehen', 'notif.action.view_packing': 'Packliste ansehen', diff --git a/shared/src/i18n/de/storage.ts b/shared/src/i18n/de/storage.ts new file mode 100644 index 000000000..130d794c5 --- /dev/null +++ b/shared/src/i18n/de/storage.ts @@ -0,0 +1,142 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Stammverzeichnis', + 'storage.help.root': 'Absoluter Pfad auf dem Server, unter dem dieser Backend-Dienst seine Objekte speichert.', + 'storage.field.endpoint': 'Endpunkt-URL', + 'storage.help.endpoint': + 'Basis-URL des S3-kompatiblen Dienstes, z. B. https://s3.example.com oder http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Zugriffsschlüssel-ID', + 'storage.field.secretAccessKey': 'Geheimer Zugriffsschlüssel', + 'storage.field.region': 'Region', + 'storage.help.region': 'Belasse die Standardeinstellung, sofern dein Anbieter keine bestimmte Region verlangt.', + 'storage.field.keyPrefix': 'Schlüsselpräfix', + 'storage.help.keyPrefix': 'Optionales Präfix, das jedem Objektschlüssel vorangestellt wird, z. B. trek/prod.', + 'storage.field.retries': 'Wiederholungsversuche', + 'storage.field.timeoutMs': 'Zeitlimit (ms)', + 'storage.field.primary': 'Primäres Backend', + 'storage.field.replicas': 'Repliken', + 'storage.title': 'Speicher', + 'storage.description': + 'Wo TREK hochgeladene Dateien, Fotos und Backups aufbewahrt. Es ändert sich nichts, bis du speicherst.', + 'storage.loading': 'Wird geladen…', + 'storage.saved': 'Speicherkonfiguration gespeichert', + 'storage.save': 'Änderungen speichern', + 'storage.unsaved': 'Ungespeicherte Änderungen', + 'storage.saveConflict': + 'Die Speicherkonfiguration hat sich seit dem Laden geändert, deshalb wurden Ihre Änderungen nicht gespeichert. Verwerfen Sie sie und laden Sie die gespeicherten Einstellungen neu, um von vorn zu beginnen.', + 'storage.discardAndReload': 'Meine Änderungen verwerfen und neu laden', + 'storage.configError.banner': + 'Die gespeicherten Speichereinstellungen konnten nicht geladen werden — durch Speichern werden sie ersetzt: {error}', + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Backend hinzufügen', + 'storage.backends.usedBy': 'Verwendet von: {categories}', + 'storage.backends.unused': 'Keiner Kategorie zugewiesen', + 'storage.backends.envReadOnly': 'Durch eine Umgebungsvariable definiert — schreibgeschützt', + 'storage.source.built-in': 'Integriert', + 'storage.source.env': 'Umgebung', + 'storage.source.settings': 'Einstellungen', + 'storage.type.local': 'Lokal', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Spiegel', + 'storage.actions.test': 'Testen', + 'storage.actions.edit': 'Bearbeiten', + 'storage.actions.remove': 'Entfernen', + 'storage.test.running': 'Wird getestet…', + 'storage.test.ok': 'Verbindung erfolgreich', + 'storage.test.failed': 'Test fehlgeschlagen', + 'storage.remove.title': 'Backend entfernen', + 'storage.remove.body': + '{name} aus der Konfiguration entfernen? Der Server lehnt das Speichern ab, wenn noch etwas davon abhängt.', + 'storage.remove.stillAssigned': 'Noch zugewiesen zu: {categories}', + 'storage.form.addTitle': 'Backend hinzufügen', + 'storage.form.editTitle': 'Backend bearbeiten', + 'storage.form.name': 'Name', + 'storage.form.type': 'Typ', + 'storage.form.apply': 'Übernehmen', + 'storage.form.cancel': 'Abbrechen', + 'storage.form.duplicateName': 'Ein Backend namens {name} existiert bereits', + 'storage.categories.title': 'Kategorien', + 'storage.categories.default': 'Standard', + 'storage.categories.reassignWarning': + 'Vorhandene Objekte werden nicht verschoben: Neue Objekte gehen an das neu zugewiesene Backend, alte bleiben, wo sie sind.', + 'storage.category.files': 'Reisedokumente', + 'storage.category.journey': 'Journey-Fotos', + 'storage.category.covers': 'Titelbilder', + 'storage.category.avatars': 'Profilbilder', + 'storage.category.places': 'Ortsbilder', + 'storage.category.photos-google': 'Google-Fotocache', + 'storage.category.photos-trek': 'TREK-Fotocache', + 'storage.category.backups': 'Backups', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Dateianhänge, die zu Reisen hochgeladen wurden — Tickets, PDFs, Buchungsbestätigungen und im Reise-Chat geteilte Dateien.', + 'storage.categoryDesc.journey': 'Fotos und Vorschaubilder, die an Journey-Einträge angehängt sind.', + 'storage.categoryDesc.covers': 'Titelbilder von Reisen und Sammlungen, einschließlich Titelbildern von Unsplash.', + 'storage.categoryDesc.avatars': 'Profilbilder von Benutzerkonten.', + 'storage.categoryDesc.places': 'Bilder, die an Orte und Sammlungsorte angehängt sind — hochgeladen oder importiert.', + 'storage.categoryDesc.photos-google': + 'Zwischengespeicherte Kopien von Google-Places-Fotos — können erneut abgerufen werden, ein Verlust ist unbedenklich.', + 'storage.categoryDesc.photos-trek': + 'Zwischengespeicherte Fotos vom TREK-Fotodienst, der von Fotos (Memories) verwendet wird — können erneut abgerufen werden, ein Verlust ist unbedenklich.', + 'storage.categoryDesc.backups': 'Server-Backup-Archive, erstellt vom Backup-Panel oder nach Zeitplan.', + 'storage.health.title': 'Zustand', + 'storage.health.allClear': 'Keine Replikatfehler aufgezeichnet.', + 'storage.health.seedFile': + 'Eine storage-config.json-Seed-Datei ist vorhanden, wird aber ignoriert — Konfigurationszeilen existieren bereits. Verwalte den Speicher hier.', + 'storage.health.failureLine': '{op} von {key} auf {backend} fehlgeschlagen: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Spiegelziele', + 'storage.mirror.targetsHelp': 'Jeder Schreibvorgang auf diesem Backend wird auch auf jedes ausgewählte Ziel kopiert.', + 'storage.mirror.latencyNote': + 'Repliken werden bei jedem Upload nacheinander geschrieben — ein langsames oder nicht erreichbares Ziel verlangsamt jeden Upload jeder Kategorie auf diesem Backend.', + 'storage.mirror.mirroredTo': 'Gespiegelt auf: {targets}', + 'storage.mirror.replicaOf': 'Replik von: {primaries}', + 'storage.mirror.cacheWarning': + 'Nicht empfohlen: Diese Kategorie enthält erneut abrufbare Inhalte — sie zu replizieren ist meist verschwendet.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Ein zweiter Spiegel umschließt {primary} — das Panel verwaltet nur den ersten; entferne diesen, um die Spiegelung ab {primary} zu verwalten.', + 'storage.mirror.degenerate.env-primary': + 'Umschließt ein durch Umgebungsvariable definiertes Backend — hier nicht bearbeitbar.', + 'storage.mirror.degenerate.missing-primary': 'Verweist auf ein Backend, das nicht mehr existiert.', + 'storage.remove.usedAsReplicaBy': 'Als Replik verwendet von: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Jetzt synchronisieren', + 'storage.sync.running': 'Synchronisiere… {done}/{total}', + 'storage.sync.counts': '{copied} kopiert · {skipped} übersprungen · {failed} fehlgeschlagen', + 'storage.sync.cancel': 'Synchronisierung abbrechen', + 'storage.sync.done': 'Synchronisierung abgeschlossen: {copied} kopiert, {deleted} gelöscht, {failed} fehlgeschlagen', + 'storage.sync.cancelled': 'Synchronisierung abgebrochen', + 'storage.sync.error': 'Synchronisierung fehlgeschlagen: {error}', + 'storage.sync.prompt': 'Vorhandene Objekte wurden noch nicht repliziert — jetzt synchronisieren?', + 'storage.sync.dismiss': 'Verwerfen', + 'storage.usage.line': '{objects} Objekte · {size}', + 'storage.usage.computed': 'Nutzung berechnet {age}', + 'storage.usage.never': 'Nutzung noch nicht berechnet', + 'storage.usage.refresh': 'Aktualisieren', + 'storage.usage.compute': 'Jetzt berechnen', + 'storage.usage.legacyNote': 'enthält die alte Fotobibliothek', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Vorhandene Objekte auf das neue Backend verschieben?', + 'storage.migrate.promptLine': '{category}: {objects} Objekte ({size}) von {from} nach {to}', + 'storage.migrate.promptLineUnknown': + '{category}: unbekannte Größe (Nutzung noch nicht gescannt) von {from} nach {to}', + 'storage.migrate.move': 'Vorhandene Objekte verschieben', + 'storage.migrate.routeOnly': 'Nur neue Schreibvorgänge umleiten', + 'storage.migrate.running': 'Verschiebe {category}… {done}/{total}', + 'storage.migrate.done': 'Verschieben abgeschlossen: {copied} kopiert, {skipped} übersprungen', + 'storage.migrate.doneFailures': '{failed} fehlgeschlagen — diese Objekte wurden nicht auf das neue Backend kopiert', + 'storage.migrate.failed': 'Verschieben fehlgeschlagen: {error} — die Kategorie wurde nicht umgestellt', + 'storage.migrate.cancelled': 'Verschieben abgebrochen — nichts wurde umgestellt', + 'storage.migrate.reclaimable': '{objects} Objekte ({size}) verbleiben auf {from} — manuell freigeben', + 'storage.migrate.cancel': 'Verschieben abbrechen', + 'storage.migrate.promptCancel': 'Abbrechen', + 'storage.migrate.queued': 'In Warteschlange: {categories}', + 'storage.migrate.queueDropped': 'Die nächste Verschiebung konnte nicht gestartet werden — die verbleibende Warteschlange wurde geleert: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/en/admin.ts b/shared/src/i18n/en/admin.ts index b5d4696ca..87fe57bf0 100644 --- a/shared/src/i18n/en/admin.ts +++ b/shared/src/i18n/en/admin.ts @@ -219,6 +219,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Failed to save', 'admin.tabs.addons': 'Addons', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Storage', 'admin.plugins.rescan': 'Rescan', 'admin.plugins.rescanned': 'Rescanned the plugins folder', 'admin.plugins.upload': 'Upload plugin', diff --git a/shared/src/i18n/en/externalNotifications.ts b/shared/src/i18n/en/externalNotifications.ts index 53c6c7fbf..846179460 100644 --- a/shared/src/i18n/en/externalNotifications.ts +++ b/shared/src/i18n/en/externalNotifications.ts @@ -52,6 +52,12 @@ const en: NotificationLocale = { title: 'New TREK version available', body: `TREK ${p.version} is now available. Visit the admin panel to update.`, }), + replica_failure: (p) => ({ + title: 'Storage replica failure', + body: + `Replica write failed on '${p.backend}': ${p.op} of ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} more failures were suppressed since the last notification.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology session cleared', body: 'Your Synology account or URL changed. You have been logged out of Synology Photos.', diff --git a/shared/src/i18n/en/index.ts b/shared/src/i18n/en/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/en/index.ts +++ b/shared/src/i18n/en/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/en/notif.ts b/shared/src/i18n/en/notif.ts index ed191640d..b0c8fd7a6 100644 --- a/shared/src/i18n/en/notif.ts +++ b/shared/src/i18n/en/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} assigned you to {category} in {trip}', 'notif.version_available.title': 'New Version Available', 'notif.version_available.text': 'TREK {version} is now available', + 'notif.replica_failure.title': 'Storage replica failure', + 'notif.replica_failure.text': "Replica write failed on '{backend}': {op} of {key} — {error}", + 'notif.replica_failure.textSuppressed': "Replica write failed on '{backend}': {op} of {key} — {error}. {suppressed} more failures were suppressed since the last notification.", 'notif.action.view_trip': 'View Trip', 'notif.action.view_collab': 'View Messages', 'notif.action.view_packing': 'View Packing', diff --git a/shared/src/i18n/en/storage.ts b/shared/src/i18n/en/storage.ts new file mode 100644 index 000000000..c395fbcd2 --- /dev/null +++ b/shared/src/i18n/en/storage.ts @@ -0,0 +1,153 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + // Field labels/help — these keys are pinned by STORAGE_BACKEND_TYPES in + // @trek/shared (labelKey/helpKey); renaming one breaks the admin form. + 'storage.field.root': 'Root directory', + 'storage.help.root': 'Absolute path on the server where this backend stores its objects.', + 'storage.field.endpoint': 'Endpoint URL', + 'storage.help.endpoint': + 'Base URL of the S3-compatible service, e.g. https://s3.example.com or http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Access key ID', + 'storage.field.secretAccessKey': 'Secret access key', + 'storage.field.region': 'Region', + 'storage.help.region': 'Keep the default unless your provider requires a specific region.', + 'storage.field.keyPrefix': 'Key prefix', + 'storage.help.keyPrefix': 'Optional prefix added to every object key, e.g. trek/prod.', + 'storage.field.retries': 'Retries', + 'storage.field.timeoutMs': 'Timeout (ms)', + 'storage.field.primary': 'Primary backend', + 'storage.field.replicas': 'Replicas', + + // Panel chrome + 'storage.title': 'Storage', + 'storage.description': 'Where TREK keeps uploaded files, photos and backups. Nothing changes until you save.', + 'storage.loading': 'Loading…', + 'storage.saved': 'Storage configuration saved', + 'storage.save': 'Save changes', + 'storage.unsaved': 'Unsaved changes', + + 'storage.saveConflict': + 'Storage settings changed since you loaded them, so your changes were not saved. Discard them and reload the saved settings to start over.', + 'storage.discardAndReload': 'Discard my changes and reload', + 'storage.configError.banner': 'Stored storage settings failed to load — saving will replace them: {error}', + // Backends list + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Add backend', + 'storage.backends.usedBy': 'Used by: {categories}', + 'storage.backends.unused': 'Not assigned to any category', + 'storage.backends.envReadOnly': 'Defined by an environment variable — read-only', + 'storage.source.built-in': 'Built-in', + 'storage.source.env': 'Environment', + 'storage.source.settings': 'Settings', + 'storage.type.local': 'Local', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Mirror', + 'storage.actions.test': 'Test', + 'storage.actions.edit': 'Edit', + 'storage.actions.remove': 'Remove', + + // Test-connection results + 'storage.test.running': 'Testing…', + 'storage.test.ok': 'Connection OK', + 'storage.test.failed': 'Test failed', + + // Remove pre-check (friendly message; the server stays authoritative) + 'storage.remove.title': 'Remove backend', + 'storage.remove.body': + 'Remove {name} from the configuration? The server rejects the save if anything still depends on it.', + 'storage.remove.stillAssigned': 'Still assigned to: {categories}', + + // Backend form + 'storage.form.addTitle': 'Add backend', + 'storage.form.editTitle': 'Edit backend', + 'storage.form.name': 'Name', + 'storage.form.type': 'Type', + 'storage.form.apply': 'Apply', + 'storage.form.cancel': 'Cancel', + 'storage.form.duplicateName': 'A backend named {name} already exists', + + // Category map + 'storage.categories.title': 'Categories', + 'storage.categories.default': 'default', + 'storage.categories.reassignWarning': + 'Existing objects do not move: new objects go to the newly assigned backend, old ones stay where they are.', + 'storage.category.files': 'Trip documents', + 'storage.category.journey': 'Journey photos', + 'storage.category.covers': 'Cover images', + 'storage.category.avatars': 'Profile pictures', + 'storage.category.places': 'Place images', + 'storage.category.photos-google': 'Google photo cache', + 'storage.category.photos-trek': 'TREK photo cache', + 'storage.category.backups': 'Backups', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'File attachments uploaded to trips — tickets, PDFs, booking confirmations, and files shared in trip chat.', + 'storage.categoryDesc.journey': 'Photos and thumbnails attached to journey entries.', + 'storage.categoryDesc.covers': 'Trip and collection cover images, including covers fetched from Unsplash.', + 'storage.categoryDesc.avatars': 'User account profile pictures.', + 'storage.categoryDesc.places': 'Images attached to places and collection places — uploaded or imported.', + 'storage.categoryDesc.photos-google': 'Cached copies of Google Places photos — re-fetchable, safe to lose.', + 'storage.categoryDesc.photos-trek': + 'Cached photos from the TREK photo service used by Memories — re-fetchable, safe to lose.', + 'storage.categoryDesc.backups': 'Server backup archives created by the Backup panel or schedule.', + + // Health strip + 'storage.health.title': 'Health', + 'storage.health.allClear': 'No replica failures recorded.', + 'storage.health.seedFile': + 'A storage-config.json seed file is present but ignored — configuration rows already exist. Manage storage here.', + 'storage.health.failureLine': '{op} of {key} on {backend} failed: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Mirror targets', + 'storage.mirror.targetsHelp': 'Every write to this backend is also copied to each selected target.', + 'storage.mirror.latencyNote': + 'Replicas are written one after another during each upload — a slow or unreachable target slows every upload of every category on this backend.', + 'storage.mirror.mirroredTo': 'Mirrored to: {targets}', + 'storage.mirror.replicaOf': 'Replica of: {primaries}', + 'storage.mirror.cacheWarning': + 'Not recommended: this category holds re-fetchable content — replicating it is usually wasteful.', + 'storage.mirror.degenerate.duplicate-mirror': + 'A second mirror wraps {primary} — the panel manages only the first; remove this one to manage mirroring from {primary}.', + 'storage.mirror.degenerate.env-primary': 'Wraps an environment-defined backend — not editable here.', + 'storage.mirror.degenerate.missing-primary': 'References a backend that no longer exists.', + 'storage.remove.usedAsReplicaBy': 'Used as replica by: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sync now', + 'storage.sync.running': 'Syncing… {done}/{total}', + 'storage.sync.counts': '{copied} copied · {skipped} skipped · {failed} failed', + 'storage.sync.cancel': 'Cancel sync', + 'storage.sync.done': 'Sync finished: {copied} copied, {deleted} deleted, {failed} failed', + 'storage.sync.cancelled': 'Sync cancelled', + 'storage.sync.error': 'Sync failed: {error}', + 'storage.sync.prompt': 'Existing objects are not replicated yet — sync now?', + 'storage.sync.dismiss': 'Dismiss', + 'storage.usage.line': '{objects} objects · {size}', + 'storage.usage.computed': 'Usage computed {age}', + 'storage.usage.never': 'Usage not computed yet', + 'storage.usage.refresh': 'Refresh', + 'storage.usage.compute': 'Compute now', + 'storage.usage.legacyNote': 'includes the legacy photo library', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Move existing objects to the new backend?', + 'storage.migrate.promptLine': '{category}: {objects} objects ({size}) from {from} to {to}', + 'storage.migrate.promptLineUnknown': '{category}: unknown size (no usage scan yet) from {from} to {to}', + 'storage.migrate.move': 'Move existing objects', + 'storage.migrate.routeOnly': 'Just route new writes', + 'storage.migrate.running': 'Moving {category}… {done}/{total}', + 'storage.migrate.done': 'Move finished: {copied} copied, {skipped} skipped', + 'storage.migrate.doneFailures': '{failed} failed — those objects were not copied to the new backend', + 'storage.migrate.failed': 'Move failed: {error} — the category was not switched', + 'storage.migrate.cancelled': 'Move cancelled — nothing was switched', + 'storage.migrate.reclaimable': '{objects} objects ({size}) remain on {from} — reclaim manually', + 'storage.migrate.cancel': 'Cancel move', + 'storage.migrate.promptCancel': 'Cancel', + 'storage.migrate.queued': 'Queued: {categories}', + 'storage.migrate.queueDropped': 'Could not start the next migration — the remaining queue was cleared: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/es/admin.ts b/shared/src/i18n/es/admin.ts index 1d5f4be51..d33e2957a 100644 --- a/shared/src/i18n/es/admin.ts +++ b/shared/src/i18n/es/admin.ts @@ -166,6 +166,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Error al guardar', 'admin.tabs.addons': 'Complementos', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Almacenamiento', 'admin.plugins.rescan': 'Volver a escanear', 'admin.plugins.rescanned': 'Carpeta de plugins reescaneada', 'admin.plugins.upload': 'Subir plugin', diff --git a/shared/src/i18n/es/externalNotifications.ts b/shared/src/i18n/es/externalNotifications.ts index 853e26290..f1b18abc5 100644 --- a/shared/src/i18n/es/externalNotifications.ts +++ b/shared/src/i18n/es/externalNotifications.ts @@ -52,6 +52,12 @@ const es: NotificationLocale = { title: 'Nueva versión de TREK disponible', body: `TREK ${p.version} ya está disponible. Visita el panel de administración para actualizar.`, }), + replica_failure: (p) => ({ + title: 'Fallo de réplica de almacenamiento', + body: + `Error al escribir en la réplica '${p.backend}': ${p.op} de ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Se suprimieron ${p.suppressed} errores más desde la última notificación.` : ''), + }), synology_session_cleared: () => ({ title: 'Sesión de Synology cerrada', body: 'Tu cuenta o URL de Synology ha cambiado. Has cerrado sesión en Synology Photos.', diff --git a/shared/src/i18n/es/index.ts b/shared/src/i18n/es/index.ts index 8b945855b..ee2ee75ac 100644 --- a/shared/src/i18n/es/index.ts +++ b/shared/src/i18n/es/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/es/notif.ts b/shared/src/i18n/es/notif.ts index 3bf92e58b..d9bfc263d 100644 --- a/shared/src/i18n/es/notif.ts +++ b/shared/src/i18n/es/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} te asignó a {category} en {trip}', 'notif.version_available.title': 'Nueva versión disponible', 'notif.version_available.text': 'TREK {version} ya está disponible', + 'notif.replica_failure.title': 'Fallo de réplica de almacenamiento', + 'notif.replica_failure.text': "Error al escribir en la réplica '{backend}': {op} de {key} — {error}", + 'notif.replica_failure.textSuppressed': "Error al escribir en la réplica '{backend}': {op} de {key} — {error}. Se suprimieron {suppressed} errores más desde la última notificación.", 'notif.action.view_trip': 'Ver viaje', 'notif.action.view_collab': 'Ver mensajes', 'notif.action.view_packing': 'Ver equipaje', diff --git a/shared/src/i18n/es/storage.ts b/shared/src/i18n/es/storage.ts new file mode 100644 index 000000000..32a2870c0 --- /dev/null +++ b/shared/src/i18n/es/storage.ts @@ -0,0 +1,143 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Directorio raíz', + 'storage.help.root': 'Ruta absoluta en el servidor donde este backend almacena sus objetos.', + 'storage.field.endpoint': 'URL del endpoint', + 'storage.help.endpoint': + 'URL base del servicio compatible con S3, p. ej. https://s3.example.com o http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID de clave de acceso', + 'storage.field.secretAccessKey': 'Clave de acceso secreta', + 'storage.field.region': 'Región', + 'storage.help.region': 'Mantén el valor predeterminado a menos que tu proveedor requiera una región específica.', + 'storage.field.keyPrefix': 'Prefijo de clave', + 'storage.help.keyPrefix': 'Prefijo opcional añadido a cada clave de objeto, p. ej. trek/prod.', + 'storage.field.retries': 'Reintentos', + 'storage.field.timeoutMs': 'Tiempo de espera (ms)', + 'storage.field.primary': 'Backend principal', + 'storage.field.replicas': 'Réplicas', + 'storage.title': 'Almacenamiento', + 'storage.description': + 'Dónde guarda TREK los archivos, fotos y copias de seguridad subidos. Nada cambia hasta que guardes.', + 'storage.loading': 'Cargando…', + 'storage.saved': 'Configuración de almacenamiento guardada', + 'storage.save': 'Guardar cambios', + 'storage.unsaved': 'Cambios sin guardar', + 'storage.saveConflict': + 'La configuración de almacenamiento cambió desde que la cargaste, así que tus cambios no se guardaron. Descártalos y recarga la configuración guardada para empezar de nuevo.', + 'storage.discardAndReload': 'Descartar mis cambios y recargar', + 'storage.configError.banner': + 'No se pudieron cargar los ajustes de almacenamiento guardados — guardar los reemplazará: {error}', + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Añadir backend', + 'storage.backends.usedBy': 'Usado por: {categories}', + 'storage.backends.unused': 'No asignado a ninguna categoría', + 'storage.backends.envReadOnly': 'Definido por una variable de entorno — solo lectura', + 'storage.source.built-in': 'Integrado', + 'storage.source.env': 'Entorno', + 'storage.source.settings': 'Configuración', + 'storage.type.local': 'Local', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Espejo', + 'storage.actions.test': 'Probar', + 'storage.actions.edit': 'Editar', + 'storage.actions.remove': 'Quitar', + 'storage.test.running': 'Probando…', + 'storage.test.ok': 'Conexión correcta', + 'storage.test.failed': 'Prueba fallida', + 'storage.remove.title': 'Quitar backend', + 'storage.remove.body': + '¿Quitar {name} de la configuración? El servidor rechaza el guardado si algo todavía depende de él.', + 'storage.remove.stillAssigned': 'Aún asignado a: {categories}', + 'storage.form.addTitle': 'Añadir backend', + 'storage.form.editTitle': 'Editar backend', + 'storage.form.name': 'Nombre', + 'storage.form.type': 'Tipo', + 'storage.form.apply': 'Aplicar', + 'storage.form.cancel': 'Cancelar', + 'storage.form.duplicateName': 'Ya existe un backend llamado {name}', + 'storage.categories.title': 'Categorías', + 'storage.categories.default': 'predeterminada', + 'storage.categories.reassignWarning': + 'Los objetos existentes no se mueven: los nuevos objetos van al backend recién asignado, los antiguos permanecen donde están.', + 'storage.category.files': 'Documentos del viaje', + 'storage.category.journey': 'Fotos de la travesía', + 'storage.category.covers': 'Imágenes de portada', + 'storage.category.avatars': 'Fotos de perfil', + 'storage.category.places': 'Imágenes de lugares', + 'storage.category.photos-google': 'Caché de fotos de Google', + 'storage.category.photos-trek': 'Caché de fotos de TREK', + 'storage.category.backups': 'Copias de seguridad', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Archivos adjuntos subidos a los viajes — billetes, PDF, confirmaciones de reserva y archivos compartidos en el chat del viaje.', + 'storage.categoryDesc.journey': 'Fotos y miniaturas adjuntas a las entradas de la travesía.', + 'storage.categoryDesc.covers': + 'Imágenes de portada de viajes y colecciones, incluidas las portadas obtenidas de Unsplash.', + 'storage.categoryDesc.avatars': 'Fotos de perfil de las cuentas de usuario.', + 'storage.categoryDesc.places': 'Imágenes adjuntas a lugares y lugares de colecciones — subidas o importadas.', + 'storage.categoryDesc.photos-google': + 'Copias en caché de fotos de Google Places — se pueden volver a obtener, es seguro perderlas.', + 'storage.categoryDesc.photos-trek': + 'Fotos en caché del servicio de fotos de TREK usado por Fotos (Memories) — se pueden volver a obtener, es seguro perderlas.', + 'storage.categoryDesc.backups': + 'Archivos de copia de seguridad del servidor creados por el panel de Copias de seguridad o su programación.', + 'storage.health.title': 'Estado', + 'storage.health.allClear': 'No se registraron fallos de réplica.', + 'storage.health.seedFile': + 'Hay un archivo semilla storage-config.json presente pero ignorado — ya existen filas de configuración. Gestiona el almacenamiento aquí.', + 'storage.health.failureLine': '{op} de {key} en {backend} falló: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Destinos del espejo', + 'storage.mirror.targetsHelp': 'Cada escritura en este backend también se copia a cada destino seleccionado.', + 'storage.mirror.latencyNote': + 'Las réplicas se escriben una tras otra durante cada subida — un destino lento o inaccesible ralentiza cada subida de cada categoría en este backend.', + 'storage.mirror.mirroredTo': 'Reflejado en: {targets}', + 'storage.mirror.replicaOf': 'Réplica de: {primaries}', + 'storage.mirror.cacheWarning': + 'No recomendado: esta categoría contiene contenido que se puede volver a obtener — replicarlo suele ser un desperdicio.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Un segundo espejo envuelve {primary} — el panel solo gestiona el primero; elimina este para gestionar el espejado desde {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Envuelve un backend definido por una variable de entorno — no editable aquí.', + 'storage.mirror.degenerate.missing-primary': 'Hace referencia a un backend que ya no existe.', + 'storage.remove.usedAsReplicaBy': 'Usado como réplica por: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sincronizar ahora', + 'storage.sync.running': 'Sincronizando… {done}/{total}', + 'storage.sync.counts': '{copied} copiados · {skipped} omitidos · {failed} fallidos', + 'storage.sync.cancel': 'Cancelar sincronización', + 'storage.sync.done': 'Sincronización finalizada: {copied} copiados, {deleted} eliminados, {failed} fallidos', + 'storage.sync.cancelled': 'Sincronización cancelada', + 'storage.sync.error': 'Error en la sincronización: {error}', + 'storage.sync.prompt': 'Los objetos existentes aún no se han replicado — ¿sincronizar ahora?', + 'storage.sync.dismiss': 'Descartar', + 'storage.usage.line': '{objects} objetos · {size}', + 'storage.usage.computed': 'Uso calculado {age}', + 'storage.usage.never': 'Uso aún no calculado', + 'storage.usage.refresh': 'Actualizar', + 'storage.usage.compute': 'Calcular ahora', + 'storage.usage.legacyNote': 'incluye la biblioteca de fotos antigua', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': '¿Mover los objetos existentes al nuevo backend?', + 'storage.migrate.promptLine': '{category}: {objects} objetos ({size}) de {from} a {to}', + 'storage.migrate.promptLineUnknown': '{category}: tamaño desconocido (uso aún no calculado) de {from} a {to}', + 'storage.migrate.move': 'Mover objetos existentes', + 'storage.migrate.routeOnly': 'Solo enrutar las escrituras nuevas', + 'storage.migrate.running': 'Moviendo {category}… {done}/{total}', + 'storage.migrate.done': 'Movimiento finalizado: {copied} copiados, {skipped} omitidos', + 'storage.migrate.doneFailures': '{failed} fallaron — esos objetos no se copiaron al nuevo backend', + 'storage.migrate.failed': 'Error al mover: {error} — la categoría no se cambió', + 'storage.migrate.cancelled': 'Movimiento cancelado — no se cambió nada', + 'storage.migrate.reclaimable': '{objects} objetos ({size}) permanecen en {from} — recupéralos manualmente', + 'storage.migrate.cancel': 'Cancelar movimiento', + 'storage.migrate.promptCancel': 'Cancelar', + 'storage.migrate.queued': 'En cola: {categories}', + 'storage.migrate.queueDropped': 'No se pudo iniciar la siguiente migración — se vació la cola restante: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/externalNotifications/types.ts b/shared/src/i18n/externalNotifications/types.ts index cdb12f0db..534a4f58f 100644 --- a/shared/src/i18n/externalNotifications/types.ts +++ b/shared/src/i18n/externalNotifications/types.ts @@ -33,6 +33,7 @@ export type NotificationEventKey = | 'collab_message' | 'packing_tagged' | 'version_available' + | 'replica_failure' | 'synology_session_cleared' | 'plugin_notification'; diff --git a/shared/src/i18n/fr/admin.ts b/shared/src/i18n/fr/admin.ts index 4439559f6..9c4f1945c 100644 --- a/shared/src/i18n/fr/admin.ts +++ b/shared/src/i18n/fr/admin.ts @@ -168,6 +168,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Erreur de sauvegarde', 'admin.tabs.addons': 'Extensions', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Stockage', 'admin.plugins.rescan': 'Réanalyser', 'admin.plugins.rescanned': 'Dossier des plugins réanalysé', 'admin.plugins.upload': 'Téléverser un plugin', diff --git a/shared/src/i18n/fr/externalNotifications.ts b/shared/src/i18n/fr/externalNotifications.ts index 9146e7d07..b733935e3 100644 --- a/shared/src/i18n/fr/externalNotifications.ts +++ b/shared/src/i18n/fr/externalNotifications.ts @@ -52,6 +52,14 @@ const fr: NotificationLocale = { title: 'Nouvelle version TREK disponible', body: `TREK ${p.version} est maintenant disponible. Rendez-vous dans le panneau d'administration pour mettre à jour.`, }), + replica_failure: (p) => ({ + title: 'Échec de réplique de stockage', + body: + `Échec de l'écriture sur la réplique '${p.backend}' : ${p.op} de ${p.key} — ${p.error}.` + + (p.suppressed !== '0' + ? ` ${p.suppressed} erreurs supplémentaires ont été supprimées depuis la dernière notification.` + : ''), + }), synology_session_cleared: () => ({ title: 'Session Synology effacée', body: 'Votre compte ou URL Synology a changé. Vous avez été déconnecté de Synology Photos.', diff --git a/shared/src/i18n/fr/index.ts b/shared/src/i18n/fr/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/fr/index.ts +++ b/shared/src/i18n/fr/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/fr/notif.ts b/shared/src/i18n/fr/notif.ts index 267100444..3b060f089 100644 --- a/shared/src/i18n/fr/notif.ts +++ b/shared/src/i18n/fr/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} vous a assigné à {category} dans {trip}', 'notif.version_available.title': 'Nouvelle version disponible', 'notif.version_available.text': 'TREK {version} est maintenant disponible', + 'notif.replica_failure.title': 'Échec de réplique de stockage', + 'notif.replica_failure.text': "Échec de l'écriture sur la réplique '{backend}' : {op} de {key} — {error}", + 'notif.replica_failure.textSuppressed': "Échec de l'écriture sur la réplique '{backend}' : {op} de {key} — {error}. {suppressed} erreurs supplémentaires ont été supprimées depuis la dernière notification.", 'notif.action.view_trip': 'Voir le voyage', 'notif.action.view_collab': 'Voir les messages', 'notif.action.view_packing': 'Voir les bagages', diff --git a/shared/src/i18n/fr/storage.ts b/shared/src/i18n/fr/storage.ts new file mode 100644 index 000000000..32683ba08 --- /dev/null +++ b/shared/src/i18n/fr/storage.ts @@ -0,0 +1,144 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Répertoire racine', + 'storage.help.root': 'Chemin absolu sur le serveur où ce backend stocke ses objets.', + 'storage.field.endpoint': 'URL du point de terminaison', + 'storage.help.endpoint': + 'URL de base du service compatible S3, par ex. https://s3.example.com ou http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': "ID de clé d'accès", + 'storage.field.secretAccessKey': "Clé d'accès secrète", + 'storage.field.region': 'Région', + 'storage.help.region': 'Conserve la valeur par défaut, sauf si ton fournisseur exige une région spécifique.', + 'storage.field.keyPrefix': 'Préfixe de clé', + 'storage.help.keyPrefix': "Préfixe optionnel ajouté à chaque clé d'objet, par ex. trek/prod.", + 'storage.field.retries': 'Tentatives', + 'storage.field.timeoutMs': "Délai d'expiration (ms)", + 'storage.field.primary': 'Backend principal', + 'storage.field.replicas': 'Répliques', + 'storage.title': 'Stockage', + 'storage.description': + "Où TREK conserve les fichiers, photos et sauvegardes envoyés. Rien ne change tant que tu n'enregistres pas.", + 'storage.loading': 'Chargement…', + 'storage.saved': 'Configuration de stockage enregistrée', + 'storage.save': 'Enregistrer les modifications', + 'storage.unsaved': 'Modifications non enregistrées', + 'storage.saveConflict': + "La configuration de stockage a changé depuis son chargement, vos modifications n'ont donc pas été enregistrées. Abandonnez-les et rechargez la configuration enregistrée pour recommencer.", + 'storage.discardAndReload': 'Abandonner mes modifications et recharger', + 'storage.configError.banner': + "Échec du chargement des paramètres de stockage enregistrés — l'enregistrement les remplacera : {error}", + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Ajouter un backend', + 'storage.backends.usedBy': 'Utilisé par : {categories}', + 'storage.backends.unused': 'Non assigné à aucune catégorie', + 'storage.backends.envReadOnly': "Défini par une variable d'environnement — lecture seule", + 'storage.source.built-in': 'Intégré', + 'storage.source.env': 'Environnement', + 'storage.source.settings': 'Paramètres', + 'storage.type.local': 'Local', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Miroir', + 'storage.actions.test': 'Tester', + 'storage.actions.edit': 'Modifier', + 'storage.actions.remove': 'Supprimer', + 'storage.test.running': 'Test en cours…', + 'storage.test.ok': 'Connexion OK', + 'storage.test.failed': 'Échec du test', + 'storage.remove.title': 'Supprimer le backend', + 'storage.remove.body': + "Supprimer {name} de la configuration ? Le serveur refuse l'enregistrement si quelque chose en dépend encore.", + 'storage.remove.stillAssigned': 'Encore assigné à : {categories}', + 'storage.form.addTitle': 'Ajouter un backend', + 'storage.form.editTitle': 'Modifier le backend', + 'storage.form.name': 'Nom', + 'storage.form.type': 'Type', + 'storage.form.apply': 'Appliquer', + 'storage.form.cancel': 'Annuler', + 'storage.form.duplicateName': 'Un backend nommé {name} existe déjà', + 'storage.categories.title': 'Catégories', + 'storage.categories.default': 'par défaut', + 'storage.categories.reassignWarning': + 'Les objets existants ne sont pas déplacés : les nouveaux objets vont vers le backend nouvellement assigné, les anciens restent où ils sont.', + 'storage.category.files': 'Documents de voyage', + 'storage.category.journey': 'Photos du journal de voyage', + 'storage.category.covers': 'Images de couverture', + 'storage.category.avatars': 'Photos de profil', + 'storage.category.places': 'Images de lieux', + 'storage.category.photos-google': 'Cache photos Google', + 'storage.category.photos-trek': 'Cache photos TREK', + 'storage.category.backups': 'Sauvegardes', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Fichiers joints envoyés dans les voyages — billets, PDF, confirmations de réservation et fichiers partagés dans le chat du voyage.', + 'storage.categoryDesc.journey': 'Photos et vignettes associées aux entrées du journal de voyage.', + 'storage.categoryDesc.covers': + 'Images de couverture des voyages et des collections, y compris celles récupérées depuis Unsplash.', + 'storage.categoryDesc.avatars': 'Photos de profil des comptes utilisateurs.', + 'storage.categoryDesc.places': 'Images associées aux lieux et aux lieux de collections — importées ou téléversées.', + 'storage.categoryDesc.photos-google': + 'Copies en cache des photos de Google Places — récupérables à nouveau, leur perte est sans risque.', + 'storage.categoryDesc.photos-trek': + 'Photos en cache du service photo de TREK utilisé par Photos (Memories) — récupérables à nouveau, leur perte est sans risque.', + 'storage.categoryDesc.backups': + 'Archives de sauvegarde du serveur créées par le panneau Sauvegarde ou par la planification.', + 'storage.health.title': 'État', + 'storage.health.allClear': 'Aucune défaillance de réplique enregistrée.', + 'storage.health.seedFile': + 'Un fichier de départ storage-config.json est présent mais ignoré — des lignes de configuration existent déjà. Gère le stockage ici.', + 'storage.health.failureLine': '{op} de {key} sur {backend} a échoué : {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Cibles du miroir', + 'storage.mirror.targetsHelp': 'Chaque écriture sur ce backend est également copiée vers chaque cible sélectionnée.', + 'storage.mirror.latencyNote': + "Les répliques sont écrites l'une après l'autre à chaque envoi — une cible lente ou inaccessible ralentit chaque envoi de chaque catégorie sur ce backend.", + 'storage.mirror.mirroredTo': 'Reflété vers : {targets}', + 'storage.mirror.replicaOf': 'Réplique de : {primaries}', + 'storage.mirror.cacheWarning': + 'Non recommandé : cette catégorie contient du contenu récupérable à nouveau — le répliquer est généralement inutile.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Un second miroir enveloppe {primary} — le panneau ne gère que le premier ; supprime celui-ci pour gérer le miroir depuis {primary}.', + 'storage.mirror.degenerate.env-primary': + "Enveloppe un backend défini par une variable d'environnement — non modifiable ici.", + 'storage.mirror.degenerate.missing-primary': "Référence un backend qui n'existe plus.", + 'storage.remove.usedAsReplicaBy': 'Utilisé comme réplique par : {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Synchroniser maintenant', + 'storage.sync.running': 'Synchronisation… {done}/{total}', + 'storage.sync.counts': '{copied} copiés · {skipped} ignorés · {failed} échoués', + 'storage.sync.cancel': 'Annuler la synchronisation', + 'storage.sync.done': 'Synchronisation terminée : {copied} copiés, {deleted} supprimés, {failed} échoués', + 'storage.sync.cancelled': 'Synchronisation annulée', + 'storage.sync.error': 'Échec de la synchronisation : {error}', + 'storage.sync.prompt': 'Les objets existants ne sont pas encore répliqués — synchroniser maintenant ?', + 'storage.sync.dismiss': 'Ignorer', + 'storage.usage.line': '{objects} objets · {size}', + 'storage.usage.computed': 'Utilisation calculée {age}', + 'storage.usage.never': 'Utilisation pas encore calculée', + 'storage.usage.refresh': 'Actualiser', + 'storage.usage.compute': 'Calculer maintenant', + 'storage.usage.legacyNote': "inclut l'ancienne bibliothèque de photos", + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Déplacer les objets existants vers le nouveau backend ?', + 'storage.migrate.promptLine': '{category} : {objects} objets ({size}) de {from} vers {to}', + 'storage.migrate.promptLineUnknown': + '{category} : taille inconnue (utilisation pas encore calculée) de {from} vers {to}', + 'storage.migrate.move': 'Déplacer les objets existants', + 'storage.migrate.routeOnly': 'Router uniquement les nouvelles écritures', + 'storage.migrate.running': 'Déplacement de {category}… {done}/{total}', + 'storage.migrate.done': 'Déplacement terminé : {copied} copiés, {skipped} ignorés', + 'storage.migrate.doneFailures': "{failed} en échec — ces objets n'ont pas été copiés vers le nouveau backend", + 'storage.migrate.failed': "Échec du déplacement : {error} — la catégorie n'a pas été changée", + 'storage.migrate.cancelled': "Déplacement annulé — rien n'a été changé", + 'storage.migrate.reclaimable': '{objects} objets ({size}) restent sur {from} — à récupérer manuellement', + 'storage.migrate.cancel': 'Annuler le déplacement', + 'storage.migrate.promptCancel': 'Annuler', + 'storage.migrate.queued': "En file d'attente : {categories}", + 'storage.migrate.queueDropped': "Impossible de démarrer le déplacement suivant — la file d'attente restante a été vidée : {categories}", +}; +export default storage; diff --git a/shared/src/i18n/gr/admin.ts b/shared/src/i18n/gr/admin.ts index 33eed1810..cdd96211e 100644 --- a/shared/src/i18n/gr/admin.ts +++ b/shared/src/i18n/gr/admin.ts @@ -213,6 +213,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Αποτυχία αποθήκευσης', 'admin.tabs.addons': 'Πρόσθετα', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Αποθήκευση', 'admin.plugins.rescan': 'Επανασάρωση', 'admin.plugins.rescanned': 'Ο φάκελος plugins σαρώθηκε ξανά', 'admin.plugins.upload': 'Μεταφόρτωση plugin', diff --git a/shared/src/i18n/gr/externalNotifications.ts b/shared/src/i18n/gr/externalNotifications.ts index 88cc8f89e..d1137c610 100644 --- a/shared/src/i18n/gr/externalNotifications.ts +++ b/shared/src/i18n/gr/externalNotifications.ts @@ -52,6 +52,12 @@ const gr: NotificationLocale = { title: 'Νέα έκδοση TREK διαθέσιμη', body: `Η έκδοση TREK ${p.version} είναι τώρα διαθέσιμη. Επισκεφθείτε τον πίνακα διαχείρισης για να ενημερώσετε.`, }), + replica_failure: (p) => ({ + title: 'Αποτυχία αντιγράφου αποθήκευσης', + body: + `Η εγγραφή στο αντίγραφο '${p.backend}' απέτυχε: ${p.op} του ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} επιπλέον αποτυχίες αποκρύφθηκαν από την τελευταία ειδοποίηση.` : ''), + }), synology_session_cleared: () => ({ title: 'Η σύνδεση Synology τερματίστηκε', body: 'Ο λογαριασμός σας Synology ή το URL άλλαξε. Έχετε αποσυνδεθεί από το Synology Photos.', diff --git a/shared/src/i18n/gr/index.ts b/shared/src/i18n/gr/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/gr/index.ts +++ b/shared/src/i18n/gr/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/gr/notif.ts b/shared/src/i18n/gr/notif.ts index 90f463d50..1f59d1490 100644 --- a/shared/src/i18n/gr/notif.ts +++ b/shared/src/i18n/gr/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': 'Ο/Η {actor} σας ανέθεσε στο {category} του {trip}', 'notif.version_available.title': 'Νέα Έκδοση Διαθέσιμη', 'notif.version_available.text': 'Το TREK {version} είναι τώρα διαθέσιμο', + 'notif.replica_failure.title': 'Αποτυχία αντιγράφου αποθήκευσης', + 'notif.replica_failure.text': "Η εγγραφή στο αντίγραφο '{backend}' απέτυχε: {op} του {key} — {error}", + 'notif.replica_failure.textSuppressed': "Η εγγραφή στο αντίγραφο '{backend}' απέτυχε: {op} του {key} — {error}. {suppressed} επιπλέον αποτυχίες αποκρύφθηκαν από την τελευταία ειδοποίηση.", 'notif.action.view_trip': 'Προβολή Ταξιδιού', 'notif.action.view_collab': 'Προβολή Μηνυμάτων', 'notif.action.view_packing': 'Προβολή Αποσκευών', diff --git a/shared/src/i18n/gr/storage.ts b/shared/src/i18n/gr/storage.ts new file mode 100644 index 000000000..ac65236a1 --- /dev/null +++ b/shared/src/i18n/gr/storage.ts @@ -0,0 +1,145 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Ριζικός κατάλογος', + 'storage.help.root': 'Απόλυτη διαδρομή στον διακομιστή όπου αυτό το backend αποθηκεύει τα αντικείμενά του.', + 'storage.field.endpoint': 'URL τελικού σημείου', + 'storage.help.endpoint': + 'Βασικό URL της υπηρεσίας συμβατής με S3, π.χ. https://s3.example.com ή http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Αναγνωριστικό κλειδιού πρόσβασης', + 'storage.field.secretAccessKey': 'Μυστικό κλειδί πρόσβασης', + 'storage.field.region': 'Περιοχή', + 'storage.help.region': 'Διατήρησε την προεπιλογή εκτός αν ο πάροχός σου απαιτεί συγκεκριμένη περιοχή.', + 'storage.field.keyPrefix': 'Πρόθεμα κλειδιού', + 'storage.help.keyPrefix': 'Προαιρετικό πρόθεμα που προστίθεται σε κάθε κλειδί αντικειμένου, π.χ. trek/prod.', + 'storage.field.retries': 'Επαναλήψεις', + 'storage.field.timeoutMs': 'Χρονικό όριο (ms)', + 'storage.field.primary': 'Κύριο backend', + 'storage.field.replicas': 'Αντίγραφα', + 'storage.title': 'Αποθήκευση', + 'storage.description': + 'Πού διατηρεί το TREK τα αρχεία, τις φωτογραφίες και τα αντίγραφα ασφαλείας που ανεβάζεις. Τίποτα δεν αλλάζει μέχρι να αποθηκεύσεις.', + 'storage.loading': 'Φόρτωση…', + 'storage.saved': 'Η ρύθμιση αποθήκευσης αποθηκεύτηκε', + 'storage.save': 'Αποθήκευση αλλαγών', + 'storage.unsaved': 'Μη αποθηκευμένες αλλαγές', + 'storage.saveConflict': + 'Η ρύθμιση αποθήκευσης άλλαξε από τη στιγμή που φορτώθηκε, επομένως οι αλλαγές σας δεν αποθηκεύτηκαν. Απορρίψτε τις και φορτώστε ξανά τις αποθηκευμένες ρυθμίσεις για να ξεκινήσετε από την αρχή.', + 'storage.discardAndReload': 'Απόρριψη των αλλαγών μου και επαναφόρτωση', + 'storage.configError.banner': + 'Η φόρτωση των αποθηκευμένων ρυθμίσεων αποθήκευσης απέτυχε — η αποθήκευση θα τις αντικαταστήσει: {error}', + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Προσθήκη backend', + 'storage.backends.usedBy': 'Χρησιμοποιείται από: {categories}', + 'storage.backends.unused': 'Δεν έχει ανατεθεί σε καμία κατηγορία', + 'storage.backends.envReadOnly': 'Ορίζεται από μεταβλητή περιβάλλοντος — μόνο για ανάγνωση', + 'storage.source.built-in': 'Ενσωματωμένο', + 'storage.source.env': 'Περιβάλλον', + 'storage.source.settings': 'Ρυθμίσεις', + 'storage.type.local': 'Τοπικό', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Καθρέφτης', + 'storage.actions.test': 'Δοκιμή', + 'storage.actions.edit': 'Επεξεργασία', + 'storage.actions.remove': 'Αφαίρεση', + 'storage.test.running': 'Δοκιμή σε εξέλιξη…', + 'storage.test.ok': 'Η σύνδεση είναι επιτυχής', + 'storage.test.failed': 'Η δοκιμή απέτυχε', + 'storage.remove.title': 'Αφαίρεση backend', + 'storage.remove.body': + 'Αφαίρεση του {name} από τη ρύθμιση; Ο διακομιστής απορρίπτει την αποθήκευση αν κάτι εξακολουθεί να εξαρτάται από αυτό.', + 'storage.remove.stillAssigned': 'Εξακολουθεί να έχει ανατεθεί σε: {categories}', + 'storage.form.addTitle': 'Προσθήκη backend', + 'storage.form.editTitle': 'Επεξεργασία backend', + 'storage.form.name': 'Όνομα', + 'storage.form.type': 'Τύπος', + 'storage.form.apply': 'Εφαρμογή', + 'storage.form.cancel': 'Ακύρωση', + 'storage.form.duplicateName': 'Υπάρχει ήδη ένα backend με το όνομα {name}', + 'storage.categories.title': 'Κατηγορίες', + 'storage.categories.default': 'προεπιλογή', + 'storage.categories.reassignWarning': + 'Τα υπάρχοντα αντικείμενα δεν μετακινούνται: τα νέα αντικείμενα πηγαίνουν στο νεοανατεθέν backend, τα παλιά παραμένουν όπου είναι.', + 'storage.category.files': 'Έγγραφα ταξιδιού', + 'storage.category.journey': 'Φωτογραφίες ταξιδιού', + 'storage.category.covers': 'Εικόνες εξωφύλλου', + 'storage.category.avatars': 'Φωτογραφίες προφίλ', + 'storage.category.places': 'Εικόνες τοποθεσιών', + 'storage.category.photos-google': 'Κρυφή μνήμη φωτογραφιών Google', + 'storage.category.photos-trek': 'Κρυφή μνήμη φωτογραφιών TREK', + 'storage.category.backups': 'Αντίγραφα ασφαλείας', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Συνημμένα αρχεία που ανεβαίνουν στα ταξίδια — εισιτήρια, PDF, επιβεβαιώσεις κράτησης και αρχεία που κοινοποιούνται στη συνομιλία του ταξιδιού.', + 'storage.categoryDesc.journey': 'Φωτογραφίες και μικρογραφίες που επισυνάπτονται σε καταχωρίσεις ταξιδιού.', + 'storage.categoryDesc.covers': + 'Εικόνες εξωφύλλου ταξιδιών και συλλογών, συμπεριλαμβανομένων εξωφύλλων από το Unsplash.', + 'storage.categoryDesc.avatars': 'Φωτογραφίες προφίλ λογαριασμών χρηστών.', + 'storage.categoryDesc.places': + 'Εικόνες που επισυνάπτονται σε τοποθεσίες και τοποθεσίες συλλογών — μεταφορτωμένες ή εισαγόμενες.', + 'storage.categoryDesc.photos-google': + 'Αποθηκευμένα αντίγραφα φωτογραφιών από το Google Places — ανακτήσιμα ξανά, η απώλειά τους είναι ασφαλής.', + 'storage.categoryDesc.photos-trek': + 'Φωτογραφίες σε κρυφή μνήμη από την υπηρεσία φωτογραφιών TREK που χρησιμοποιείται από τις Φωτογραφίες (Memories) — ανακτήσιμες ξανά, η απώλειά τους είναι ασφαλής.', + 'storage.categoryDesc.backups': + 'Αρχεία αντιγράφων ασφαλείας του διακομιστή που δημιουργούνται από τον πίνακα Αντιγράφων Ασφαλείας ή το πρόγραμμα.', + 'storage.health.title': 'Κατάσταση', + 'storage.health.allClear': 'Δεν έχουν καταγραφεί αποτυχίες αντιγράφων.', + 'storage.health.seedFile': + 'Υπάρχει αρχείο seed storage-config.json αλλά αγνοείται — υπάρχουν ήδη γραμμές ρύθμισης. Διαχειρίσου την αποθήκευση εδώ.', + 'storage.health.failureLine': '{op} του {key} στο {backend} απέτυχε: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Στόχοι καθρέφτη', + 'storage.mirror.targetsHelp': 'Κάθε εγγραφή σε αυτό το backend αντιγράφεται επίσης σε κάθε επιλεγμένο στόχο.', + 'storage.mirror.latencyNote': + 'Τα αντίγραφα γράφονται το ένα μετά το άλλο σε κάθε μεταφόρτωση — ένας αργός ή μη προσβάσιμος στόχος επιβραδύνει κάθε μεταφόρτωση κάθε κατηγορίας σε αυτό το backend.', + 'storage.mirror.mirroredTo': 'Αντικατοπτρίζεται σε: {targets}', + 'storage.mirror.replicaOf': 'Αντίγραφο του: {primaries}', + 'storage.mirror.cacheWarning': + 'Δεν συνιστάται: αυτή η κατηγορία περιέχει περιεχόμενο που μπορεί να ανακτηθεί ξανά — η αναπαραγωγή του είναι συνήθως σπατάλη.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Ένας δεύτερος καθρέφτης τυλίγει το {primary} — ο πίνακας διαχειρίζεται μόνο τον πρώτο· αφαίρεσε αυτόν για να διαχειριστείς την αντικατοπτρισμό από το {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Τυλίγει ένα backend που ορίζεται από μεταβλητή περιβάλλοντος — δεν επεξεργάζεται εδώ.', + 'storage.mirror.degenerate.missing-primary': 'Αναφέρεται σε ένα backend που δεν υπάρχει πια.', + 'storage.remove.usedAsReplicaBy': 'Χρησιμοποιείται ως αντίγραφο από: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Συγχρονισμός τώρα', + 'storage.sync.running': 'Συγχρονισμός… {done}/{total}', + 'storage.sync.counts': '{copied} αντιγράφηκαν · {skipped} παραλείφθηκαν · {failed} απέτυχαν', + 'storage.sync.cancel': 'Ακύρωση συγχρονισμού', + 'storage.sync.done': 'Ο συγχρονισμός ολοκληρώθηκε: {copied} αντιγράφηκαν, {deleted} διαγράφηκαν, {failed} απέτυχαν', + 'storage.sync.cancelled': 'Ο συγχρονισμός ακυρώθηκε', + 'storage.sync.error': 'Ο συγχρονισμός απέτυχε: {error}', + 'storage.sync.prompt': 'Τα υπάρχοντα αντικείμενα δεν έχουν αναπαραχθεί ακόμα — συγχρονισμός τώρα;', + 'storage.sync.dismiss': 'Απόρριψη', + 'storage.usage.line': '{objects} αντικείμενα · {size}', + 'storage.usage.computed': 'Η χρήση υπολογίστηκε {age}', + 'storage.usage.never': 'Η χρήση δεν έχει υπολογιστεί ακόμα', + 'storage.usage.refresh': 'Ανανέωση', + 'storage.usage.compute': 'Υπολογισμός τώρα', + 'storage.usage.legacyNote': 'περιλαμβάνει την παλιά βιβλιοθήκη φωτογραφιών', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Μετακίνηση των υπαρχόντων αντικειμένων στο νέο backend;', + 'storage.migrate.promptLine': '{category}: {objects} αντικείμενα ({size}) από {from} σε {to}', + 'storage.migrate.promptLineUnknown': + '{category}: άγνωστο μέγεθος (δεν έχει γίνει ακόμα σάρωση χρήσης) από {from} σε {to}', + 'storage.migrate.move': 'Μετακίνηση υπαρχόντων αντικειμένων', + 'storage.migrate.routeOnly': 'Δρομολόγηση μόνο των νέων εγγραφών', + 'storage.migrate.running': 'Μετακίνηση {category}… {done}/{total}', + 'storage.migrate.done': 'Η μετακίνηση ολοκληρώθηκε: {copied} αντιγράφηκαν, {skipped} παραλείφθηκαν', + 'storage.migrate.doneFailures': '{failed} απέτυχαν — αυτά τα αντικείμενα δεν αντιγράφηκαν στο νέο backend', + 'storage.migrate.failed': 'Η μετακίνηση απέτυχε: {error} — η κατηγορία δεν άλλαξε', + 'storage.migrate.cancelled': 'Η μετακίνηση ακυρώθηκε — τίποτα δεν άλλαξε', + 'storage.migrate.reclaimable': '{objects} αντικείμενα ({size}) παραμένουν στο {from} — ανάκτησέ τα χειροκίνητα', + 'storage.migrate.cancel': 'Ακύρωση μετακίνησης', + 'storage.migrate.promptCancel': 'Ακύρωση', + 'storage.migrate.queued': 'Σε αναμονή: {categories}', + 'storage.migrate.queueDropped': 'Δεν ήταν δυνατή η έναρξη της επόμενης μετακίνησης — η υπόλοιπη ουρά διαγράφηκε: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/hu/admin.ts b/shared/src/i18n/hu/admin.ts index e9c9f90d9..9a31eb110 100644 --- a/shared/src/i18n/hu/admin.ts +++ b/shared/src/i18n/hu/admin.ts @@ -167,6 +167,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Nem sikerült menteni', 'admin.tabs.addons': 'Bővítmények', 'admin.tabs.plugins': 'Pluginok', + 'admin.tabs.storage': 'Tárhely', 'admin.plugins.rescan': 'Újraszkennelés', 'admin.plugins.rescanned': 'Plugin mappa újraszkennelve', 'admin.plugins.upload': 'Plugin feltöltése', diff --git a/shared/src/i18n/hu/externalNotifications.ts b/shared/src/i18n/hu/externalNotifications.ts index 10b02b5a9..fc76e88e4 100644 --- a/shared/src/i18n/hu/externalNotifications.ts +++ b/shared/src/i18n/hu/externalNotifications.ts @@ -52,6 +52,12 @@ const hu: NotificationLocale = { title: 'Új TREK verzió érhető el', body: `A TREK ${p.version} elérhető. Látogasson el az adminisztrációs panelre a frissítéshez.`, }), + replica_failure: (p) => ({ + title: 'Tárhely-replika hiba', + body: + `Sikertelen írás a(z) '${p.backend}' replikán: ${p.op} / ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Az utolsó értesítés óta ${p.suppressed} további hiba lett elnyomva.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology munkamenet törölve', body: 'A Synology fiókja vagy URL-je megváltozott. Kijelentkeztek a Synology Photos-ból.', diff --git a/shared/src/i18n/hu/index.ts b/shared/src/i18n/hu/index.ts index 71788f536..f7465a624 100644 --- a/shared/src/i18n/hu/index.ts +++ b/shared/src/i18n/hu/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/hu/notif.ts b/shared/src/i18n/hu/notif.ts index ee4e8a5be..a119f4278 100644 --- a/shared/src/i18n/hu/notif.ts +++ b/shared/src/i18n/hu/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} hozzárendelte Önt a {category} kategóriához a(z) {trip} utazásban', 'notif.version_available.title': 'Új verzió elérhető', 'notif.version_available.text': 'A TREK {version} elérhető', + 'notif.replica_failure.title': 'Tárhely-replika hiba', + 'notif.replica_failure.text': "Sikertelen írás a(z) '{backend}' replikán: {op} / {key} — {error}", + 'notif.replica_failure.textSuppressed': "Sikertelen írás a(z) '{backend}' replikán: {op} / {key} — {error}. Az utolsó értesítés óta {suppressed} további hiba lett elnyomva.", 'notif.action.view_trip': 'Utazás megtekintése', 'notif.action.view_collab': 'Üzenetek megtekintése', 'notif.action.view_packing': 'Csomagolás megtekintése', diff --git a/shared/src/i18n/hu/storage.ts b/shared/src/i18n/hu/storage.ts new file mode 100644 index 000000000..a9cc66c02 --- /dev/null +++ b/shared/src/i18n/hu/storage.ts @@ -0,0 +1,143 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Gyökérkönyvtár', + 'storage.help.root': 'Abszolút elérési út a szerveren, ahol ez a backend tárolja az objektumait.', + 'storage.field.endpoint': 'Végpont URL', + 'storage.help.endpoint': + 'Az S3-kompatibilis szolgáltatás alap URL-je, pl. https://s3.example.com vagy http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Hozzáférési kulcs azonosító', + 'storage.field.secretAccessKey': 'Titkos hozzáférési kulcs', + 'storage.field.region': 'Régió', + 'storage.help.region': 'Hagyd az alapértelmezettet, hacsak a szolgáltatód nem igényel konkrét régiót.', + 'storage.field.keyPrefix': 'Kulcs-előtag', + 'storage.help.keyPrefix': 'Opcionális előtag, amely minden objektumkulcs elé kerül, pl. trek/prod.', + 'storage.field.retries': 'Újrapróbálkozások', + 'storage.field.timeoutMs': 'Időtúllépés (ms)', + 'storage.field.primary': 'Elsődleges backend', + 'storage.field.replicas': 'Replikák', + 'storage.title': 'Tárhely', + 'storage.description': + 'Itt tárolja a TREK a feltöltött fájlokat, fotókat és biztonsági mentéseket. Semmi sem változik, amíg nem mentesz.', + 'storage.loading': 'Betöltés…', + 'storage.saved': 'A tárhely-konfiguráció mentve', + 'storage.save': 'Módosítások mentése', + 'storage.unsaved': 'Nem mentett módosítások', + 'storage.saveConflict': + 'A tárhely-konfiguráció megváltozott a betöltése óta, ezért a módosításai nem lettek mentve. Vesse el őket, és töltse be újra a mentett beállításokat az újrakezdéshez.', + 'storage.discardAndReload': 'Módosításaim elvetése és újratöltés', + 'storage.configError.banner': + 'A mentett tárhely-beállítások betöltése nem sikerült — a mentés felülírja őket: {error}', + 'storage.backends.title': 'Backendek', + 'storage.backends.add': 'Backend hozzáadása', + 'storage.backends.usedBy': 'Használja: {categories}', + 'storage.backends.unused': 'Nincs kategóriához rendelve', + 'storage.backends.envReadOnly': 'Környezeti változó definiálja — csak olvasható', + 'storage.source.built-in': 'Beépített', + 'storage.source.env': 'Környezet', + 'storage.source.settings': 'Beállítások', + 'storage.type.local': 'Helyi', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Tükör', + 'storage.actions.test': 'Tesztelés', + 'storage.actions.edit': 'Szerkesztés', + 'storage.actions.remove': 'Eltávolítás', + 'storage.test.running': 'Tesztelés…', + 'storage.test.ok': 'A kapcsolat rendben', + 'storage.test.failed': 'A teszt sikertelen', + 'storage.remove.title': 'Backend eltávolítása', + 'storage.remove.body': + 'Eltávolítod a(z) {name} elemet a konfigurációból? A szerver elutasítja a mentést, ha még bármi függ tőle.', + 'storage.remove.stillAssigned': 'Még mindig hozzá van rendelve: {categories}', + 'storage.form.addTitle': 'Backend hozzáadása', + 'storage.form.editTitle': 'Backend szerkesztése', + 'storage.form.name': 'Név', + 'storage.form.type': 'Típus', + 'storage.form.apply': 'Alkalmaz', + 'storage.form.cancel': 'Mégse', + 'storage.form.duplicateName': 'Már létezik {name} nevű backend', + 'storage.categories.title': 'Kategóriák', + 'storage.categories.default': 'alapértelmezett', + 'storage.categories.reassignWarning': + 'A meglévő objektumok nem mozdulnak: az új objektumok az újonnan hozzárendelt backendbe kerülnek, a régiek ott maradnak, ahol vannak.', + 'storage.category.files': 'Útidokumentumok', + 'storage.category.journey': 'Útinapló-fotók', + 'storage.category.covers': 'Borítóképek', + 'storage.category.avatars': 'Profilképek', + 'storage.category.places': 'Helyképek', + 'storage.category.photos-google': 'Google fotó gyorsítótár', + 'storage.category.photos-trek': 'TREK fotó gyorsítótár', + 'storage.category.backups': 'Biztonsági mentések', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Az utazásokhoz feltöltött fájlmellékletek — jegyek, PDF-ek, foglalási visszaigazolások és az útcsevegésben megosztott fájlok.', + 'storage.categoryDesc.journey': 'Az útinapló bejegyzésekhez csatolt fotók és bélyegképek.', + 'storage.categoryDesc.covers': 'Utazások és gyűjtemények borítóképei, beleértve az Unsplashről lekért borítókat is.', + 'storage.categoryDesc.avatars': 'Felhasználói fiókok profilképei.', + 'storage.categoryDesc.places': 'Helyekhez és gyűjteményhelyekhez csatolt képek — feltöltve vagy importálva.', + 'storage.categoryDesc.photos-google': + 'A Google Places fotóinak gyorsítótárazott másolatai — újra lekérhetők, biztonságosan elveszíthetők.', + 'storage.categoryDesc.photos-trek': + 'A Fotók (Memories) funkció által használt TREK fotószolgáltatás gyorsítótárazott fotói — újra lekérhetők, biztonságosan elveszíthetők.', + 'storage.categoryDesc.backups': + 'A Biztonsági mentés panel vagy az ütemezés által létrehozott szerver-mentési archívumok.', + 'storage.health.title': 'Állapot', + 'storage.health.allClear': 'Nincs rögzített replikahiba.', + 'storage.health.seedFile': + 'Egy storage-config.json seed fájl jelen van, de figyelmen kívül marad — konfigurációs sorok már léteznek. A tárhelyet itt kezelheted.', + 'storage.health.failureLine': '{op} sikertelen: {key} ezen: {backend} — {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Tükör célok', + 'storage.mirror.targetsHelp': 'Az ebbe a backendbe történő minden írás minden kiválasztott célra is átmásolódik.', + 'storage.mirror.latencyNote': + 'A replikák egymás után íródnak minden feltöltés során — egy lassú vagy elérhetetlen cél lelassítja e backend minden kategóriájának minden feltöltését.', + 'storage.mirror.mirroredTo': 'Tükrözve ide: {targets}', + 'storage.mirror.replicaOf': 'Replika innen: {primaries}', + 'storage.mirror.cacheWarning': + 'Nem ajánlott: ez a kategória újra lekérhető tartalmat tartalmaz — a replikálása általában felesleges.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Egy második tükör veszi körül a(z) {primary} elemet — a panel csak az elsőt kezeli; távolítsd el ezt, hogy a tükrözést a(z) {primary} elemtől kezelhesd.', + 'storage.mirror.degenerate.env-primary': + 'Egy környezeti változóval definiált backendet vesz körül — itt nem szerkeszthető.', + 'storage.mirror.degenerate.missing-primary': 'Egy már nem létező backendre hivatkozik.', + 'storage.remove.usedAsReplicaBy': 'Replikaként használja: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Szinkronizálás most', + 'storage.sync.running': 'Szinkronizálás… {done}/{total}', + 'storage.sync.counts': '{copied} másolva · {skipped} kihagyva · {failed} sikertelen', + 'storage.sync.cancel': 'Szinkronizálás megszakítása', + 'storage.sync.done': 'Szinkronizálás befejezve: {copied} másolva, {deleted} törölve, {failed} sikertelen', + 'storage.sync.cancelled': 'Szinkronizálás megszakítva', + 'storage.sync.error': 'A szinkronizálás sikertelen: {error}', + 'storage.sync.prompt': 'A meglévő objektumok még nincsenek replikálva — szinkronizálsz most?', + 'storage.sync.dismiss': 'Elvetés', + 'storage.usage.line': '{objects} objektum · {size}', + 'storage.usage.computed': 'A használat kiszámítva {age}', + 'storage.usage.never': 'A használat még nincs kiszámítva', + 'storage.usage.refresh': 'Frissítés', + 'storage.usage.compute': 'Számítás most', + 'storage.usage.legacyNote': 'tartalmazza a régi fotókönyvtárat', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Áthelyezed a meglévő objektumokat az új backendre?', + 'storage.migrate.promptLine': '{category}: {objects} objektum ({size}) innen: {from} ide: {to}', + 'storage.migrate.promptLineUnknown': + '{category}: ismeretlen méret (a használat még nincs kiszámítva) innen: {from} ide: {to}', + 'storage.migrate.move': 'Meglévő objektumok áthelyezése', + 'storage.migrate.routeOnly': 'Csak az új írások irányítása', + 'storage.migrate.running': '{category} áthelyezése… {done}/{total}', + 'storage.migrate.done': 'Áthelyezés kész: {copied} másolva, {skipped} kihagyva', + 'storage.migrate.doneFailures': '{failed} sikertelen — ezek az objektumok nem lettek átmásolva az új backendre', + 'storage.migrate.failed': 'Az áthelyezés sikertelen: {error} — a kategória nem lett átváltva', + 'storage.migrate.cancelled': 'Áthelyezés megszakítva — semmi sem lett átváltva', + 'storage.migrate.reclaimable': '{objects} objektum ({size}) marad a(z) {from} helyen — kézzel szabadítsd fel', + 'storage.migrate.cancel': 'Áthelyezés megszakítása', + 'storage.migrate.promptCancel': 'Mégse', + 'storage.migrate.queued': 'Várólistán: {categories}', + 'storage.migrate.queueDropped': 'A következő áthelyezés nem indítható el — a hátralévő várólista törlésre került: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/id/admin.ts b/shared/src/i18n/id/admin.ts index 96af5b525..7e6e3eb47 100644 --- a/shared/src/i18n/id/admin.ts +++ b/shared/src/i18n/id/admin.ts @@ -209,6 +209,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Gagal menyimpan', 'admin.tabs.addons': 'Addon', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Penyimpanan', 'admin.plugins.rescan': 'Pindai ulang', 'admin.plugins.rescanned': 'Folder plugin dipindai ulang', 'admin.plugins.upload': 'Unggah plugin', diff --git a/shared/src/i18n/id/externalNotifications.ts b/shared/src/i18n/id/externalNotifications.ts index e40667766..161a15a9a 100644 --- a/shared/src/i18n/id/externalNotifications.ts +++ b/shared/src/i18n/id/externalNotifications.ts @@ -52,6 +52,12 @@ const id: NotificationLocale = { title: 'Versi TREK baru tersedia', body: `TREK ${p.version} sekarang tersedia. Kunjungi panel admin untuk memperbarui.`, }), + replica_failure: (p) => ({ + title: 'Kegagalan replika penyimpanan', + body: + `Penulisan replika gagal pada '${p.backend}': ${p.op} dari ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} kegagalan lainnya diabaikan sejak notifikasi terakhir.` : ''), + }), synology_session_cleared: () => ({ title: 'Sesi Synology dihapus', body: 'Akun atau URL Synology Anda berubah. Anda telah keluar dari Synology Photos.', diff --git a/shared/src/i18n/id/index.ts b/shared/src/i18n/id/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/id/index.ts +++ b/shared/src/i18n/id/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/id/notif.ts b/shared/src/i18n/id/notif.ts index 841f53948..1d7fbf56c 100644 --- a/shared/src/i18n/id/notif.ts +++ b/shared/src/i18n/id/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} menugaskanmu ke {category} di {trip}', 'notif.version_available.title': 'Versi Baru Tersedia', 'notif.version_available.text': 'TREK {version} kini tersedia', + 'notif.replica_failure.title': 'Kegagalan replika penyimpanan', + 'notif.replica_failure.text': "Penulisan replika gagal pada '{backend}': {op} dari {key} — {error}", + 'notif.replica_failure.textSuppressed': "Penulisan replika gagal pada '{backend}': {op} dari {key} — {error}. {suppressed} kegagalan lainnya diabaikan sejak notifikasi terakhir.", 'notif.action.view_trip': 'Lihat Perjalanan', 'notif.action.view_collab': 'Lihat Pesan', 'notif.action.view_packing': 'Lihat Perlengkapan', diff --git a/shared/src/i18n/id/storage.ts b/shared/src/i18n/id/storage.ts new file mode 100644 index 000000000..e126dd65e --- /dev/null +++ b/shared/src/i18n/id/storage.ts @@ -0,0 +1,141 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Direktori root', + 'storage.help.root': 'Jalur absolut di server tempat backend ini menyimpan objeknya.', + 'storage.field.endpoint': 'URL endpoint', + 'storage.help.endpoint': + 'URL dasar layanan yang kompatibel dengan S3, mis. https://s3.example.com atau http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID kunci akses', + 'storage.field.secretAccessKey': 'Kunci akses rahasia', + 'storage.field.region': 'Wilayah', + 'storage.help.region': 'Biarkan default kecuali penyedia Anda memerlukan wilayah tertentu.', + 'storage.field.keyPrefix': 'Awalan kunci', + 'storage.help.keyPrefix': 'Awalan opsional yang ditambahkan ke setiap kunci objek, mis. trek/prod.', + 'storage.field.retries': 'Percobaan ulang', + 'storage.field.timeoutMs': 'Batas waktu (md)', + 'storage.field.primary': 'Backend utama', + 'storage.field.replicas': 'Replika', + 'storage.title': 'Penyimpanan', + 'storage.description': + 'Tempat TREK menyimpan file, foto, dan cadangan yang diunggah. Tidak ada yang berubah sampai Anda menyimpan.', + 'storage.loading': 'Memuat…', + 'storage.saved': 'Konfigurasi penyimpanan disimpan', + 'storage.save': 'Simpan perubahan', + 'storage.unsaved': 'Perubahan belum disimpan', + 'storage.saveConflict': + 'Konfigurasi penyimpanan berubah sejak dimuat, jadi perubahan Anda tidak tersimpan. Buang perubahan itu dan muat ulang pengaturan tersimpan untuk mulai lagi.', + 'storage.discardAndReload': 'Buang perubahan saya dan muat ulang', + 'storage.configError.banner': 'Gagal memuat pengaturan penyimpanan tersimpan — menyimpan akan menggantinya: {error}', + 'storage.backends.title': 'Backend', + 'storage.backends.add': 'Tambah backend', + 'storage.backends.usedBy': 'Digunakan oleh: {categories}', + 'storage.backends.unused': 'Tidak ditetapkan ke kategori mana pun', + 'storage.backends.envReadOnly': 'Ditentukan oleh variabel lingkungan — hanya baca', + 'storage.source.built-in': 'Bawaan', + 'storage.source.env': 'Lingkungan', + 'storage.source.settings': 'Pengaturan', + 'storage.type.local': 'Lokal', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Cermin', + 'storage.actions.test': 'Uji', + 'storage.actions.edit': 'Sunting', + 'storage.actions.remove': 'Hapus', + 'storage.test.running': 'Menguji…', + 'storage.test.ok': 'Koneksi OK', + 'storage.test.failed': 'Uji gagal', + 'storage.remove.title': 'Hapus backend', + 'storage.remove.body': + 'Hapus {name} dari konfigurasi? Server menolak penyimpanan jika masih ada yang bergantung padanya.', + 'storage.remove.stillAssigned': 'Masih ditetapkan ke: {categories}', + 'storage.form.addTitle': 'Tambah backend', + 'storage.form.editTitle': 'Sunting backend', + 'storage.form.name': 'Nama', + 'storage.form.type': 'Tipe', + 'storage.form.apply': 'Terapkan', + 'storage.form.cancel': 'Batal', + 'storage.form.duplicateName': 'Backend bernama {name} sudah ada', + 'storage.categories.title': 'Kategori', + 'storage.categories.default': 'default', + 'storage.categories.reassignWarning': + 'Objek yang ada tidak dipindahkan: objek baru masuk ke backend yang baru ditetapkan, yang lama tetap di tempatnya.', + 'storage.category.files': 'Dokumen perjalanan', + 'storage.category.journey': 'Foto Journey', + 'storage.category.covers': 'Gambar sampul', + 'storage.category.avatars': 'Foto profil', + 'storage.category.places': 'Gambar tempat', + 'storage.category.photos-google': 'Cache foto Google', + 'storage.category.photos-trek': 'Cache foto TREK', + 'storage.category.backups': 'Cadangan', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Lampiran berkas yang diunggah ke perjalanan — tiket, PDF, konfirmasi pemesanan, dan berkas yang dibagikan di obrolan perjalanan.', + 'storage.categoryDesc.journey': 'Foto dan thumbnail yang dilampirkan ke entri Journey.', + 'storage.categoryDesc.covers': 'Gambar sampul perjalanan dan koleksi, termasuk sampul yang diambil dari Unsplash.', + 'storage.categoryDesc.avatars': 'Foto profil akun pengguna.', + 'storage.categoryDesc.places': 'Gambar yang dilampirkan ke tempat dan tempat koleksi — diunggah atau diimpor.', + 'storage.categoryDesc.photos-google': + 'Salinan foto Google Places yang di-cache — dapat diambil ulang, aman jika hilang.', + 'storage.categoryDesc.photos-trek': + 'Foto ter-cache dari layanan foto TREK yang digunakan oleh Foto (Memories) — dapat diambil ulang, aman jika hilang.', + 'storage.categoryDesc.backups': 'Arsip cadangan server yang dibuat oleh panel Cadangan atau jadwal.', + 'storage.health.title': 'Kesehatan', + 'storage.health.allClear': 'Tidak ada kegagalan replika yang tercatat.', + 'storage.health.seedFile': + 'File seed storage-config.json ada tetapi diabaikan — baris konfigurasi sudah ada. Kelola penyimpanan di sini.', + 'storage.health.failureLine': '{op} pada {key} di {backend} gagal: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Target cermin', + 'storage.mirror.targetsHelp': 'Setiap penulisan ke backend ini juga disalin ke setiap target yang dipilih.', + 'storage.mirror.latencyNote': + 'Replika ditulis satu per satu selama setiap unggahan — target yang lambat atau tidak terjangkau memperlambat setiap unggahan setiap kategori pada backend ini.', + 'storage.mirror.mirroredTo': 'Dicerminkan ke: {targets}', + 'storage.mirror.replicaOf': 'Replika dari: {primaries}', + 'storage.mirror.cacheWarning': + 'Tidak disarankan: kategori ini menyimpan konten yang dapat diambil ulang — mereplikasinya biasanya sia-sia.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Cermin kedua membungkus {primary} — panel hanya mengelola yang pertama; hapus yang ini untuk mengelola pencerminan dari {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Membungkus backend yang ditentukan oleh variabel lingkungan — tidak dapat diedit di sini.', + 'storage.mirror.degenerate.missing-primary': 'Merujuk ke backend yang sudah tidak ada.', + 'storage.remove.usedAsReplicaBy': 'Digunakan sebagai replika oleh: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sinkronkan sekarang', + 'storage.sync.running': 'Menyinkronkan… {done}/{total}', + 'storage.sync.counts': '{copied} disalin · {skipped} dilewati · {failed} gagal', + 'storage.sync.cancel': 'Batalkan sinkronisasi', + 'storage.sync.done': 'Sinkronisasi selesai: {copied} disalin, {deleted} dihapus, {failed} gagal', + 'storage.sync.cancelled': 'Sinkronisasi dibatalkan', + 'storage.sync.error': 'Sinkronisasi gagal: {error}', + 'storage.sync.prompt': 'Objek yang ada belum direplikasi — sinkronkan sekarang?', + 'storage.sync.dismiss': 'Abaikan', + 'storage.usage.line': '{objects} objek · {size}', + 'storage.usage.computed': 'Penggunaan dihitung {age}', + 'storage.usage.never': 'Penggunaan belum dihitung', + 'storage.usage.refresh': 'Segarkan', + 'storage.usage.compute': 'Hitung sekarang', + 'storage.usage.legacyNote': 'termasuk pustaka foto lama', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Pindahkan objek yang ada ke backend baru?', + 'storage.migrate.promptLine': '{category}: {objects} objek ({size}) dari {from} ke {to}', + 'storage.migrate.promptLineUnknown': + '{category}: ukuran tidak diketahui (penggunaan belum dipindai) dari {from} ke {to}', + 'storage.migrate.move': 'Pindahkan objek yang ada', + 'storage.migrate.routeOnly': 'Hanya arahkan penulisan baru', + 'storage.migrate.running': 'Memindahkan {category}… {done}/{total}', + 'storage.migrate.done': 'Pemindahan selesai: {copied} disalin, {skipped} dilewati', + 'storage.migrate.doneFailures': '{failed} gagal — objek tersebut tidak disalin ke backend baru', + 'storage.migrate.failed': 'Pemindahan gagal: {error} — kategori tidak diubah', + 'storage.migrate.cancelled': 'Pemindahan dibatalkan — tidak ada yang diubah', + 'storage.migrate.reclaimable': '{objects} objek ({size}) masih tersisa di {from} — ambil kembali secara manual', + 'storage.migrate.cancel': 'Batalkan pemindahan', + 'storage.migrate.promptCancel': 'Batal', + 'storage.migrate.queued': 'Dalam antrean: {categories}', + 'storage.migrate.queueDropped': 'Tidak dapat memulai pemindahan berikutnya — antrean yang tersisa telah dihapus: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/it/admin.ts b/shared/src/i18n/it/admin.ts index 8d7c468a8..8f3f2e79e 100644 --- a/shared/src/i18n/it/admin.ts +++ b/shared/src/i18n/it/admin.ts @@ -166,6 +166,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Impossibile salvare', 'admin.tabs.addons': 'Moduli', 'admin.tabs.plugins': 'Plugin', + 'admin.tabs.storage': 'Archiviazione', 'admin.plugins.rescan': 'Riscansiona', 'admin.plugins.rescanned': 'Cartella dei plugin riscansionata', 'admin.plugins.upload': 'Carica plugin', diff --git a/shared/src/i18n/it/externalNotifications.ts b/shared/src/i18n/it/externalNotifications.ts index 29fc136f4..912ccb312 100644 --- a/shared/src/i18n/it/externalNotifications.ts +++ b/shared/src/i18n/it/externalNotifications.ts @@ -52,6 +52,12 @@ const it: NotificationLocale = { title: 'Nuova versione TREK disponibile', body: `TREK ${p.version} è ora disponibile. Visita il pannello di amministrazione per aggiornare.`, }), + replica_failure: (p) => ({ + title: "Errore di replica dell'archiviazione", + body: + `Scrittura sulla replica non riuscita su '${p.backend}': ${p.op} di ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Altri ${p.suppressed} errori sono stati soppressi dall'ultima notifica.` : ''), + }), synology_session_cleared: () => ({ title: 'Sessione Synology rimossa', body: 'Il tuo account o URL Synology è cambiato. Sei stato disconnesso da Synology Photos.', diff --git a/shared/src/i18n/it/index.ts b/shared/src/i18n/it/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/it/index.ts +++ b/shared/src/i18n/it/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/it/notif.ts b/shared/src/i18n/it/notif.ts index 5278aae7b..d7d6696fe 100644 --- a/shared/src/i18n/it/notif.ts +++ b/shared/src/i18n/it/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} ti ha assegnato a {category} in {trip}', 'notif.version_available.title': 'Nuova versione disponibile', 'notif.version_available.text': 'TREK {version} è ora disponibile', + 'notif.replica_failure.title': "Errore di replica dell'archiviazione", + 'notif.replica_failure.text': "Scrittura sulla replica non riuscita su '{backend}': {op} di {key} — {error}", + 'notif.replica_failure.textSuppressed': "Scrittura sulla replica non riuscita su '{backend}': {op} di {key} — {error}. Altri {suppressed} errori sono stati soppressi dall'ultima notifica.", 'notif.action.view_trip': 'Vedi viaggio', 'notif.action.view_collab': 'Vedi messaggi', 'notif.action.view_packing': 'Vedi bagagli', diff --git a/shared/src/i18n/it/storage.ts b/shared/src/i18n/it/storage.ts new file mode 100644 index 000000000..ec2e86bee --- /dev/null +++ b/shared/src/i18n/it/storage.ts @@ -0,0 +1,143 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Directory principale', + 'storage.help.root': 'Percorso assoluto sul server in cui questo backend memorizza i propri oggetti.', + 'storage.field.endpoint': 'URL endpoint', + 'storage.help.endpoint': + 'URL di base del servizio compatibile con S3, es. https://s3.example.com o http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID chiave di accesso', + 'storage.field.secretAccessKey': 'Chiave di accesso segreta', + 'storage.field.region': 'Regione', + 'storage.help.region': + 'Mantieni il valore predefinito a meno che il tuo provider non richieda una regione specifica.', + 'storage.field.keyPrefix': 'Prefisso chiave', + 'storage.help.keyPrefix': 'Prefisso facoltativo aggiunto a ogni chiave oggetto, es. trek/prod.', + 'storage.field.retries': 'Tentativi', + 'storage.field.timeoutMs': 'Timeout (ms)', + 'storage.field.primary': 'Backend primario', + 'storage.field.replicas': 'Repliche', + 'storage.title': 'Archiviazione', + 'storage.description': 'Dove TREK conserva i file caricati, le foto e i backup. Nulla cambia finché non salvi.', + 'storage.loading': 'Caricamento…', + 'storage.saved': 'Configurazione di archiviazione salvata', + 'storage.save': 'Salva modifiche', + 'storage.unsaved': 'Modifiche non salvate', + 'storage.saveConflict': + 'La configurazione di archiviazione è cambiata da quando è stata caricata, quindi le tue modifiche non sono state salvate. Scartale e ricarica la configurazione salvata per ricominciare.', + 'storage.discardAndReload': 'Scarta le mie modifiche e ricarica', + 'storage.configError.banner': + 'Impossibile caricare le impostazioni di archiviazione salvate — il salvataggio le sostituirà: {error}', + 'storage.backends.title': 'Backend', + 'storage.backends.add': 'Aggiungi backend', + 'storage.backends.usedBy': 'Usato da: {categories}', + 'storage.backends.unused': 'Non assegnato a nessuna categoria', + 'storage.backends.envReadOnly': "Definito da una variabile d'ambiente — sola lettura", + 'storage.source.built-in': 'Integrato', + 'storage.source.env': 'Ambiente', + 'storage.source.settings': 'Impostazioni', + 'storage.type.local': 'Locale', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Mirror', + 'storage.actions.test': 'Testa', + 'storage.actions.edit': 'Modifica', + 'storage.actions.remove': 'Rimuovi', + 'storage.test.running': 'Test in corso…', + 'storage.test.ok': 'Connessione riuscita', + 'storage.test.failed': 'Test non riuscito', + 'storage.remove.title': 'Rimuovi backend', + 'storage.remove.body': + 'Rimuovere {name} dalla configurazione? Il server rifiuta il salvataggio se qualcosa dipende ancora da esso.', + 'storage.remove.stillAssigned': 'Ancora assegnato a: {categories}', + 'storage.form.addTitle': 'Aggiungi backend', + 'storage.form.editTitle': 'Modifica backend', + 'storage.form.name': 'Nome', + 'storage.form.type': 'Tipo', + 'storage.form.apply': 'Applica', + 'storage.form.cancel': 'Annulla', + 'storage.form.duplicateName': 'Esiste già un backend chiamato {name}', + 'storage.categories.title': 'Categorie', + 'storage.categories.default': 'predefinita', + 'storage.categories.reassignWarning': + 'Gli oggetti esistenti non vengono spostati: i nuovi oggetti vanno al backend appena assegnato, quelli vecchi restano dove sono.', + 'storage.category.files': 'Documenti di viaggio', + 'storage.category.journey': 'Foto del diario di viaggio', + 'storage.category.covers': 'Immagini di copertina', + 'storage.category.avatars': 'Foto profilo', + 'storage.category.places': 'Immagini dei luoghi', + 'storage.category.photos-google': 'Cache foto di Google', + 'storage.category.photos-trek': 'Cache foto TREK', + 'storage.category.backups': 'Backup', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Allegati caricati nei viaggi — biglietti, PDF, conferme di prenotazione e file condivisi nella chat del viaggio.', + 'storage.categoryDesc.journey': 'Foto e miniature allegate alle voci del diario di viaggio.', + 'storage.categoryDesc.covers': 'Immagini di copertina di viaggi e raccolte, incluse quelle recuperate da Unsplash.', + 'storage.categoryDesc.avatars': 'Foto profilo degli account utente.', + 'storage.categoryDesc.places': 'Immagini allegate ai luoghi e ai luoghi delle raccolte — caricate o importate.', + 'storage.categoryDesc.photos-google': + 'Copie in cache delle foto di Google Places — recuperabili di nuovo, perderle è sicuro.', + 'storage.categoryDesc.photos-trek': + 'Foto in cache dal servizio foto TREK usato da Foto (Memories) — recuperabili di nuovo, perderle è sicuro.', + 'storage.categoryDesc.backups': 'Archivi di backup del server creati dal pannello Backup o dalla pianificazione.', + 'storage.health.title': 'Stato', + 'storage.health.allClear': 'Nessun errore di replica registrato.', + 'storage.health.seedFile': + "È presente un file seed storage-config.json ma viene ignorato — esistono già righe di configurazione. Gestisci l'archiviazione qui.", + 'storage.health.failureLine': '{op} di {key} su {backend} non riuscito: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Destinazioni mirror', + 'storage.mirror.targetsHelp': + 'Ogni scrittura su questo backend viene copiata anche su ogni destinazione selezionata.', + 'storage.mirror.latencyNote': + "Le repliche vengono scritte una dopo l'altra durante ogni caricamento — una destinazione lenta o irraggiungibile rallenta ogni caricamento di ogni categoria su questo backend.", + 'storage.mirror.mirroredTo': 'Rispecchiato su: {targets}', + 'storage.mirror.replicaOf': 'Replica di: {primaries}', + 'storage.mirror.cacheWarning': + 'Non consigliato: questa categoria contiene contenuti riottenibili — replicarli è di solito uno spreco.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Un secondo mirror avvolge {primary} — il pannello gestisce solo il primo; rimuovi questo per gestire il mirroring da {primary}.', + 'storage.mirror.degenerate.env-primary': + "Avvolge un backend definito da una variabile d'ambiente — non modificabile qui.", + 'storage.mirror.degenerate.missing-primary': 'Fa riferimento a un backend che non esiste più.', + 'storage.remove.usedAsReplicaBy': 'Usato come replica da: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Sincronizza ora', + 'storage.sync.running': 'Sincronizzazione… {done}/{total}', + 'storage.sync.counts': '{copied} copiati · {skipped} saltati · {failed} falliti', + 'storage.sync.cancel': 'Annulla sincronizzazione', + 'storage.sync.done': 'Sincronizzazione completata: {copied} copiati, {deleted} eliminati, {failed} falliti', + 'storage.sync.cancelled': 'Sincronizzazione annullata', + 'storage.sync.error': 'Sincronizzazione non riuscita: {error}', + 'storage.sync.prompt': 'Gli oggetti esistenti non sono ancora replicati — sincronizzare ora?', + 'storage.sync.dismiss': 'Ignora', + 'storage.usage.line': '{objects} oggetti · {size}', + 'storage.usage.computed': 'Utilizzo calcolato {age}', + 'storage.usage.never': 'Utilizzo non ancora calcolato', + 'storage.usage.refresh': 'Aggiorna', + 'storage.usage.compute': 'Calcola ora', + 'storage.usage.legacyNote': 'include la libreria foto precedente', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Spostare gli oggetti esistenti nel nuovo backend?', + 'storage.migrate.promptLine': '{category}: {objects} oggetti ({size}) da {from} a {to}', + 'storage.migrate.promptLineUnknown': + '{category}: dimensione sconosciuta (utilizzo non ancora calcolato) da {from} a {to}', + 'storage.migrate.move': 'Sposta oggetti esistenti', + 'storage.migrate.routeOnly': 'Instrada solo le nuove scritture', + 'storage.migrate.running': 'Spostamento di {category}… {done}/{total}', + 'storage.migrate.done': 'Spostamento completato: {copied} copiati, {skipped} saltati', + 'storage.migrate.doneFailures': '{failed} non riusciti — questi oggetti non sono stati copiati nel nuovo backend', + 'storage.migrate.failed': 'Spostamento non riuscito: {error} — la categoria non è stata cambiata', + 'storage.migrate.cancelled': 'Spostamento annullato — nulla è stato cambiato', + 'storage.migrate.reclaimable': '{objects} oggetti ({size}) restano su {from} — recuperali manualmente', + 'storage.migrate.cancel': 'Annulla spostamento', + 'storage.migrate.promptCancel': 'Annulla', + 'storage.migrate.queued': 'In coda: {categories}', + 'storage.migrate.queueDropped': 'Impossibile avviare lo spostamento successivo — la coda rimanente è stata svuotata: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/ja/admin.ts b/shared/src/i18n/ja/admin.ts index b53b05713..fefd15f7d 100644 --- a/shared/src/i18n/ja/admin.ts +++ b/shared/src/i18n/ja/admin.ts @@ -198,6 +198,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': '保存に失敗しました', 'admin.tabs.addons': 'アドオン', 'admin.tabs.plugins': 'プラグイン', + 'admin.tabs.storage': 'ストレージ', 'admin.plugins.rescan': '再スキャン', 'admin.plugins.rescanned': 'プラグインフォルダを再スキャンしました', 'admin.plugins.upload': 'プラグインをアップロード', diff --git a/shared/src/i18n/ja/externalNotifications.ts b/shared/src/i18n/ja/externalNotifications.ts index 7b5a91957..ffc93d14d 100644 --- a/shared/src/i18n/ja/externalNotifications.ts +++ b/shared/src/i18n/ja/externalNotifications.ts @@ -52,6 +52,12 @@ const ja: NotificationLocale = { title: '新しいTREKバージョンが利用可能', body: `TREK ${p.version}が利用可能になりました。管理パネルからアップデートしてください。`, }), + replica_failure: (p) => ({ + title: 'ストレージレプリカの障害', + body: + `レプリカ '${p.backend}' への書き込みに失敗しました：${p.op} / ${p.key} — ${p.error}。` + + (p.suppressed !== '0' ? `前回の通知以降、${p.suppressed}件の追加の失敗が抑制されました。` : ''), + }), synology_session_cleared: () => ({ title: 'Synologyセッションがクリアされました', body: 'SynologyアカウントまたはURLが変更されました。Synology Photosからログアウトされました。', diff --git a/shared/src/i18n/ja/index.ts b/shared/src/i18n/ja/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/ja/index.ts +++ b/shared/src/i18n/ja/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/ja/notif.ts b/shared/src/i18n/ja/notif.ts index 58a72cf73..bda4aa6fa 100644 --- a/shared/src/i18n/ja/notif.ts +++ b/shared/src/i18n/ja/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor}が「{trip}」の{category}をあなたに割り当てました', 'notif.version_available.title': '新しいバージョンがあります', 'notif.version_available.text': 'TREK {version}が利用可能です', + 'notif.replica_failure.title': 'ストレージレプリカの障害', + 'notif.replica_failure.text': "レプリカ '{backend}' への書き込みに失敗しました：{op} / {key} — {error}", + 'notif.replica_failure.textSuppressed': "レプリカ '{backend}' への書き込みに失敗しました：{op} / {key} — {error}。前回の通知以降、{suppressed}件の追加の失敗が抑制されました。", 'notif.action.view_trip': '旅行を見る', 'notif.action.view_collab': 'メッセージを見る', 'notif.action.view_packing': '持ち物を見る', diff --git a/shared/src/i18n/ja/storage.ts b/shared/src/i18n/ja/storage.ts new file mode 100644 index 000000000..b06cae880 --- /dev/null +++ b/shared/src/i18n/ja/storage.ts @@ -0,0 +1,139 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'ルートディレクトリ', + 'storage.help.root': 'このバックエンドがオブジェクトを保存するサーバー上の絶対パス。', + 'storage.field.endpoint': 'エンドポイントURL', + 'storage.help.endpoint': 'S3互換サービスのベースURL。例: https://s3.example.com または http://127.0.0.1:9000。', + 'storage.field.bucket': 'バケット', + 'storage.field.accessKeyId': 'アクセスキーID', + 'storage.field.secretAccessKey': 'シークレットアクセスキー', + 'storage.field.region': 'リージョン', + 'storage.help.region': 'プロバイダーが特定のリージョンを要求しない限り、デフォルトのままにしてください。', + 'storage.field.keyPrefix': 'キープレフィックス', + 'storage.help.keyPrefix': '各オブジェクトキーに追加するオプションのプレフィックス。例: trek/prod。', + 'storage.field.retries': 'リトライ回数', + 'storage.field.timeoutMs': 'タイムアウト（ms）', + 'storage.field.primary': 'プライマリバックエンド', + 'storage.field.replicas': 'レプリカ', + 'storage.title': 'ストレージ', + 'storage.description': + 'TREKがアップロードされたファイル、写真、バックアップを保存する場所です。保存するまで何も変更されません。', + 'storage.loading': '読み込み中…', + 'storage.saved': 'ストレージ設定を保存しました', + 'storage.save': '変更を保存', + 'storage.unsaved': '未保存の変更', + 'storage.saveConflict': + 'ストレージ設定は読み込み後に変更されたため、変更内容は保存されませんでした。変更を破棄して保存済みの設定を再読み込みし、やり直してください。', + 'storage.discardAndReload': '変更を破棄して再読み込み', + 'storage.configError.banner': + '保存済みのストレージ設定の読み込みに失敗しました — 保存すると置き換えられます: {error}', + 'storage.backends.title': 'バックエンド', + 'storage.backends.add': 'バックエンドを追加', + 'storage.backends.usedBy': '使用中: {categories}', + 'storage.backends.unused': 'どのカテゴリにも割り当てられていません', + 'storage.backends.envReadOnly': '環境変数で定義されています — 読み取り専用', + 'storage.source.built-in': '組み込み', + 'storage.source.env': '環境変数', + 'storage.source.settings': '設定', + 'storage.type.local': 'ローカル', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'ミラー', + 'storage.actions.test': 'テスト', + 'storage.actions.edit': '編集', + 'storage.actions.remove': '削除', + 'storage.test.running': 'テスト中…', + 'storage.test.ok': '接続OK', + 'storage.test.failed': 'テスト失敗', + 'storage.remove.title': 'バックエンドを削除', + 'storage.remove.body': '{name}を設定から削除しますか？何かがまだ依存している場合、サーバーは保存を拒否します。', + 'storage.remove.stillAssigned': 'まだ割り当てられています: {categories}', + 'storage.form.addTitle': 'バックエンドを追加', + 'storage.form.editTitle': 'バックエンドを編集', + 'storage.form.name': '名前', + 'storage.form.type': 'タイプ', + 'storage.form.apply': '適用', + 'storage.form.cancel': 'キャンセル', + 'storage.form.duplicateName': '{name}という名前のバックエンドは既に存在します', + 'storage.categories.title': 'カテゴリ', + 'storage.categories.default': 'デフォルト', + 'storage.categories.reassignWarning': + '既存のオブジェクトは移動しません。新しいオブジェクトは新しく割り当てられたバックエンドに保存され、古いオブジェクトはそのまま残ります。', + 'storage.category.files': '旅行の書類', + 'storage.category.journey': '日記の写真', + 'storage.category.covers': 'カバー画像', + 'storage.category.avatars': 'プロフィール写真', + 'storage.category.places': '場所の画像', + 'storage.category.photos-google': 'Googleフォトキャッシュ', + 'storage.category.photos-trek': 'TREKフォトキャッシュ', + 'storage.category.backups': 'バックアップ', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + '旅行にアップロードされたファイル添付 — チケット、PDF、予約確認書、旅行チャットで共有されたファイル。', + 'storage.categoryDesc.journey': '日記のエントリーに添付された写真とサムネイル。', + 'storage.categoryDesc.covers': '旅行やコレクションのカバー画像。Unsplashから取得したカバーを含む。', + 'storage.categoryDesc.avatars': 'ユーザーアカウントのプロフィール写真。', + 'storage.categoryDesc.places': '場所やコレクションの場所に添付された画像 — アップロードまたはインポートされたもの。', + 'storage.categoryDesc.photos-google': + 'Google Placesの写真のキャッシュコピー — 再取得可能で、失っても問題ありません。', + 'storage.categoryDesc.photos-trek': + '「写真」機能（Memories）が使用するTREKフォトサービスのキャッシュ写真 — 再取得可能で、失っても問題ありません。', + 'storage.categoryDesc.backups': + 'バックアップパネルまたはスケジュールによって作成されたサーバーのバックアップアーカイブ。', + 'storage.health.title': 'ヘルス', + 'storage.health.allClear': 'レプリカの失敗は記録されていません。', + 'storage.health.seedFile': + 'storage-config.jsonのシードファイルが存在しますが無視されます — 設定行は既に存在します。ここでストレージを管理してください。', + 'storage.health.failureLine': '{backend}での{key}の{op}が失敗しました: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'ミラー先', + 'storage.mirror.targetsHelp': 'このバックエンドへの書き込みは、選択した各ターゲットにもコピーされます。', + 'storage.mirror.latencyNote': + 'レプリカは各アップロード中に順番に書き込まれます — 遅いターゲットや到達不能なターゲットがあると、このバックエンド上のすべてのカテゴリのすべてのアップロードが遅くなります。', + 'storage.mirror.mirroredTo': 'ミラー先: {targets}', + 'storage.mirror.replicaOf': 'レプリカ元: {primaries}', + 'storage.mirror.cacheWarning': '非推奨: このカテゴリは再取得可能なコンテンツを保持しています — 複製は通常無駄です。', + 'storage.mirror.degenerate.duplicate-mirror': + '2つ目のミラーが{primary}をラップしています — パネルは最初のものだけを管理します。{primary}からのミラーリングを管理するには、これを削除してください。', + 'storage.mirror.degenerate.env-primary': + '環境変数で定義されたバックエンドをラップしています — ここでは編集できません。', + 'storage.mirror.degenerate.missing-primary': 'もう存在しないバックエンドを参照しています。', + 'storage.remove.usedAsReplicaBy': 'レプリカとして使用中: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': '今すぐ同期', + 'storage.sync.running': '同期中… {done}/{total}', + 'storage.sync.counts': '{copied}件コピー · {skipped}件スキップ · {failed}件失敗', + 'storage.sync.cancel': '同期をキャンセル', + 'storage.sync.done': '同期完了: {copied}件コピー、{deleted}件削除、{failed}件失敗', + 'storage.sync.cancelled': '同期をキャンセルしました', + 'storage.sync.error': '同期に失敗しました: {error}', + 'storage.sync.prompt': '既存のオブジェクトはまだ複製されていません — 今すぐ同期しますか？', + 'storage.sync.dismiss': '閉じる', + 'storage.usage.line': '{objects}個のオブジェクト · {size}', + 'storage.usage.computed': '使用量計算済み: {age}', + 'storage.usage.never': '使用量はまだ計算されていません', + 'storage.usage.refresh': '更新', + 'storage.usage.compute': '今すぐ計算', + 'storage.usage.legacyNote': '従来の写真ライブラリを含む', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': '既存のオブジェクトを新しいバックエンドに移動しますか？', + 'storage.migrate.promptLine': '{category}: {from}から{to}へ {objects}件のオブジェクト（{size}）', + 'storage.migrate.promptLineUnknown': '{category}: サイズ不明（使用量スキャン未実施）{from}から{to}へ', + 'storage.migrate.move': '既存のオブジェクトを移動', + 'storage.migrate.routeOnly': '新規書き込みのみ切り替える', + 'storage.migrate.running': '{category}を移動中… {done}/{total}', + 'storage.migrate.done': '移動完了: {copied}件コピー、{skipped}件スキップ', + 'storage.migrate.doneFailures': '{failed}件失敗 — それらのオブジェクトは新しいバックエンドにコピーされませんでした', + 'storage.migrate.failed': '移動に失敗しました: {error} — カテゴリは切り替えられませんでした', + 'storage.migrate.cancelled': '移動をキャンセルしました — 何も切り替えられていません', + 'storage.migrate.reclaimable': '{objects}件のオブジェクト（{size}）が{from}に残っています — 手動で回収してください', + 'storage.migrate.cancel': '移動をキャンセル', + 'storage.migrate.promptCancel': 'キャンセル', + 'storage.migrate.queued': 'キュー待ち: {categories}', + 'storage.migrate.queueDropped': '次の移動を開始できませんでした — 残りのキューはクリアされました: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/ko/admin.ts b/shared/src/i18n/ko/admin.ts index ea5f4df7a..9ad49d963 100644 --- a/shared/src/i18n/ko/admin.ts +++ b/shared/src/i18n/ko/admin.ts @@ -204,6 +204,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': '저장 실패', 'admin.tabs.addons': '애드온', 'admin.tabs.plugins': '플러그인', + 'admin.tabs.storage': '스토리지', 'admin.plugins.rescan': '다시 스캔', 'admin.plugins.rescanned': '플러그인 폴더를 다시 스캔했습니다', 'admin.plugins.upload': '플러그인 업로드', diff --git a/shared/src/i18n/ko/externalNotifications.ts b/shared/src/i18n/ko/externalNotifications.ts index 5209cfc27..e7dfb02d5 100644 --- a/shared/src/i18n/ko/externalNotifications.ts +++ b/shared/src/i18n/ko/externalNotifications.ts @@ -52,6 +52,12 @@ const ko: NotificationLocale = { title: '새 TREK 버전 사용 가능', body: `TREK ${p.version}을 사용할 수 있습니다. 관리자 패널에서 업데이트하세요.`, }), + replica_failure: (p) => ({ + title: '스토리지 복제본 오류', + body: + `복제본 '${p.backend}'에 쓰기가 실패했습니다: ${p.op} / ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? `마지막 알림 이후 ${p.suppressed}개의 추가 실패가 억제되었습니다.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology 세션이 초기화되었습니다', body: 'Synology 계정 또는 URL이 변경되었습니다. Synology Photos에서 로그아웃되었습니다.', diff --git a/shared/src/i18n/ko/index.ts b/shared/src/i18n/ko/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/ko/index.ts +++ b/shared/src/i18n/ko/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/ko/notif.ts b/shared/src/i18n/ko/notif.ts index 339095447..5fac76899 100644 --- a/shared/src/i18n/ko/notif.ts +++ b/shared/src/i18n/ko/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor}이(가) {trip}의 {category}에 배정했습니다', 'notif.version_available.title': '새 버전 사용 가능', 'notif.version_available.text': 'TREK {version}이(가) 사용 가능합니다', + 'notif.replica_failure.title': '스토리지 복제본 오류', + 'notif.replica_failure.text': "복제본 '{backend}'에 쓰기가 실패했습니다: {op} / {key} — {error}", + 'notif.replica_failure.textSuppressed': "복제본 '{backend}'에 쓰기가 실패했습니다: {op} / {key} — {error}.마지막 알림 이후 {suppressed}개의 추가 실패가 억제되었습니다.", 'notif.action.view_trip': '여행 보기', 'notif.action.view_collab': '메시지 보기', 'notif.action.view_packing': '짐 목록 보기', diff --git a/shared/src/i18n/ko/storage.ts b/shared/src/i18n/ko/storage.ts new file mode 100644 index 000000000..bbbd830e9 --- /dev/null +++ b/shared/src/i18n/ko/storage.ts @@ -0,0 +1,136 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': '루트 디렉터리', + 'storage.help.root': '이 백엔드가 객체를 저장하는 서버의 절대 경로입니다.', + 'storage.field.endpoint': '엔드포인트 URL', + 'storage.help.endpoint': 'S3 호환 서비스의 기본 URL. 예: https://s3.example.com 또는 http://127.0.0.1:9000.', + 'storage.field.bucket': '버킷', + 'storage.field.accessKeyId': '액세스 키 ID', + 'storage.field.secretAccessKey': '비밀 액세스 키', + 'storage.field.region': '리전', + 'storage.help.region': '제공업체가 특정 리전을 요구하지 않는 한 기본값을 유지하세요.', + 'storage.field.keyPrefix': '키 접두사', + 'storage.help.keyPrefix': '모든 객체 키에 추가되는 선택적 접두사입니다. 예: trek/prod.', + 'storage.field.retries': '재시도 횟수', + 'storage.field.timeoutMs': '제한 시간(ms)', + 'storage.field.primary': '기본 백엔드', + 'storage.field.replicas': '복제본', + 'storage.title': '스토리지', + 'storage.description': + 'TREK이 업로드된 파일, 사진, 백업을 보관하는 곳입니다. 저장하기 전까지는 아무것도 변경되지 않습니다.', + 'storage.loading': '로딩 중…', + 'storage.saved': '스토리지 설정이 저장되었습니다', + 'storage.save': '변경 사항 저장', + 'storage.unsaved': '저장되지 않은 변경 사항', + 'storage.saveConflict': + '스토리지 설정이 불러온 이후 변경되어 변경 사항이 저장되지 않았습니다. 변경 사항을 버리고 저장된 설정을 다시 불러와 처음부터 시작하세요.', + 'storage.discardAndReload': '변경 사항 버리고 다시 불러오기', + 'storage.configError.banner': '저장된 스토리지 설정을 불러오지 못했습니다 — 저장하면 대체됩니다: {error}', + 'storage.backends.title': '백엔드', + 'storage.backends.add': '백엔드 추가', + 'storage.backends.usedBy': '사용 중: {categories}', + 'storage.backends.unused': '어떤 카테고리에도 할당되지 않음', + 'storage.backends.envReadOnly': '환경 변수로 정의됨 — 읽기 전용', + 'storage.source.built-in': '내장', + 'storage.source.env': '환경', + 'storage.source.settings': '설정', + 'storage.type.local': '로컬', + 'storage.type.s3': 'S3', + 'storage.type.mirror': '미러', + 'storage.actions.test': '테스트', + 'storage.actions.edit': '편집', + 'storage.actions.remove': '제거', + 'storage.test.running': '테스트 중…', + 'storage.test.ok': '연결 성공', + 'storage.test.failed': '테스트 실패', + 'storage.remove.title': '백엔드 제거', + 'storage.remove.body': '구성에서 {name}을(를) 제거할까요? 아직 이것에 의존하는 것이 있으면 서버가 저장을 거부합니다.', + 'storage.remove.stillAssigned': '여전히 할당됨: {categories}', + 'storage.form.addTitle': '백엔드 추가', + 'storage.form.editTitle': '백엔드 편집', + 'storage.form.name': '이름', + 'storage.form.type': '유형', + 'storage.form.apply': '적용', + 'storage.form.cancel': '취소', + 'storage.form.duplicateName': '{name}이라는 이름의 백엔드가 이미 존재합니다', + 'storage.categories.title': '카테고리', + 'storage.categories.default': '기본값', + 'storage.categories.reassignWarning': + '기존 객체는 이동하지 않습니다. 새 객체는 새로 할당된 백엔드로 이동하고, 기존 객체는 있던 곳에 그대로 남습니다.', + 'storage.category.files': '여행 문서', + 'storage.category.journey': 'Journey 사진', + 'storage.category.covers': '커버 이미지', + 'storage.category.avatars': '프로필 사진', + 'storage.category.places': '장소 이미지', + 'storage.category.photos-google': 'Google 사진 캐시', + 'storage.category.photos-trek': 'TREK 사진 캐시', + 'storage.category.backups': '백업', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': '여행에 업로드된 파일 첨부 — 티켓, PDF, 예약 확인서, 여행 채팅에서 공유된 파일.', + 'storage.categoryDesc.journey': 'Journey 항목에 첨부된 사진과 썸네일.', + 'storage.categoryDesc.covers': 'Unsplash에서 가져온 커버를 포함한 여행 및 컬렉션 커버 이미지.', + 'storage.categoryDesc.avatars': '사용자 계정 프로필 사진.', + 'storage.categoryDesc.places': '장소 및 컬렉션 장소에 첨부된 이미지 — 업로드되었거나 가져온 것.', + 'storage.categoryDesc.photos-google': + 'Google Places 사진의 캐시된 사본 — 다시 가져올 수 있으므로 손실되어도 안전합니다.', + 'storage.categoryDesc.photos-trek': + '사진(Memories) 기능이 사용하는 TREK 사진 서비스의 캐시된 사진 — 다시 가져올 수 있으므로 손실되어도 안전합니다.', + 'storage.categoryDesc.backups': '백업 패널 또는 일정에 의해 생성된 서버 백업 아카이브.', + 'storage.health.title': '상태', + 'storage.health.allClear': '기록된 복제본 오류가 없습니다.', + 'storage.health.seedFile': + 'storage-config.json 시드 파일이 있지만 무시됩니다 — 구성 행이 이미 존재합니다. 여기에서 스토리지를 관리하세요.', + 'storage.health.failureLine': '{backend}에서 {key}의 {op}이(가) 실패했습니다: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': '미러 대상', + 'storage.mirror.targetsHelp': '이 백엔드에 대한 모든 쓰기는 선택된 각 대상에도 복사됩니다.', + 'storage.mirror.latencyNote': + '복제본은 업로드할 때마다 순차적으로 기록됩니다 — 느리거나 접근할 수 없는 대상이 있으면 이 백엔드의 모든 카테고리의 모든 업로드가 느려집니다.', + 'storage.mirror.mirroredTo': '미러링 대상: {targets}', + 'storage.mirror.replicaOf': '복제본 출처: {primaries}', + 'storage.mirror.cacheWarning': + '권장하지 않음: 이 카테고리는 다시 가져올 수 있는 콘텐츠를 포함합니다 — 복제하는 것은 대개 낭비입니다.', + 'storage.mirror.degenerate.duplicate-mirror': + '두 번째 미러가 {primary}을(를) 감싸고 있습니다 — 패널은 첫 번째만 관리합니다. {primary}에서 미러링을 관리하려면 이것을 제거하세요.', + 'storage.mirror.degenerate.env-primary': '환경 변수로 정의된 백엔드를 감싸고 있습니다 — 여기서는 편집할 수 없습니다.', + 'storage.mirror.degenerate.missing-primary': '더 이상 존재하지 않는 백엔드를 참조합니다.', + 'storage.remove.usedAsReplicaBy': '다음의 복제본으로 사용됨: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': '지금 동기화', + 'storage.sync.running': '동기화 중… {done}/{total}', + 'storage.sync.counts': '{copied}개 복사 · {skipped}개 건너뜀 · {failed}개 실패', + 'storage.sync.cancel': '동기화 취소', + 'storage.sync.done': '동기화 완료: {copied}개 복사, {deleted}개 삭제, {failed}개 실패', + 'storage.sync.cancelled': '동기화가 취소되었습니다', + 'storage.sync.error': '동기화 실패: {error}', + 'storage.sync.prompt': '기존 객체가 아직 복제되지 않았습니다 — 지금 동기화할까요?', + 'storage.sync.dismiss': '닫기', + 'storage.usage.line': '{objects}개 객체 · {size}', + 'storage.usage.computed': '사용량 계산됨 {age}', + 'storage.usage.never': '아직 사용량이 계산되지 않았습니다', + 'storage.usage.refresh': '새로고침', + 'storage.usage.compute': '지금 계산', + 'storage.usage.legacyNote': '기존 사진 라이브러리 포함', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': '기존 객체를 새 백엔드로 이동할까요?', + 'storage.migrate.promptLine': '{category}: {from}에서 {to}(으)로 {objects}개 객체 ({size})', + 'storage.migrate.promptLineUnknown': '{category}: 크기 알 수 없음 (아직 사용량 스캔 안 됨) {from}에서 {to}(으)로', + 'storage.migrate.move': '기존 객체 이동', + 'storage.migrate.routeOnly': '새 쓰기만 전환', + 'storage.migrate.running': '{category} 이동 중… {done}/{total}', + 'storage.migrate.done': '이동 완료: {copied}개 복사, {skipped}개 건너뜀', + 'storage.migrate.doneFailures': '{failed}개 실패 — 해당 객체는 새 백엔드로 복사되지 않았습니다', + 'storage.migrate.failed': '이동 실패: {error} — 카테고리가 전환되지 않았습니다', + 'storage.migrate.cancelled': '이동이 취소되었습니다 — 아무것도 전환되지 않았습니다', + 'storage.migrate.reclaimable': '{objects}개 객체 ({size})가 {from}에 남아 있습니다 — 수동으로 회수하세요', + 'storage.migrate.cancel': '이동 취소', + 'storage.migrate.promptCancel': '취소', + 'storage.migrate.queued': '대기 중: {categories}', + 'storage.migrate.queueDropped': '다음 이동을 시작할 수 없습니다 — 남은 대기열이 지워졌습니다: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/nl/admin.ts b/shared/src/i18n/nl/admin.ts index 7f779b8b8..ccd4cd437 100644 --- a/shared/src/i18n/nl/admin.ts +++ b/shared/src/i18n/nl/admin.ts @@ -167,6 +167,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Fout bij opslaan', 'admin.tabs.addons': 'Add-ons', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Opslag', 'admin.plugins.rescan': 'Opnieuw scannen', 'admin.plugins.rescanned': 'Pluginmap opnieuw gescand', 'admin.plugins.upload': 'Plugin uploaden', diff --git a/shared/src/i18n/nl/externalNotifications.ts b/shared/src/i18n/nl/externalNotifications.ts index 357214326..496873744 100644 --- a/shared/src/i18n/nl/externalNotifications.ts +++ b/shared/src/i18n/nl/externalNotifications.ts @@ -52,6 +52,12 @@ const nl: NotificationLocale = { title: 'Nieuwe TREK-versie beschikbaar', body: `TREK ${p.version} is nu beschikbaar. Bezoek het beheerderspaneel om bij te werken.`, }), + replica_failure: (p) => ({ + title: 'Opslagreplica mislukt', + body: + `Schrijven naar replica '${p.backend}' is mislukt: ${p.op} van ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} extra fout(en) zijn onderdrukt sinds de laatste melding.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology-sessie gewist', body: 'Je Synology-account of URL is gewijzigd. Je bent uitgelogd bij Synology Photos.', diff --git a/shared/src/i18n/nl/index.ts b/shared/src/i18n/nl/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/nl/index.ts +++ b/shared/src/i18n/nl/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/nl/notif.ts b/shared/src/i18n/nl/notif.ts index 33c15be5e..7d4e7002f 100644 --- a/shared/src/i18n/nl/notif.ts +++ b/shared/src/i18n/nl/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} heeft je toegewezen aan {category} in {trip}', 'notif.version_available.title': 'Nieuwe versie beschikbaar', 'notif.version_available.text': 'TREK {version} is nu beschikbaar', + 'notif.replica_failure.title': 'Opslagreplica mislukt', + 'notif.replica_failure.text': "Schrijven naar replica '{backend}' is mislukt: {op} van {key} — {error}", + 'notif.replica_failure.textSuppressed': "Schrijven naar replica '{backend}' is mislukt: {op} van {key} — {error}. {suppressed} extra fout(en) zijn onderdrukt sinds de laatste melding.", 'notif.action.view_trip': 'Reis bekijken', 'notif.action.view_collab': 'Berichten bekijken', 'notif.action.view_packing': 'Paklijst bekijken', diff --git a/shared/src/i18n/nl/storage.ts b/shared/src/i18n/nl/storage.ts new file mode 100644 index 000000000..9b575de20 --- /dev/null +++ b/shared/src/i18n/nl/storage.ts @@ -0,0 +1,142 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Hoofdmap', + 'storage.help.root': 'Absoluut pad op de server waar deze backend zijn objecten opslaat.', + 'storage.field.endpoint': 'Endpoint-URL', + 'storage.help.endpoint': + 'Basis-URL van de S3-compatibele service, bijv. https://s3.example.com of http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Toegangssleutel-ID', + 'storage.field.secretAccessKey': 'Geheime toegangssleutel', + 'storage.field.region': 'Regio', + 'storage.help.region': 'Laat de standaard staan tenzij je provider een specifieke regio vereist.', + 'storage.field.keyPrefix': 'Sleutelvoorvoegsel', + 'storage.help.keyPrefix': 'Optioneel voorvoegsel toegevoegd aan elke objectsleutel, bijv. trek/prod.', + 'storage.field.retries': 'Nieuwe pogingen', + 'storage.field.timeoutMs': 'Time-out (ms)', + 'storage.field.primary': 'Primaire backend', + 'storage.field.replicas': "Replica's", + 'storage.title': 'Opslag', + 'storage.description': + "Waar TREK geüploade bestanden, foto's en back-ups bewaart. Er verandert niets totdat je opslaat.", + 'storage.loading': 'Laden…', + 'storage.saved': 'Opslagconfiguratie opgeslagen', + 'storage.save': 'Wijzigingen opslaan', + 'storage.unsaved': 'Niet-opgeslagen wijzigingen', + 'storage.saveConflict': + 'De opslagconfiguratie is gewijzigd sinds deze werd geladen, dus je wijzigingen zijn niet opgeslagen. Verwerp ze en laad de opgeslagen instellingen opnieuw om opnieuw te beginnen.', + 'storage.discardAndReload': 'Mijn wijzigingen verwerpen en opnieuw laden', + 'storage.configError.banner': + 'Het laden van de opgeslagen opslaginstellingen is mislukt — opslaan zal ze vervangen: {error}', + 'storage.backends.title': 'Backends', + 'storage.backends.add': 'Backend toevoegen', + 'storage.backends.usedBy': 'Gebruikt door: {categories}', + 'storage.backends.unused': 'Niet toegewezen aan een categorie', + 'storage.backends.envReadOnly': 'Gedefinieerd door een omgevingsvariabele — alleen-lezen', + 'storage.source.built-in': 'Ingebouwd', + 'storage.source.env': 'Omgeving', + 'storage.source.settings': 'Instellingen', + 'storage.type.local': 'Lokaal', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Mirror', + 'storage.actions.test': 'Testen', + 'storage.actions.edit': 'Bewerken', + 'storage.actions.remove': 'Verwijderen', + 'storage.test.running': 'Bezig met testen…', + 'storage.test.ok': 'Verbinding OK', + 'storage.test.failed': 'Test mislukt', + 'storage.remove.title': 'Backend verwijderen', + 'storage.remove.body': + '{name} verwijderen uit de configuratie? De server weigert het opslaan als er nog iets van afhankelijk is.', + 'storage.remove.stillAssigned': 'Nog toegewezen aan: {categories}', + 'storage.form.addTitle': 'Backend toevoegen', + 'storage.form.editTitle': 'Backend bewerken', + 'storage.form.name': 'Naam', + 'storage.form.type': 'Type', + 'storage.form.apply': 'Toepassen', + 'storage.form.cancel': 'Annuleren', + 'storage.form.duplicateName': 'Er bestaat al een backend met de naam {name}', + 'storage.categories.title': 'Categorieën', + 'storage.categories.default': 'standaard', + 'storage.categories.reassignWarning': + 'Bestaande objecten worden niet verplaatst: nieuwe objecten gaan naar de nieuw toegewezen backend, oude blijven waar ze zijn.', + 'storage.category.files': 'Reisdocumenten', + 'storage.category.journey': "Reisverslag-foto's", + 'storage.category.covers': 'Omslagafbeeldingen', + 'storage.category.avatars': "Profielfoto's", + 'storage.category.places': 'Locatieafbeeldingen', + 'storage.category.photos-google': 'Google-fotocache', + 'storage.category.photos-trek': 'TREK-fotocache', + 'storage.category.backups': 'Back-ups', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + "Bestandsbijlagen die naar reizen zijn geüpload — tickets, PDF's, boekingsbevestigingen en bestanden gedeeld in de reischat.", + 'storage.categoryDesc.journey': "Foto's en miniaturen die zijn gekoppeld aan reisverslag-items.", + 'storage.categoryDesc.covers': + 'Omslagafbeeldingen van reizen en collecties, inclusief omslagen opgehaald van Unsplash.', + 'storage.categoryDesc.avatars': "Profielfoto's van gebruikersaccounts.", + 'storage.categoryDesc.places': 'Afbeeldingen gekoppeld aan locaties en collectielocaties — geüpload of geïmporteerd.', + 'storage.categoryDesc.photos-google': + "Gecachte kopieën van Google Places-foto's — opnieuw op te halen, veilig om te verliezen.", + 'storage.categoryDesc.photos-trek': + "Gecachte foto's van de TREK-fotoservice die wordt gebruikt door Foto's (Memories) — opnieuw op te halen, veilig om te verliezen.", + 'storage.categoryDesc.backups': 'Serverback-uparchieven aangemaakt door het back-uppaneel of de planning.', + 'storage.health.title': 'Status', + 'storage.health.allClear': 'Geen replicafouten geregistreerd.', + 'storage.health.seedFile': + 'Er is een storage-config.json seedbestand aanwezig maar dit wordt genegeerd — configuratierijen bestaan al. Beheer de opslag hier.', + 'storage.health.failureLine': '{op} van {key} op {backend} mislukt: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Mirrordoelen', + 'storage.mirror.targetsHelp': 'Elke schrijfactie naar deze backend wordt ook gekopieerd naar elk geselecteerd doel.', + 'storage.mirror.latencyNote': + "Replica's worden bij elke upload één voor één geschreven — een traag of onbereikbaar doel vertraagt elke upload van elke categorie op deze backend.", + 'storage.mirror.mirroredTo': 'Gespiegeld naar: {targets}', + 'storage.mirror.replicaOf': 'Replica van: {primaries}', + 'storage.mirror.cacheWarning': + 'Niet aanbevolen: deze categorie bevat opnieuw ophaalbare inhoud — repliceren is meestal verspilling.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Een tweede mirror omhult {primary} — het paneel beheert alleen de eerste; verwijder deze om spiegeling vanaf {primary} te beheren.', + 'storage.mirror.degenerate.env-primary': + 'Omhult een backend die door een omgevingsvariabele is gedefinieerd — hier niet bewerkbaar.', + 'storage.mirror.degenerate.missing-primary': 'Verwijst naar een backend die niet meer bestaat.', + 'storage.remove.usedAsReplicaBy': 'Gebruikt als replica door: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Nu synchroniseren', + 'storage.sync.running': 'Synchroniseren… {done}/{total}', + 'storage.sync.counts': '{copied} gekopieerd · {skipped} overgeslagen · {failed} mislukt', + 'storage.sync.cancel': 'Synchronisatie annuleren', + 'storage.sync.done': 'Synchronisatie voltooid: {copied} gekopieerd, {deleted} verwijderd, {failed} mislukt', + 'storage.sync.cancelled': 'Synchronisatie geannuleerd', + 'storage.sync.error': 'Synchronisatie mislukt: {error}', + 'storage.sync.prompt': 'Bestaande objecten zijn nog niet gerepliceerd — nu synchroniseren?', + 'storage.sync.dismiss': 'Negeren', + 'storage.usage.line': '{objects} objecten · {size}', + 'storage.usage.computed': 'Gebruik berekend {age}', + 'storage.usage.never': 'Gebruik nog niet berekend', + 'storage.usage.refresh': 'Vernieuwen', + 'storage.usage.compute': 'Nu berekenen', + 'storage.usage.legacyNote': 'inclusief de oude fotobibliotheek', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Bestaande objecten naar de nieuwe backend verplaatsen?', + 'storage.migrate.promptLine': '{category}: {objects} objecten ({size}) van {from} naar {to}', + 'storage.migrate.promptLineUnknown': '{category}: onbekende grootte (gebruik nog niet gescand) van {from} naar {to}', + 'storage.migrate.move': 'Bestaande objecten verplaatsen', + 'storage.migrate.routeOnly': 'Alleen nieuwe schrijfacties omleiden', + 'storage.migrate.running': '{category} verplaatsen… {done}/{total}', + 'storage.migrate.done': 'Verplaatsen voltooid: {copied} gekopieerd, {skipped} overgeslagen', + 'storage.migrate.doneFailures': '{failed} mislukt — die objecten zijn niet gekopieerd naar de nieuwe backend', + 'storage.migrate.failed': 'Verplaatsen mislukt: {error} — de categorie is niet omgezet', + 'storage.migrate.cancelled': 'Verplaatsen geannuleerd — er is niets omgezet', + 'storage.migrate.reclaimable': '{objects} objecten ({size}) blijven op {from} — handmatig terugwinnen', + 'storage.migrate.cancel': 'Verplaatsen annuleren', + 'storage.migrate.promptCancel': 'Annuleren', + 'storage.migrate.queued': 'In wachtrij: {categories}', + 'storage.migrate.queueDropped': 'Kon de volgende verplaatsing niet starten — de resterende wachtrij is gewist: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/pl/admin.ts b/shared/src/i18n/pl/admin.ts index 92fd96fa7..d0c49e62c 100644 --- a/shared/src/i18n/pl/admin.ts +++ b/shared/src/i18n/pl/admin.ts @@ -160,6 +160,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Nie udało się zapisać szablonu', 'admin.tabs.addons': 'Dodatki', 'admin.tabs.plugins': 'Wtyczki', + 'admin.tabs.storage': 'Magazyn', 'admin.plugins.rescan': 'Skanuj ponownie', 'admin.plugins.rescanned': 'Folder wtyczek został ponownie przeskanowany', 'admin.plugins.upload': 'Prześlij wtyczkę', diff --git a/shared/src/i18n/pl/externalNotifications.ts b/shared/src/i18n/pl/externalNotifications.ts index 84d8fa48f..68f1e91b8 100644 --- a/shared/src/i18n/pl/externalNotifications.ts +++ b/shared/src/i18n/pl/externalNotifications.ts @@ -52,6 +52,12 @@ const pl: NotificationLocale = { title: 'Nowa wersja TREK dostępna', body: `TREK ${p.version} jest teraz dostępny. Odwiedź panel administracyjny, aby zaktualizować.`, }), + replica_failure: (p) => ({ + title: 'Awaria repliki magazynu', + body: + `Zapis do repliki '${p.backend}' nie powiódł się: ${p.op} dla ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Od ostatniego powiadomienia ukryto ${p.suppressed} kolejnych błędów.` : ''), + }), synology_session_cleared: () => ({ title: 'Sesja Synology wyczyszczona', body: 'Twoje konto lub URL Synology uległo zmianie. Zostałeś wylogowany z Synology Photos.', diff --git a/shared/src/i18n/pl/index.ts b/shared/src/i18n/pl/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/pl/index.ts +++ b/shared/src/i18n/pl/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/pl/notif.ts b/shared/src/i18n/pl/notif.ts index c6811b4a3..eadfc2aa2 100644 --- a/shared/src/i18n/pl/notif.ts +++ b/shared/src/i18n/pl/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} przypisał Cię do {category} w {trip}', 'notif.version_available.title': 'Nowa wersja dostępna', 'notif.version_available.text': 'TREK {version} jest teraz dostępny', + 'notif.replica_failure.title': 'Awaria repliki magazynu', + 'notif.replica_failure.text': "Zapis do repliki '{backend}' nie powiódł się: {op} dla {key} — {error}", + 'notif.replica_failure.textSuppressed': "Zapis do repliki '{backend}' nie powiódł się: {op} dla {key} — {error}. Od ostatniego powiadomienia ukryto {suppressed} kolejnych błędów.", 'notif.action.view_trip': 'Zobacz podróż', 'notif.action.view_collab': 'Zobacz wiadomości', 'notif.action.view_packing': 'Zobacz pakowanie', diff --git a/shared/src/i18n/pl/storage.ts b/shared/src/i18n/pl/storage.ts new file mode 100644 index 000000000..a68c7e78d --- /dev/null +++ b/shared/src/i18n/pl/storage.ts @@ -0,0 +1,141 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Katalog główny', + 'storage.help.root': 'Bezwzględna ścieżka na serwerze, w której ten backend przechowuje swoje obiekty.', + 'storage.field.endpoint': 'Adres URL punktu końcowego', + 'storage.help.endpoint': + 'Podstawowy adres URL usługi zgodnej z S3, np. https://s3.example.com lub http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Identyfikator klucza dostępu', + 'storage.field.secretAccessKey': 'Tajny klucz dostępu', + 'storage.field.region': 'Region', + 'storage.help.region': 'Pozostaw wartość domyślną, chyba że twój dostawca wymaga konkretnego regionu.', + 'storage.field.keyPrefix': 'Prefiks klucza', + 'storage.help.keyPrefix': 'Opcjonalny prefiks dodawany do każdego klucza obiektu, np. trek/prod.', + 'storage.field.retries': 'Liczba ponowień', + 'storage.field.timeoutMs': 'Limit czasu (ms)', + 'storage.field.primary': 'Backend podstawowy', + 'storage.field.replicas': 'Repliki', + 'storage.title': 'Magazyn', + 'storage.description': + 'Gdzie TREK przechowuje przesłane pliki, zdjęcia i kopie zapasowe. Nic się nie zmienia, dopóki nie zapiszesz.', + 'storage.loading': 'Ładowanie…', + 'storage.saved': 'Zapisano konfigurację magazynu', + 'storage.save': 'Zapisz zmiany', + 'storage.unsaved': 'Niezapisane zmiany', + 'storage.saveConflict': + 'Konfiguracja magazynu zmieniła się od czasu jej wczytania, więc Twoje zmiany nie zostały zapisane. Odrzuć je i wczytaj zapisaną konfigurację ponownie, aby zacząć od nowa.', + 'storage.discardAndReload': 'Odrzuć moje zmiany i wczytaj ponownie', + 'storage.configError.banner': 'Nie udało się wczytać zapisanej konfiguracji magazynu — zapisanie ją zastąpi: {error}', + 'storage.backends.title': 'Backendy', + 'storage.backends.add': 'Dodaj backend', + 'storage.backends.usedBy': 'Używany przez: {categories}', + 'storage.backends.unused': 'Nieprzypisany do żadnej kategorii', + 'storage.backends.envReadOnly': 'Zdefiniowany przez zmienną środowiskową — tylko do odczytu', + 'storage.source.built-in': 'Wbudowany', + 'storage.source.env': 'Środowisko', + 'storage.source.settings': 'Ustawienia', + 'storage.type.local': 'Lokalny', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Mirror', + 'storage.actions.test': 'Testuj', + 'storage.actions.edit': 'Edytuj', + 'storage.actions.remove': 'Usuń', + 'storage.test.running': 'Testowanie…', + 'storage.test.ok': 'Połączenie OK', + 'storage.test.failed': 'Test nieudany', + 'storage.remove.title': 'Usuń backend', + 'storage.remove.body': 'Usunąć {name} z konfiguracji? Serwer odrzuci zapis, jeśli coś nadal od niego zależy.', + 'storage.remove.stillAssigned': 'Nadal przypisany do: {categories}', + 'storage.form.addTitle': 'Dodaj backend', + 'storage.form.editTitle': 'Edytuj backend', + 'storage.form.name': 'Nazwa', + 'storage.form.type': 'Typ', + 'storage.form.apply': 'Zastosuj', + 'storage.form.cancel': 'Anuluj', + 'storage.form.duplicateName': 'Backend o nazwie {name} już istnieje', + 'storage.categories.title': 'Kategorie', + 'storage.categories.default': 'domyślna', + 'storage.categories.reassignWarning': + 'Istniejące obiekty nie są przenoszone: nowe obiekty trafiają do nowo przypisanego backendu, stare pozostają tam, gdzie są.', + 'storage.category.files': 'Dokumenty podróży', + 'storage.category.journey': 'Zdjęcia z dziennika podróży', + 'storage.category.covers': 'Zdjęcia okładek', + 'storage.category.avatars': 'Zdjęcia profilowe', + 'storage.category.places': 'Zdjęcia miejsc', + 'storage.category.photos-google': 'Pamięć podręczna zdjęć Google', + 'storage.category.photos-trek': 'Pamięć podręczna zdjęć TREK', + 'storage.category.backups': 'Kopie zapasowe', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Załączniki plikowe przesłane do podróży — bilety, PDF-y, potwierdzenia rezerwacji i pliki udostępnione na czacie podróży.', + 'storage.categoryDesc.journey': 'Zdjęcia i miniatury dołączone do wpisów dziennika podróży.', + 'storage.categoryDesc.covers': 'Zdjęcia okładek podróży i kolekcji, w tym okładki pobrane z Unsplash.', + 'storage.categoryDesc.avatars': 'Zdjęcia profilowe kont użytkowników.', + 'storage.categoryDesc.places': 'Obrazy dołączone do miejsc i miejsc w kolekcjach — przesłane lub zaimportowane.', + 'storage.categoryDesc.photos-google': + 'Kopie zdjęć z Google Places przechowywane w pamięci podręcznej — można je pobrać ponownie, ich utrata jest bezpieczna.', + 'storage.categoryDesc.photos-trek': + 'Zdjęcia w pamięci podręcznej z usługi zdjęć TREK używanej przez funkcję Zdjęcia (Memories) — można je pobrać ponownie, ich utrata jest bezpieczna.', + 'storage.categoryDesc.backups': + 'Archiwa kopii zapasowych serwera tworzone przez panel Kopia zapasowa lub harmonogram.', + 'storage.health.title': 'Stan', + 'storage.health.allClear': 'Nie odnotowano żadnych błędów replik.', + 'storage.health.seedFile': + 'Plik startowy storage-config.json jest obecny, ale ignorowany — wiersze konfiguracji już istnieją. Zarządzaj magazynem tutaj.', + 'storage.health.failureLine': '{op} dla {key} na {backend} nie powiodło się: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Cele mirrora', + 'storage.mirror.targetsHelp': 'Każdy zapis do tego backendu jest również kopiowany do każdego wybranego celu.', + 'storage.mirror.latencyNote': + 'Repliki są zapisywane jedna po drugiej podczas każdego przesyłania — wolny lub niedostępny cel spowalnia każde przesyłanie każdej kategorii na tym backendzie.', + 'storage.mirror.mirroredTo': 'Lustrzane do: {targets}', + 'storage.mirror.replicaOf': 'Replika: {primaries}', + 'storage.mirror.cacheWarning': + 'Niezalecane: ta kategoria zawiera treści możliwe do ponownego pobrania — ich replikowanie jest zwykle marnotrawstwem.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Drugi mirror otacza {primary} — panel zarządza tylko pierwszym; usuń ten, aby zarządzać mirrorowaniem z {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Otacza backend zdefiniowany przez zmienną środowiskową — nieedytowalny tutaj.', + 'storage.mirror.degenerate.missing-primary': 'Odwołuje się do backendu, który już nie istnieje.', + 'storage.remove.usedAsReplicaBy': 'Używany jako replika przez: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Synchronizuj teraz', + 'storage.sync.running': 'Synchronizacja… {done}/{total}', + 'storage.sync.counts': '{copied} skopiowanych · {skipped} pominiętych · {failed} nieudanych', + 'storage.sync.cancel': 'Anuluj synchronizację', + 'storage.sync.done': 'Synchronizacja zakończona: {copied} skopiowanych, {deleted} usuniętych, {failed} nieudanych', + 'storage.sync.cancelled': 'Synchronizacja anulowana', + 'storage.sync.error': 'Synchronizacja nie powiodła się: {error}', + 'storage.sync.prompt': 'Istniejące obiekty nie zostały jeszcze zreplikowane — zsynchronizować teraz?', + 'storage.sync.dismiss': 'Odrzuć', + 'storage.usage.line': '{objects} obiektów · {size}', + 'storage.usage.computed': 'Wykorzystanie obliczone {age}', + 'storage.usage.never': 'Wykorzystanie nie zostało jeszcze obliczone', + 'storage.usage.refresh': 'Odśwież', + 'storage.usage.compute': 'Oblicz teraz', + 'storage.usage.legacyNote': 'obejmuje starą bibliotekę zdjęć', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Przenieść istniejące obiekty do nowego backendu?', + 'storage.migrate.promptLine': '{category}: {objects} obiektów ({size}) z {from} do {to}', + 'storage.migrate.promptLineUnknown': + '{category}: nieznany rozmiar (wykorzystanie jeszcze nie obliczone) z {from} do {to}', + 'storage.migrate.move': 'Przenieś istniejące obiekty', + 'storage.migrate.routeOnly': 'Przekieruj tylko nowe zapisy', + 'storage.migrate.running': 'Przenoszenie {category}… {done}/{total}', + 'storage.migrate.done': 'Przenoszenie zakończone: {copied} skopiowanych, {skipped} pominiętych', + 'storage.migrate.doneFailures': '{failed} nie powiodło się — te obiekty nie zostały skopiowane do nowego backendu', + 'storage.migrate.failed': 'Przenoszenie nie powiodło się: {error} — kategoria nie została przełączona', + 'storage.migrate.cancelled': 'Przenoszenie anulowane — nic nie zostało przełączone', + 'storage.migrate.reclaimable': '{objects} obiektów ({size}) pozostaje na {from} — odzyskaj ręcznie', + 'storage.migrate.cancel': 'Anuluj przenoszenie', + 'storage.migrate.promptCancel': 'Anuluj', + 'storage.migrate.queued': 'W kolejce: {categories}', + 'storage.migrate.queueDropped': 'Nie udało się rozpocząć kolejnego przenoszenia — pozostała kolejka została wyczyszczona: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/ru/admin.ts b/shared/src/i18n/ru/admin.ts index b0fbbab5a..0beb196f5 100644 --- a/shared/src/i18n/ru/admin.ts +++ b/shared/src/i18n/ru/admin.ts @@ -167,6 +167,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Ошибка сохранения', 'admin.tabs.addons': 'Дополнения', 'admin.tabs.plugins': 'Плагины', + 'admin.tabs.storage': 'Хранилище', 'admin.plugins.rescan': 'Пересканировать', 'admin.plugins.rescanned': 'Папка плагинов пересканирована', 'admin.plugins.upload': 'Загрузить плагин', diff --git a/shared/src/i18n/ru/externalNotifications.ts b/shared/src/i18n/ru/externalNotifications.ts index 41ab512a2..f5c7a8006 100644 --- a/shared/src/i18n/ru/externalNotifications.ts +++ b/shared/src/i18n/ru/externalNotifications.ts @@ -52,6 +52,12 @@ const ru: NotificationLocale = { title: 'Доступна новая версия TREK', body: `TREK ${p.version} теперь доступен. Перейдите в панель администратора для обновления.`, }), + replica_failure: (p) => ({ + title: 'Сбой реплики хранилища', + body: + `Ошибка записи в реплику '${p.backend}': ${p.op} для ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` С момента последнего уведомления подавлено ещё ${p.suppressed} ошибок.` : ''), + }), synology_session_cleared: () => ({ title: 'Сессия Synology сброшена', body: 'Ваш аккаунт или URL Synology изменился. Вы вышли из Synology Photos.', diff --git a/shared/src/i18n/ru/index.ts b/shared/src/i18n/ru/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/ru/index.ts +++ b/shared/src/i18n/ru/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/ru/notif.ts b/shared/src/i18n/ru/notif.ts index ee67d0b21..46d076e5f 100644 --- a/shared/src/i18n/ru/notif.ts +++ b/shared/src/i18n/ru/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} назначил вас в {category} в {trip}', 'notif.version_available.title': 'Доступна новая версия', 'notif.version_available.text': 'TREK {version} теперь доступен', + 'notif.replica_failure.title': 'Сбой реплики хранилища', + 'notif.replica_failure.text': "Ошибка записи в реплику '{backend}': {op} для {key} — {error}", + 'notif.replica_failure.textSuppressed': "Ошибка записи в реплику '{backend}': {op} для {key} — {error}. С момента последнего уведомления подавлено ещё {suppressed} ошибок.", 'notif.action.view_trip': 'Открыть поездку', 'notif.action.view_collab': 'Открыть сообщения', 'notif.action.view_packing': 'Открыть упаковку', diff --git a/shared/src/i18n/ru/storage.ts b/shared/src/i18n/ru/storage.ts new file mode 100644 index 000000000..334a4394a --- /dev/null +++ b/shared/src/i18n/ru/storage.ts @@ -0,0 +1,143 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Корневой каталог', + 'storage.help.root': 'Абсолютный путь на сервере, где этот бэкенд хранит свои объекты.', + 'storage.field.endpoint': 'URL конечной точки', + 'storage.help.endpoint': + 'Базовый URL S3-совместимого сервиса, например https://s3.example.com или http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID ключа доступа', + 'storage.field.secretAccessKey': 'Секретный ключ доступа', + 'storage.field.region': 'Регион', + 'storage.help.region': 'Оставьте значение по умолчанию, если провайдер не требует конкретный регион.', + 'storage.field.keyPrefix': 'Префикс ключа', + 'storage.help.keyPrefix': 'Необязательный префикс, добавляемый к каждому ключу объекта, например trek/prod.', + 'storage.field.retries': 'Повторные попытки', + 'storage.field.timeoutMs': 'Тайм-аут (мс)', + 'storage.field.primary': 'Основной бэкенд', + 'storage.field.replicas': 'Реплики', + 'storage.title': 'Хранилище', + 'storage.description': + 'Где TREK хранит загруженные файлы, фото и резервные копии. Ничего не изменится, пока вы не сохраните.', + 'storage.loading': 'Загрузка…', + 'storage.saved': 'Конфигурация хранилища сохранена', + 'storage.save': 'Сохранить изменения', + 'storage.unsaved': 'Несохранённые изменения', + 'storage.saveConflict': + 'Конфигурация хранилища изменилась с момента загрузки, поэтому ваши изменения не сохранены. Отмените их и загрузите сохранённые настройки заново, чтобы начать сначала.', + 'storage.discardAndReload': 'Отменить мои изменения и перезагрузить', + 'storage.configError.banner': 'Не удалось загрузить сохранённые настройки хранилища — сохранение заменит их: {error}', + 'storage.backends.title': 'Бэкенды', + 'storage.backends.add': 'Добавить бэкенд', + 'storage.backends.usedBy': 'Используется в: {categories}', + 'storage.backends.unused': 'Не назначен ни одной категории', + 'storage.backends.envReadOnly': 'Определено переменной окружения — только для чтения', + 'storage.source.built-in': 'Встроенный', + 'storage.source.env': 'Окружение', + 'storage.source.settings': 'Настройки', + 'storage.type.local': 'Локальный', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Зеркало', + 'storage.actions.test': 'Проверить', + 'storage.actions.edit': 'Изменить', + 'storage.actions.remove': 'Удалить', + 'storage.test.running': 'Проверка…', + 'storage.test.ok': 'Соединение установлено', + 'storage.test.failed': 'Проверка не удалась', + 'storage.remove.title': 'Удалить бэкенд', + 'storage.remove.body': + 'Удалить {name} из конфигурации? Сервер отклонит сохранение, если от него всё ещё что-то зависит.', + 'storage.remove.stillAssigned': 'Всё ещё назначен: {categories}', + 'storage.form.addTitle': 'Добавить бэкенд', + 'storage.form.editTitle': 'Изменить бэкенд', + 'storage.form.name': 'Название', + 'storage.form.type': 'Тип', + 'storage.form.apply': 'Применить', + 'storage.form.cancel': 'Отмена', + 'storage.form.duplicateName': 'Бэкенд с названием {name} уже существует', + 'storage.categories.title': 'Категории', + 'storage.categories.default': 'по умолчанию', + 'storage.categories.reassignWarning': + 'Существующие объекты не перемещаются: новые объекты идут во вновь назначенный бэкенд, старые остаются на месте.', + 'storage.category.files': 'Документы поездки', + 'storage.category.journey': 'Фото путешествия', + 'storage.category.covers': 'Обложки', + 'storage.category.avatars': 'Фото профиля', + 'storage.category.places': 'Изображения мест', + 'storage.category.photos-google': 'Кэш фото Google', + 'storage.category.photos-trek': 'Кэш фото TREK', + 'storage.category.backups': 'Резервные копии', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Файловые вложения, загруженные в поездки — билеты, PDF, подтверждения бронирования и файлы, которыми поделились в чате поездки.', + 'storage.categoryDesc.journey': 'Фото и миниатюры, прикреплённые к записям путешествия.', + 'storage.categoryDesc.covers': 'Обложки поездок и коллекций, включая обложки, полученные с Unsplash.', + 'storage.categoryDesc.avatars': 'Фото профиля учётных записей пользователей.', + 'storage.categoryDesc.places': + 'Изображения, прикреплённые к местам и местам коллекций — загруженные или импортированные.', + 'storage.categoryDesc.photos-google': + 'Кэшированные копии фото Google Places — их можно получить заново, потеря безопасна.', + 'storage.categoryDesc.photos-trek': + 'Кэшированные фото из фотосервиса TREK, используемого функцией Фото (Memories) — их можно получить заново, потеря безопасна.', + 'storage.categoryDesc.backups': + 'Архивы резервных копий сервера, созданные панелью резервного копирования или по расписанию.', + 'storage.health.title': 'Состояние', + 'storage.health.allClear': 'Сбоев реплик не зафиксировано.', + 'storage.health.seedFile': + 'Присутствует файл-заготовка storage-config.json, но он игнорируется — строки конфигурации уже существуют. Управляйте хранилищем здесь.', + 'storage.health.failureLine': '{op} для {key} на {backend} не удалось: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Цели зеркала', + 'storage.mirror.targetsHelp': 'Каждая запись в этот бэкенд также копируется в каждую выбранную цель.', + 'storage.mirror.latencyNote': + 'Реплики записываются последовательно, одна за другой, при каждой загрузке — медленная или недоступная цель замедляет каждую загрузку каждой категории на этом бэкенде.', + 'storage.mirror.mirroredTo': 'Зеркалируется в: {targets}', + 'storage.mirror.replicaOf': 'Реплика с: {primaries}', + 'storage.mirror.cacheWarning': + 'Не рекомендуется: эта категория содержит контент, который можно получить заново — его репликация обычно бесполезна.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Второе зеркало оборачивает {primary} — панель управляет только первым; удалите это, чтобы управлять зеркалированием от {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Оборачивает бэкенд, определённый переменной окружения — здесь не редактируется.', + 'storage.mirror.degenerate.missing-primary': 'Ссылается на бэкенд, которого больше не существует.', + 'storage.remove.usedAsReplicaBy': 'Используется как реплика в: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Синхронизировать сейчас', + 'storage.sync.running': 'Синхронизация… {done}/{total}', + 'storage.sync.counts': '{copied} скопировано · {skipped} пропущено · {failed} не удалось', + 'storage.sync.cancel': 'Отменить синхронизацию', + 'storage.sync.done': 'Синхронизация завершена: {copied} скопировано, {deleted} удалено, {failed} не удалось', + 'storage.sync.cancelled': 'Синхронизация отменена', + 'storage.sync.error': 'Ошибка синхронизации: {error}', + 'storage.sync.prompt': 'Существующие объекты ещё не реплицированы — синхронизировать сейчас?', + 'storage.sync.dismiss': 'Скрыть', + 'storage.usage.line': '{objects} объектов · {size}', + 'storage.usage.computed': 'Использование рассчитано {age}', + 'storage.usage.never': 'Использование ещё не рассчитано', + 'storage.usage.refresh': 'Обновить', + 'storage.usage.compute': 'Рассчитать сейчас', + 'storage.usage.legacyNote': 'включает старую библиотеку фото', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Перенести существующие объекты на новый бэкенд?', + 'storage.migrate.promptLine': '{category}: {objects} объектов ({size}) из {from} в {to}', + 'storage.migrate.promptLineUnknown': + '{category}: неизвестный размер (использование ещё не просканировано) из {from} в {to}', + 'storage.migrate.move': 'Перенести существующие объекты', + 'storage.migrate.routeOnly': 'Перенаправить только новые записи', + 'storage.migrate.running': 'Перенос {category}… {done}/{total}', + 'storage.migrate.done': 'Перенос завершён: {copied} скопировано, {skipped} пропущено', + 'storage.migrate.doneFailures': '{failed} не удалось — эти объекты не были скопированы на новый бэкенд', + 'storage.migrate.failed': 'Перенос не удался: {error} — категория не была переключена', + 'storage.migrate.cancelled': 'Перенос отменён — ничего не было переключено', + 'storage.migrate.reclaimable': '{objects} объектов ({size}) остаются на {from} — освободите вручную', + 'storage.migrate.cancel': 'Отменить перенос', + 'storage.migrate.promptCancel': 'Отмена', + 'storage.migrate.queued': 'В очереди: {categories}', + 'storage.migrate.queueDropped': 'Не удалось запустить следующий перенос — оставшаяся очередь очищена: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/sv/admin.ts b/shared/src/i18n/sv/admin.ts index bb504ef5f..4b5927316 100644 --- a/shared/src/i18n/sv/admin.ts +++ b/shared/src/i18n/sv/admin.ts @@ -223,6 +223,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Det gick inte att spara', 'admin.tabs.addons': 'Tillägg', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Lagring', 'admin.plugins.rescan': 'Skanna om', 'admin.plugins.rescanned': 'Plugin-mappen har skannats om', 'admin.plugins.upload': 'Ladda upp plugin', diff --git a/shared/src/i18n/sv/externalNotifications.ts b/shared/src/i18n/sv/externalNotifications.ts index c1c5d1605..6db520cb2 100644 --- a/shared/src/i18n/sv/externalNotifications.ts +++ b/shared/src/i18n/sv/externalNotifications.ts @@ -52,6 +52,12 @@ const en: NotificationLocale = { title: 'Ny TREK version tillgänglig', body: `TREK ${p.version} är nu tillgänglig. Gå till adminpanelen för att uppdatera.`, }), + replica_failure: (p) => ({ + title: 'Fel i lagringsreplik', + body: + `Skrivning till replik '${p.backend}' misslyckades: ${p.op} för ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} ytterligare fel har undertryckts sedan senaste aviseringen.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology session rensad', body: 'Ditt Synology-konto eller din webbadress har ändrats. Du har loggats ut från Synology Photos.', diff --git a/shared/src/i18n/sv/index.ts b/shared/src/i18n/sv/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/sv/index.ts +++ b/shared/src/i18n/sv/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/sv/notif.ts b/shared/src/i18n/sv/notif.ts index 65506fd73..058a1583c 100644 --- a/shared/src/i18n/sv/notif.ts +++ b/shared/src/i18n/sv/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} har tilldelat dig {category} i {trip}', 'notif.version_available.title': 'Ny version tillgänglig', 'notif.version_available.text': 'TREK {version} finns nu tillgängligt', + 'notif.replica_failure.title': 'Fel i lagringsreplik', + 'notif.replica_failure.text': "Skrivning till replik '{backend}' misslyckades: {op} för {key} — {error}", + 'notif.replica_failure.textSuppressed': "Skrivning till replik '{backend}' misslyckades: {op} för {key} — {error}. {suppressed} ytterligare fel har undertryckts sedan senaste aviseringen.", 'notif.action.view_trip': 'Visa resan', 'notif.action.view_collab': 'Visa meddelanden', 'notif.action.view_packing': 'Visa förpackning', diff --git a/shared/src/i18n/sv/storage.ts b/shared/src/i18n/sv/storage.ts new file mode 100644 index 000000000..a0ac11c01 --- /dev/null +++ b/shared/src/i18n/sv/storage.ts @@ -0,0 +1,139 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Rotkatalog', + 'storage.help.root': 'Absolut sökväg på servern där denna backend lagrar sina objekt.', + 'storage.field.endpoint': 'Slutpunkts-URL', + 'storage.help.endpoint': + 'Bas-URL för den S3-kompatibla tjänsten, t.ex. https://s3.example.com eller http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Åtkomstnyckel-ID', + 'storage.field.secretAccessKey': 'Hemlig åtkomstnyckel', + 'storage.field.region': 'Region', + 'storage.help.region': 'Behåll standardvärdet om inte din leverantör kräver en specifik region.', + 'storage.field.keyPrefix': 'Nyckelprefix', + 'storage.help.keyPrefix': 'Valfritt prefix som läggs till varje objektnyckel, t.ex. trek/prod.', + 'storage.field.retries': 'Nya försök', + 'storage.field.timeoutMs': 'Tidsgräns (ms)', + 'storage.field.primary': 'Primär backend', + 'storage.field.replicas': 'Repliker', + 'storage.title': 'Lagring', + 'storage.description': 'Var TREK sparar uppladdade filer, foton och säkerhetskopior. Inget ändras förrän du sparar.', + 'storage.loading': 'Laddar…', + 'storage.saved': 'Lagringskonfiguration sparad', + 'storage.save': 'Spara ändringar', + 'storage.unsaved': 'Osparade ändringar', + 'storage.saveConflict': + 'Lagringskonfigurationen har ändrats sedan den lästes in, så dina ändringar sparades inte. Kasta dem och läs in de sparade inställningarna igen för att börja om.', + 'storage.discardAndReload': 'Kasta mina ändringar och läs in igen', + 'storage.configError.banner': + 'Det gick inte att läsa in de sparade lagringsinställningarna — att spara ersätter dem: {error}', + 'storage.backends.title': 'Backender', + 'storage.backends.add': 'Lägg till backend', + 'storage.backends.usedBy': 'Används av: {categories}', + 'storage.backends.unused': 'Inte tilldelad någon kategori', + 'storage.backends.envReadOnly': 'Definierad av en miljövariabel — skrivskyddad', + 'storage.source.built-in': 'Inbyggd', + 'storage.source.env': 'Miljö', + 'storage.source.settings': 'Inställningar', + 'storage.type.local': 'Lokal', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Spegel', + 'storage.actions.test': 'Testa', + 'storage.actions.edit': 'Redigera', + 'storage.actions.remove': 'Ta bort', + 'storage.test.running': 'Testar…', + 'storage.test.ok': 'Anslutning OK', + 'storage.test.failed': 'Test misslyckades', + 'storage.remove.title': 'Ta bort backend', + 'storage.remove.body': + 'Ta bort {name} från konfigurationen? Servern avvisar sparningen om något fortfarande beror på den.', + 'storage.remove.stillAssigned': 'Fortfarande tilldelad: {categories}', + 'storage.form.addTitle': 'Lägg till backend', + 'storage.form.editTitle': 'Redigera backend', + 'storage.form.name': 'Namn', + 'storage.form.type': 'Typ', + 'storage.form.apply': 'Tillämpa', + 'storage.form.cancel': 'Avbryt', + 'storage.form.duplicateName': 'En backend med namnet {name} finns redan', + 'storage.categories.title': 'Kategorier', + 'storage.categories.default': 'standard', + 'storage.categories.reassignWarning': + 'Befintliga objekt flyttas inte: nya objekt går till den nyligen tilldelade backenden, gamla stannar där de är.', + 'storage.category.files': 'Resedokument', + 'storage.category.journey': 'Journey-foton', + 'storage.category.covers': 'Omslagsbilder', + 'storage.category.avatars': 'Profilbilder', + 'storage.category.places': 'Platsbilder', + 'storage.category.photos-google': 'Google-fotocache', + 'storage.category.photos-trek': 'TREK-fotocache', + 'storage.category.backups': 'Säkerhetskopior', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Filbilagor uppladdade till resor — biljetter, PDF:er, bokningsbekräftelser och filer delade i resechatten.', + 'storage.categoryDesc.journey': 'Foton och miniatyrer kopplade till Journey-poster.', + 'storage.categoryDesc.covers': 'Omslagsbilder för resor och samlingar, inklusive omslag hämtade från Unsplash.', + 'storage.categoryDesc.avatars': 'Profilbilder för användarkonton.', + 'storage.categoryDesc.places': 'Bilder kopplade till platser och samlingsplatser — uppladdade eller importerade.', + 'storage.categoryDesc.photos-google': + 'Cachade kopior av foton från Google Places — kan hämtas igen, säkert att förlora.', + 'storage.categoryDesc.photos-trek': + 'Cachade foton från TREK-fototjänsten som används av Foton (Memories) — kan hämtas igen, säkert att förlora.', + 'storage.categoryDesc.backups': 'Serverbackuparkiv skapade av backuppanelen eller schemat.', + 'storage.health.title': 'Hälsa', + 'storage.health.allClear': 'Inga replikfel registrerade.', + 'storage.health.seedFile': + 'En storage-config.json-startfil finns men ignoreras — konfigurationsrader finns redan. Hantera lagringen här.', + 'storage.health.failureLine': '{op} av {key} på {backend} misslyckades: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Spegelmål', + 'storage.mirror.targetsHelp': 'Varje skrivning till denna backend kopieras även till varje valt mål.', + 'storage.mirror.latencyNote': + 'Repliker skrivs en efter en under varje uppladdning — ett långsamt eller onåbart mål saktar ner varje uppladdning av varje kategori på denna backend.', + 'storage.mirror.mirroredTo': 'Speglad till: {targets}', + 'storage.mirror.replicaOf': 'Replik av: {primaries}', + 'storage.mirror.cacheWarning': + 'Rekommenderas inte: denna kategori innehåller innehåll som kan hämtas på nytt — att replikera det är oftast slöseri.', + 'storage.mirror.degenerate.duplicate-mirror': + 'En andra spegel omsluter {primary} — panelen hanterar bara den första; ta bort den här för att hantera spegling från {primary}.', + 'storage.mirror.degenerate.env-primary': 'Omsluter en backend definierad av en miljövariabel — inte redigerbar här.', + 'storage.mirror.degenerate.missing-primary': 'Refererar till en backend som inte längre finns.', + 'storage.remove.usedAsReplicaBy': 'Används som replik av: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Synkronisera nu', + 'storage.sync.running': 'Synkroniserar… {done}/{total}', + 'storage.sync.counts': '{copied} kopierade · {skipped} överhoppade · {failed} misslyckade', + 'storage.sync.cancel': 'Avbryt synkronisering', + 'storage.sync.done': 'Synkronisering klar: {copied} kopierade, {deleted} borttagna, {failed} misslyckade', + 'storage.sync.cancelled': 'Synkronisering avbruten', + 'storage.sync.error': 'Synkronisering misslyckades: {error}', + 'storage.sync.prompt': 'Befintliga objekt har inte replikerats än — synkronisera nu?', + 'storage.sync.dismiss': 'Avvisa', + 'storage.usage.line': '{objects} objekt · {size}', + 'storage.usage.computed': 'Användning beräknad {age}', + 'storage.usage.never': 'Användning inte beräknad än', + 'storage.usage.refresh': 'Uppdatera', + 'storage.usage.compute': 'Beräkna nu', + 'storage.usage.legacyNote': 'inkluderar det äldre fotobiblioteket', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Flytta befintliga objekt till den nya backend?', + 'storage.migrate.promptLine': '{category}: {objects} objekt ({size}) från {from} till {to}', + 'storage.migrate.promptLineUnknown': '{category}: okänd storlek (användning inte skannad än) från {from} till {to}', + 'storage.migrate.move': 'Flytta befintliga objekt', + 'storage.migrate.routeOnly': 'Dirigera bara nya skrivningar', + 'storage.migrate.running': 'Flyttar {category}… {done}/{total}', + 'storage.migrate.done': 'Flytt klar: {copied} kopierade, {skipped} överhoppade', + 'storage.migrate.doneFailures': '{failed} misslyckades — dessa objekt kopierades inte till den nya backend', + 'storage.migrate.failed': 'Flytt misslyckades: {error} — kategorin byttes inte', + 'storage.migrate.cancelled': 'Flytt avbruten — inget byttes', + 'storage.migrate.reclaimable': '{objects} objekt ({size}) finns kvar på {from} — återvinn manuellt', + 'storage.migrate.cancel': 'Avbryt flytt', + 'storage.migrate.promptCancel': 'Avbryt', + 'storage.migrate.queued': 'I kö: {categories}', + 'storage.migrate.queueDropped': 'Det gick inte att starta nästa flytt — den återstående kön rensades: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/tr/admin.ts b/shared/src/i18n/tr/admin.ts index 1b470360f..2dbd3f98a 100644 --- a/shared/src/i18n/tr/admin.ts +++ b/shared/src/i18n/tr/admin.ts @@ -211,6 +211,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Kaydedilemedi', 'admin.tabs.addons': 'Eklentiler', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Depolama', 'admin.plugins.rescan': 'Yeniden tara', 'admin.plugins.rescanned': 'Eklenti klasörü yeniden tarandı', 'admin.plugins.upload': 'Eklenti yükle', diff --git a/shared/src/i18n/tr/externalNotifications.ts b/shared/src/i18n/tr/externalNotifications.ts index 282d4a791..1851b9379 100644 --- a/shared/src/i18n/tr/externalNotifications.ts +++ b/shared/src/i18n/tr/externalNotifications.ts @@ -52,6 +52,12 @@ const tr: NotificationLocale = { title: 'Yeni TREK sürümü mevcut', body: `TREK ${p.version} artık mevcut. Güncellemek için yönetici panelini ziyaret edin.`, }), + replica_failure: (p) => ({ + title: 'Depolama kopyası hatası', + body: + `'${p.backend}' kopyasına yazma başarısız oldu: ${p.op} (${p.key}) — ${p.error}.` + + (p.suppressed !== '0' ? ` Son bildirimden bu yana ${p.suppressed} hata daha bastırıldı.` : ''), + }), synology_session_cleared: () => ({ title: 'Synology oturumu temizlendi', body: 'Synology hesabınız veya URL değişti. Synology Photos oturumunuz kapatıldı.', diff --git a/shared/src/i18n/tr/index.ts b/shared/src/i18n/tr/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/tr/index.ts +++ b/shared/src/i18n/tr/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/tr/notif.ts b/shared/src/i18n/tr/notif.ts index 4ec97aef3..0e3877f9c 100644 --- a/shared/src/i18n/tr/notif.ts +++ b/shared/src/i18n/tr/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} sizi {trip} içinde {category} kategorisine atadı', 'notif.version_available.title': 'Yeni Sürüm Mevcut', 'notif.version_available.text': 'TREK {version} artık kullanılabilir', + 'notif.replica_failure.title': 'Depolama kopyası hatası', + 'notif.replica_failure.text': "'{backend}' kopyasına yazma başarısız oldu: {op} ({key}) — {error}", + 'notif.replica_failure.textSuppressed': "'{backend}' kopyasına yazma başarısız oldu: {op} ({key}) — {error}. Son bildirimden bu yana {suppressed} hata daha bastırıldı.", 'notif.action.view_trip': 'Seyahati Görüntüle', 'notif.action.view_collab': 'Mesajları Görüntüle', 'notif.action.view_packing': 'Paket listesini görüntüle', diff --git a/shared/src/i18n/tr/storage.ts b/shared/src/i18n/tr/storage.ts new file mode 100644 index 000000000..0e4c76602 --- /dev/null +++ b/shared/src/i18n/tr/storage.ts @@ -0,0 +1,141 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Kök dizin', + 'storage.help.root': 'Bu arka ucun nesnelerini sakladığı sunucudaki mutlak yol.', + 'storage.field.endpoint': "Uç nokta URL'si", + 'storage.help.endpoint': "S3 uyumlu hizmetin taban URL'si, örn. https://s3.example.com veya http://127.0.0.1:9000.", + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Erişim anahtarı kimliği', + 'storage.field.secretAccessKey': 'Gizli erişim anahtarı', + 'storage.field.region': 'Bölge', + 'storage.help.region': 'Sağlayıcınız belirli bir bölge gerektirmedikçe varsayılanı koruyun.', + 'storage.field.keyPrefix': 'Anahtar öneki', + 'storage.help.keyPrefix': 'Her nesne anahtarına eklenen isteğe bağlı önek, örn. trek/prod.', + 'storage.field.retries': 'Yeniden deneme', + 'storage.field.timeoutMs': 'Zaman aşımı (ms)', + 'storage.field.primary': 'Birincil arka uç', + 'storage.field.replicas': 'Kopyalar', + 'storage.title': 'Depolama', + 'storage.description': + "TREK'in yüklenen dosyaları, fotoğrafları ve yedekleri sakladığı yer. Kaydedene kadar hiçbir şey değişmez.", + 'storage.loading': 'Yükleniyor…', + 'storage.saved': 'Depolama yapılandırması kaydedildi', + 'storage.save': 'Değişiklikleri kaydet', + 'storage.unsaved': 'Kaydedilmemiş değişiklikler', + 'storage.saveConflict': + 'Depolama yapılandırması yüklendiğinden beri değişti, bu yüzden değişiklikleriniz kaydedilmedi. Değişiklikleri atıp kayıtlı ayarları yeniden yükleyerek baştan başlayın.', + 'storage.discardAndReload': 'Değişikliklerimi at ve yeniden yükle', + 'storage.configError.banner': 'Kaydedilmiş depolama ayarları yüklenemedi — kaydetmek onların yerini alacak: {error}', + 'storage.backends.title': 'Arka uçlar', + 'storage.backends.add': 'Arka uç ekle', + 'storage.backends.usedBy': 'Kullanan: {categories}', + 'storage.backends.unused': 'Herhangi bir kategoriye atanmamış', + 'storage.backends.envReadOnly': 'Bir ortam değişkeni tarafından tanımlanmış — salt okunur', + 'storage.source.built-in': 'Yerleşik', + 'storage.source.env': 'Ortam', + 'storage.source.settings': 'Ayarlar', + 'storage.type.local': 'Yerel', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Ayna', + 'storage.actions.test': 'Test et', + 'storage.actions.edit': 'Düzenle', + 'storage.actions.remove': 'Kaldır', + 'storage.test.running': 'Test ediliyor…', + 'storage.test.ok': 'Bağlantı başarılı', + 'storage.test.failed': 'Test başarısız', + 'storage.remove.title': 'Arka ucu kaldır', + 'storage.remove.body': + '{name} yapılandırmadan kaldırılsın mı? Hâlâ ona bağımlı bir şey varsa sunucu kaydetmeyi reddeder.', + 'storage.remove.stillAssigned': 'Hâlâ şuna atanmış: {categories}', + 'storage.form.addTitle': 'Arka uç ekle', + 'storage.form.editTitle': 'Arka ucu düzenle', + 'storage.form.name': 'Ad', + 'storage.form.type': 'Tür', + 'storage.form.apply': 'Uygula', + 'storage.form.cancel': 'İptal', + 'storage.form.duplicateName': '{name} adlı bir arka uç zaten var', + 'storage.categories.title': 'Kategoriler', + 'storage.categories.default': 'varsayılan', + 'storage.categories.reassignWarning': + 'Mevcut nesneler taşınmaz: yeni nesneler yeni atanan arka uca gider, eskiler oldukları yerde kalır.', + 'storage.category.files': 'Seyahat belgeleri', + 'storage.category.journey': 'Seyahat fotoğrafları', + 'storage.category.covers': 'Kapak görselleri', + 'storage.category.avatars': 'Profil fotoğrafları', + 'storage.category.places': 'Yer görselleri', + 'storage.category.photos-google': 'Google fotoğraf önbelleği', + 'storage.category.photos-trek': 'TREK fotoğraf önbelleği', + 'storage.category.backups': 'Yedekler', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + "Seyahatlere yüklenen dosya ekleri — biletler, PDF'ler, rezervasyon onayları ve seyahat sohbetinde paylaşılan dosyalar.", + 'storage.categoryDesc.journey': 'Seyahat günlüğü kayıtlarına eklenen fotoğraflar ve küçük resimler.', + 'storage.categoryDesc.covers': + "Unsplash'tan alınan kapaklar dahil olmak üzere seyahat ve koleksiyon kapak görselleri.", + 'storage.categoryDesc.avatars': 'Kullanıcı hesabı profil fotoğrafları.', + 'storage.categoryDesc.places': 'Yerlere ve koleksiyon yerlerine eklenen görseller — yüklenmiş veya içe aktarılmış.', + 'storage.categoryDesc.photos-google': + 'Google Places fotoğraflarının önbelleğe alınmış kopyaları — yeniden alınabilir, kaybı güvenlidir.', + 'storage.categoryDesc.photos-trek': + 'Fotoğraflar (Memories) özelliği tarafından kullanılan TREK fotoğraf hizmetinden önbelleğe alınmış fotoğraflar — yeniden alınabilir, kaybı güvenlidir.', + 'storage.categoryDesc.backups': 'Yedekleme panelinin veya zamanlamanın oluşturduğu sunucu yedek arşivleri.', + 'storage.health.title': 'Sağlık', + 'storage.health.allClear': 'Kaydedilmiş kopya hatası yok.', + 'storage.health.seedFile': + 'Bir storage-config.json tohum dosyası mevcut ama yok sayılıyor — yapılandırma satırları zaten var. Depolamayı buradan yönetin.', + 'storage.health.failureLine': '{backend} üzerinde {key} için {op} başarısız oldu: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Ayna hedefleri', + 'storage.mirror.targetsHelp': 'Bu arka uca yapılan her yazma, seçilen her hedefe de kopyalanır.', + 'storage.mirror.latencyNote': + 'Kopyalar her yüklemede sırayla yazılır — yavaş veya erişilemeyen bir hedef, bu arka uçtaki her kategorinin her yüklemesini yavaşlatır.', + 'storage.mirror.mirroredTo': 'Şuraya aynalandı: {targets}', + 'storage.mirror.replicaOf': 'Şunun kopyası: {primaries}', + 'storage.mirror.cacheWarning': + 'Önerilmez: bu kategori yeniden alınabilir içerik barındırıyor — bunu çoğaltmak genellikle israftır.', + 'storage.mirror.degenerate.duplicate-mirror': + 'İkinci bir ayna {primary} öğesini sarmalıyor — panel yalnızca ilkini yönetir; {primary} üzerinden aynalamayı yönetmek için bunu kaldırın.', + 'storage.mirror.degenerate.env-primary': + 'Ortam değişkeniyle tanımlanmış bir arka ucu sarmalıyor — burada düzenlenemez.', + 'storage.mirror.degenerate.missing-primary': 'Artık var olmayan bir arka uca başvuruyor.', + 'storage.remove.usedAsReplicaBy': 'Kopya olarak kullanan: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Şimdi eşitle', + 'storage.sync.running': 'Eşitleniyor… {done}/{total}', + 'storage.sync.counts': '{copied} kopyalandı · {skipped} atlandı · {failed} başarısız', + 'storage.sync.cancel': 'Eşitlemeyi iptal et', + 'storage.sync.done': 'Eşitleme tamamlandı: {copied} kopyalandı, {deleted} silindi, {failed} başarısız', + 'storage.sync.cancelled': 'Eşitleme iptal edildi', + 'storage.sync.error': 'Eşitleme başarısız oldu: {error}', + 'storage.sync.prompt': 'Mevcut nesneler henüz çoğaltılmadı — şimdi eşitlensin mi?', + 'storage.sync.dismiss': 'Kapat', + 'storage.usage.line': '{objects} nesne · {size}', + 'storage.usage.computed': 'Kullanım hesaplandı {age}', + 'storage.usage.never': 'Kullanım henüz hesaplanmadı', + 'storage.usage.refresh': 'Yenile', + 'storage.usage.compute': 'Şimdi hesapla', + 'storage.usage.legacyNote': 'eski fotoğraf kitaplığını içerir', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Mevcut nesneler yeni arka uca taşınsın mı?', + 'storage.migrate.promptLine': '{category}: {from} konumundan {to} konumuna {objects} nesne ({size})', + 'storage.migrate.promptLineUnknown': + '{category}: bilinmeyen boyut (henüz kullanım taraması yapılmadı) {from} konumundan {to} konumuna', + 'storage.migrate.move': 'Mevcut nesneleri taşı', + 'storage.migrate.routeOnly': 'Yalnızca yeni yazmaları yönlendir', + 'storage.migrate.running': '{category} taşınıyor… {done}/{total}', + 'storage.migrate.done': 'Taşıma tamamlandı: {copied} kopyalandı, {skipped} atlandı', + 'storage.migrate.doneFailures': '{failed} başarısız oldu — bu nesneler yeni arka uca kopyalanmadı', + 'storage.migrate.failed': 'Taşıma başarısız oldu: {error} — kategori değiştirilmedi', + 'storage.migrate.cancelled': 'Taşıma iptal edildi — hiçbir şey değiştirilmedi', + 'storage.migrate.reclaimable': '{objects} nesne ({size}) {from} üzerinde kalıyor — manuel olarak geri kazanın', + 'storage.migrate.cancel': 'Taşımayı iptal et', + 'storage.migrate.promptCancel': 'İptal', + 'storage.migrate.queued': 'Sırada: {categories}', + 'storage.migrate.queueDropped': 'Sonraki taşıma başlatılamadı — kalan kuyruk temizlendi: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/uk/admin.ts b/shared/src/i18n/uk/admin.ts index a534d0810..613ef51e4 100644 --- a/shared/src/i18n/uk/admin.ts +++ b/shared/src/i18n/uk/admin.ts @@ -167,6 +167,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Помилка збереження', 'admin.tabs.addons': 'Дополнения', 'admin.tabs.plugins': 'Плагіни', + 'admin.tabs.storage': 'Сховище', 'admin.plugins.rescan': 'Пересканувати', 'admin.plugins.rescanned': 'Папку плагінів пересканувано', 'admin.plugins.upload': 'Завантажити плагін', diff --git a/shared/src/i18n/uk/externalNotifications.ts b/shared/src/i18n/uk/externalNotifications.ts index de83ed167..db3438b46 100644 --- a/shared/src/i18n/uk/externalNotifications.ts +++ b/shared/src/i18n/uk/externalNotifications.ts @@ -52,6 +52,12 @@ const uk: NotificationLocale = { title: 'Доступна нова версія TREK', body: `TREK ${p.version} тепер доступний. Перейдіть до панелі адміністратора для оновлення.`, }), + replica_failure: (p) => ({ + title: 'Збій репліки сховища', + body: + `Помилка запису в репліку '${p.backend}': ${p.op} для ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` Із моменту останнього сповіщення приховано ще ${p.suppressed} помилок.` : ''), + }), synology_session_cleared: () => ({ title: 'Сеанс Synology скинуто', body: 'Ваш обліковий запис або URL Synology змінився. Ви вийшли з Synology Photos.', diff --git a/shared/src/i18n/uk/index.ts b/shared/src/i18n/uk/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/uk/index.ts +++ b/shared/src/i18n/uk/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/uk/notif.ts b/shared/src/i18n/uk/notif.ts index 0ae172271..371e3f3d6 100644 --- a/shared/src/i18n/uk/notif.ts +++ b/shared/src/i18n/uk/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} призначив вас у {category} у {trip}', 'notif.version_available.title': 'Доступна нова версія', 'notif.version_available.text': 'TREK {version} тепер доступний', + 'notif.replica_failure.title': 'Збій репліки сховища', + 'notif.replica_failure.text': "Помилка запису в репліку '{backend}': {op} для {key} — {error}", + 'notif.replica_failure.textSuppressed': "Помилка запису в репліку '{backend}': {op} для {key} — {error}. Із моменту останнього сповіщення приховано ще {suppressed} помилок.", 'notif.action.view_trip': 'Відкрити поїздку', 'notif.action.view_collab': 'Відкрити повідомлення', 'notif.action.view_packing': 'Відкрити пакування', diff --git a/shared/src/i18n/uk/storage.ts b/shared/src/i18n/uk/storage.ts new file mode 100644 index 000000000..480946384 --- /dev/null +++ b/shared/src/i18n/uk/storage.ts @@ -0,0 +1,142 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Кореневий каталог', + 'storage.help.root': "Абсолютний шлях на сервері, де цей бекенд зберігає свої об'єкти.", + 'storage.field.endpoint': 'URL кінцевої точки', + 'storage.help.endpoint': + 'Базовий URL сумісного з S3 сервісу, напр. https://s3.example.com або http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'Ідентифікатор ключа доступу', + 'storage.field.secretAccessKey': 'Секретний ключ доступу', + 'storage.field.region': 'Регіон', + 'storage.help.region': 'Залиште значення за замовчуванням, якщо ваш провайдер не вимагає конкретного регіону.', + 'storage.field.keyPrefix': 'Префікс ключа', + 'storage.help.keyPrefix': "Необов'язковий префікс, що додається до кожного ключа об'єкта, напр. trek/prod.", + 'storage.field.retries': 'Повторні спроби', + 'storage.field.timeoutMs': 'Тайм-аут (мс)', + 'storage.field.primary': 'Основний бекенд', + 'storage.field.replicas': 'Репліки', + 'storage.title': 'Сховище', + 'storage.description': + 'Де TREK зберігає завантажені файли, фото та резервні копії. Нічого не зміниться, доки ви не збережете.', + 'storage.loading': 'Завантаження…', + 'storage.saved': 'Конфігурацію сховища збережено', + 'storage.save': 'Зберегти зміни', + 'storage.unsaved': 'Незбережені зміни', + 'storage.saveConflict': + 'Конфігурацію сховища змінено після завантаження, тож ваші зміни не збережено. Відхиліть їх і перезавантажте збережені налаштування, щоб почати спочатку.', + 'storage.discardAndReload': 'Відхилити мої зміни та перезавантажити', + 'storage.configError.banner': + 'Не вдалося завантажити збережені налаштування сховища — збереження замінить їх: {error}', + 'storage.backends.title': 'Бекенди', + 'storage.backends.add': 'Додати бекенд', + 'storage.backends.usedBy': 'Використовується у: {categories}', + 'storage.backends.unused': 'Не призначено жодній категорії', + 'storage.backends.envReadOnly': 'Визначено змінною середовища — лише для читання', + 'storage.source.built-in': 'Вбудований', + 'storage.source.env': 'Середовище', + 'storage.source.settings': 'Налаштування', + 'storage.type.local': 'Локальний', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Дзеркало', + 'storage.actions.test': 'Перевірити', + 'storage.actions.edit': 'Редагувати', + 'storage.actions.remove': 'Видалити', + 'storage.test.running': 'Перевірка…', + 'storage.test.ok': "З'єднання успішне", + 'storage.test.failed': 'Перевірка не вдалася', + 'storage.remove.title': 'Видалити бекенд', + 'storage.remove.body': + 'Видалити {name} з конфігурації? Сервер відхилить збереження, якщо щось усе ще залежить від нього.', + 'storage.remove.stillAssigned': 'Досі призначено: {categories}', + 'storage.form.addTitle': 'Додати бекенд', + 'storage.form.editTitle': 'Редагувати бекенд', + 'storage.form.name': 'Назва', + 'storage.form.type': 'Тип', + 'storage.form.apply': 'Застосувати', + 'storage.form.cancel': 'Скасувати', + 'storage.form.duplicateName': 'Бекенд з назвою {name} вже існує', + 'storage.categories.title': 'Категорії', + 'storage.categories.default': 'за замовчуванням', + 'storage.categories.reassignWarning': + "Наявні об'єкти не переміщуються: нові об'єкти йдуть у щойно призначений бекенд, старі залишаються на місці.", + 'storage.category.files': 'Документи подорожі', + 'storage.category.journey': 'Фото Journey', + 'storage.category.covers': 'Обкладинки', + 'storage.category.avatars': 'Фото профілю', + 'storage.category.places': 'Зображення місць', + 'storage.category.photos-google': 'Кеш фото Google', + 'storage.category.photos-trek': 'Кеш фото TREK', + 'storage.category.backups': 'Резервні копії', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Файлові вкладення, завантажені до подорожей — квитки, PDF, підтвердження бронювання та файли, якими поділилися в чаті подорожі.', + 'storage.categoryDesc.journey': 'Фото та мініатюри, прикріплені до записів Journey.', + 'storage.categoryDesc.covers': 'Обкладинки подорожей і колекцій, включно з обкладинками, отриманими з Unsplash.', + 'storage.categoryDesc.avatars': 'Фото профілю облікових записів користувачів.', + 'storage.categoryDesc.places': 'Зображення, прикріплені до місць і місць колекцій — завантажені або імпортовані.', + 'storage.categoryDesc.photos-google': + 'Кешовані копії фото Google Places — можуть бути отримані повторно, їх втрата безпечна.', + 'storage.categoryDesc.photos-trek': + 'Кешовані фото зі служби фото TREK, яку використовує функція Фото (Memories) — можуть бути отримані повторно, їх втрата безпечна.', + 'storage.categoryDesc.backups': + 'Архіви резервних копій сервера, створені панеллю резервного копіювання або за розкладом.', + 'storage.health.title': 'Стан', + 'storage.health.allClear': 'Збоїв реплік не зафіксовано.', + 'storage.health.seedFile': + 'Присутній файл-заготовка storage-config.json, але він ігнорується — рядки конфігурації вже існують. Керуйте сховищем тут.', + 'storage.health.failureLine': '{op} для {key} на {backend} не вдалося: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Цілі дзеркала', + 'storage.mirror.targetsHelp': 'Кожен запис у цей бекенд також копіюється в кожну вибрану ціль.', + 'storage.mirror.latencyNote': + 'Репліки записуються одна за одною під час кожного завантаження — повільна або недоступна ціль сповільнює кожне завантаження кожної категорії на цьому бекенді.', + 'storage.mirror.mirroredTo': 'Дзеркалюється в: {targets}', + 'storage.mirror.replicaOf': 'Репліка з: {primaries}', + 'storage.mirror.cacheWarning': + 'Не рекомендується: ця категорія містить контент, який можна отримати повторно — його реплікація зазвичай марна.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Друге дзеркало обгортає {primary} — панель керує лише першим; вилучіть це, щоб керувати дзеркалюванням від {primary}.', + 'storage.mirror.degenerate.env-primary': 'Обгортає бекенд, визначений змінною середовища — тут не редагується.', + 'storage.mirror.degenerate.missing-primary': 'Посилається на бекенд, якого більше не існує.', + 'storage.remove.usedAsReplicaBy': 'Використовується як репліка: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Синхронізувати зараз', + 'storage.sync.running': 'Синхронізація… {done}/{total}', + 'storage.sync.counts': '{copied} скопійовано · {skipped} пропущено · {failed} не вдалося', + 'storage.sync.cancel': 'Скасувати синхронізацію', + 'storage.sync.done': 'Синхронізацію завершено: {copied} скопійовано, {deleted} видалено, {failed} не вдалося', + 'storage.sync.cancelled': 'Синхронізацію скасовано', + 'storage.sync.error': 'Синхронізація не вдалася: {error}', + 'storage.sync.prompt': "Наявні об'єкти ще не реплікуються — синхронізувати зараз?", + 'storage.sync.dismiss': 'Приховати', + 'storage.usage.line': "{objects} об'єктів · {size}", + 'storage.usage.computed': 'Використання обчислено {age}', + 'storage.usage.never': 'Використання ще не обчислено', + 'storage.usage.refresh': 'Оновити', + 'storage.usage.compute': 'Обчислити зараз', + 'storage.usage.legacyNote': 'включає стару фотобібліотеку', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': "Перенести наявні об'єкти на новий бекенд?", + 'storage.migrate.promptLine': "{category}: {objects} об'єктів ({size}) з {from} до {to}", + 'storage.migrate.promptLineUnknown': + '{category}: невідомий розмір (використання ще не проскановано) з {from} до {to}', + 'storage.migrate.move': "Перенести наявні об'єкти", + 'storage.migrate.routeOnly': 'Перенаправити лише нові записи', + 'storage.migrate.running': 'Перенесення {category}… {done}/{total}', + 'storage.migrate.done': 'Перенесення завершено: {copied} скопійовано, {skipped} пропущено', + 'storage.migrate.doneFailures': "{failed} не вдалося — ці об'єкти не було скопійовано на новий бекенд", + 'storage.migrate.failed': 'Перенесення не вдалося: {error} — категорію не перемкнуто', + 'storage.migrate.cancelled': 'Перенесення скасовано — нічого не перемкнуто', + 'storage.migrate.reclaimable': "{objects} об'єктів ({size}) залишаються на {from} — звільніть вручну", + 'storage.migrate.cancel': 'Скасувати перенесення', + 'storage.migrate.promptCancel': 'Скасувати', + 'storage.migrate.queued': 'У черзі: {categories}', + 'storage.migrate.queueDropped': 'Не вдалося розпочати наступне перенесення — залишок черги очищено: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/vi/admin.ts b/shared/src/i18n/vi/admin.ts index b52f593b3..e51a318b6 100644 --- a/shared/src/i18n/vi/admin.ts +++ b/shared/src/i18n/vi/admin.ts @@ -222,6 +222,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': 'Không lưu được', 'admin.tabs.addons': 'Tiện ích bổ sung', 'admin.tabs.plugins': 'Plugins', + 'admin.tabs.storage': 'Lưu trữ', 'admin.plugins.rescan': 'Quét lại', 'admin.plugins.rescanned': 'Đã quét lại thư mục plugin', 'admin.plugins.upload': 'Tải plugin lên', diff --git a/shared/src/i18n/vi/externalNotifications.ts b/shared/src/i18n/vi/externalNotifications.ts index ca4ab2830..7943ee1ee 100644 --- a/shared/src/i18n/vi/externalNotifications.ts +++ b/shared/src/i18n/vi/externalNotifications.ts @@ -52,6 +52,12 @@ const en: NotificationLocale = { title: 'Đã có phiên bản TREK mới', body: `TREK ${p.version} có bản mới. Vui lòng truy cập bảng điều khiển quản trị để cập nhật.`, }), + replica_failure: (p) => ({ + title: 'Lỗi bản sao lưu trữ', + body: + `Ghi vào bản sao '${p.backend}' thất bại: ${p.op} của ${p.key} — ${p.error}.` + + (p.suppressed !== '0' ? ` ${p.suppressed} lỗi khác đã bị bỏ qua kể từ thông báo trước đó.` : ''), + }), synology_session_cleared: () => ({ title: 'Đã xóa phiên Synology', body: 'Tài khoản Synology của bạn hoặc URL đã thay đổi. Bạn đã đăng xuất khỏi Synology Photos.', diff --git a/shared/src/i18n/vi/index.ts b/shared/src/i18n/vi/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/vi/index.ts +++ b/shared/src/i18n/vi/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/vi/notif.ts b/shared/src/i18n/vi/notif.ts index a2dcb9dfc..e7c60ac2c 100644 --- a/shared/src/i18n/vi/notif.ts +++ b/shared/src/i18n/vi/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} đã giao cho bạn {category} trong {trip}', 'notif.version_available.title': 'Phiên bản mới có sẵn', 'notif.version_available.text': 'TREK {version} hiện có sẵn', + 'notif.replica_failure.title': 'Lỗi bản sao lưu trữ', + 'notif.replica_failure.text': "Ghi vào bản sao '{backend}' thất bại: {op} của {key} — {error}", + 'notif.replica_failure.textSuppressed': "Ghi vào bản sao '{backend}' thất bại: {op} của {key} — {error}. {suppressed} lỗi khác đã bị bỏ qua kể từ thông báo trước đó.", 'notif.action.view_trip': 'Xem chuyến đi', 'notif.action.view_collab': 'Xem tin nhắn', 'notif.action.view_packing': 'Xem đóng gói', diff --git a/shared/src/i18n/vi/storage.ts b/shared/src/i18n/vi/storage.ts new file mode 100644 index 000000000..87ba41a06 --- /dev/null +++ b/shared/src/i18n/vi/storage.ts @@ -0,0 +1,140 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': 'Thư mục gốc', + 'storage.help.root': 'Đường dẫn tuyệt đối trên máy chủ nơi backend này lưu trữ các đối tượng của nó.', + 'storage.field.endpoint': 'URL điểm cuối', + 'storage.help.endpoint': + 'URL cơ sở của dịch vụ tương thích S3, ví dụ: https://s3.example.com hoặc http://127.0.0.1:9000.', + 'storage.field.bucket': 'Bucket', + 'storage.field.accessKeyId': 'ID khóa truy cập', + 'storage.field.secretAccessKey': 'Khóa truy cập bí mật', + 'storage.field.region': 'Khu vực', + 'storage.help.region': 'Giữ giá trị mặc định trừ khi nhà cung cấp của bạn yêu cầu một khu vực cụ thể.', + 'storage.field.keyPrefix': 'Tiền tố khóa', + 'storage.help.keyPrefix': 'Tiền tố tùy chọn được thêm vào mỗi khóa đối tượng, ví dụ: trek/prod.', + 'storage.field.retries': 'Số lần thử lại', + 'storage.field.timeoutMs': 'Thời gian chờ (ms)', + 'storage.field.primary': 'Backend chính', + 'storage.field.replicas': 'Bản sao', + 'storage.title': 'Lưu trữ', + 'storage.description': + 'Nơi TREK lưu giữ các tệp, ảnh và bản sao lưu đã tải lên. Không có gì thay đổi cho đến khi bạn lưu.', + 'storage.loading': 'Đang tải…', + 'storage.saved': 'Đã lưu cấu hình lưu trữ', + 'storage.save': 'Lưu thay đổi', + 'storage.unsaved': 'Thay đổi chưa lưu', + 'storage.saveConflict': + 'Cấu hình lưu trữ đã thay đổi kể từ khi được tải, nên các thay đổi của bạn chưa được lưu. Hãy hủy chúng và tải lại cấu hình đã lưu để bắt đầu lại.', + 'storage.discardAndReload': 'Hủy thay đổi của tôi và tải lại', + 'storage.configError.banner': 'Không thể tải cài đặt lưu trữ đã lưu — việc lưu sẽ thay thế chúng: {error}', + 'storage.backends.title': 'Backend', + 'storage.backends.add': 'Thêm backend', + 'storage.backends.usedBy': 'Được sử dụng bởi: {categories}', + 'storage.backends.unused': 'Chưa được gán cho danh mục nào', + 'storage.backends.envReadOnly': 'Được xác định bởi biến môi trường — chỉ đọc', + 'storage.source.built-in': 'Tích hợp sẵn', + 'storage.source.env': 'Môi trường', + 'storage.source.settings': 'Cài đặt', + 'storage.type.local': 'Cục bộ', + 'storage.type.s3': 'S3', + 'storage.type.mirror': 'Bản sao gương', + 'storage.actions.test': 'Kiểm tra', + 'storage.actions.edit': 'Chỉnh sửa', + 'storage.actions.remove': 'Xóa', + 'storage.test.running': 'Đang kiểm tra…', + 'storage.test.ok': 'Kết nối OK', + 'storage.test.failed': 'Kiểm tra thất bại', + 'storage.remove.title': 'Xóa backend', + 'storage.remove.body': 'Xóa {name} khỏi cấu hình? Máy chủ sẽ từ chối lưu nếu vẫn còn thứ gì đó phụ thuộc vào nó.', + 'storage.remove.stillAssigned': 'Vẫn được gán cho: {categories}', + 'storage.form.addTitle': 'Thêm backend', + 'storage.form.editTitle': 'Chỉnh sửa backend', + 'storage.form.name': 'Tên', + 'storage.form.type': 'Loại', + 'storage.form.apply': 'Áp dụng', + 'storage.form.cancel': 'Hủy', + 'storage.form.duplicateName': 'Đã tồn tại backend có tên {name}', + 'storage.categories.title': 'Danh mục', + 'storage.categories.default': 'mặc định', + 'storage.categories.reassignWarning': + 'Các đối tượng hiện có không di chuyển: đối tượng mới sẽ vào backend vừa được gán, đối tượng cũ vẫn ở nguyên vị trí.', + 'storage.category.files': 'Tài liệu chuyến đi', + 'storage.category.journey': 'Ảnh hành trình', + 'storage.category.covers': 'Ảnh bìa', + 'storage.category.avatars': 'Ảnh đại diện', + 'storage.category.places': 'Hình ảnh địa điểm', + 'storage.category.photos-google': 'Bộ nhớ đệm ảnh Google', + 'storage.category.photos-trek': 'Bộ nhớ đệm ảnh TREK', + 'storage.category.backups': 'Bản sao lưu', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': + 'Tệp đính kèm được tải lên các chuyến đi — vé, PDF, xác nhận đặt chỗ và tệp được chia sẻ trong trò chuyện của chuyến đi.', + 'storage.categoryDesc.journey': 'Ảnh và hình thu nhỏ đính kèm với các mục hành trình.', + 'storage.categoryDesc.covers': 'Ảnh bìa của chuyến đi và bộ sưu tập, bao gồm cả ảnh bìa lấy từ Unsplash.', + 'storage.categoryDesc.avatars': 'Ảnh đại diện của tài khoản người dùng.', + 'storage.categoryDesc.places': + 'Hình ảnh đính kèm với địa điểm và địa điểm trong bộ sưu tập — được tải lên hoặc nhập vào.', + 'storage.categoryDesc.photos-google': + 'Bản sao được lưu trong bộ nhớ đệm của ảnh Google Places — có thể lấy lại được, mất đi vẫn an toàn.', + 'storage.categoryDesc.photos-trek': + 'Ảnh được lưu trong bộ nhớ đệm từ dịch vụ ảnh TREK được tính năng Ảnh (Memories) sử dụng — có thể lấy lại được, mất đi vẫn an toàn.', + 'storage.categoryDesc.backups': 'Kho lưu trữ sao lưu máy chủ được tạo bởi bảng Sao lưu hoặc theo lịch trình.', + 'storage.health.title': 'Tình trạng', + 'storage.health.allClear': 'Không có lỗi bản sao nào được ghi nhận.', + 'storage.health.seedFile': + 'Có tệp khởi tạo storage-config.json nhưng bị bỏ qua — các dòng cấu hình đã tồn tại. Quản lý lưu trữ tại đây.', + 'storage.health.failureLine': '{op} của {key} trên {backend} thất bại: {error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': 'Đích của bản sao gương', + 'storage.mirror.targetsHelp': 'Mọi thao tác ghi vào backend này cũng được sao chép đến từng đích đã chọn.', + 'storage.mirror.latencyNote': + 'Các bản sao được ghi lần lượt trong mỗi lần tải lên — một đích chậm hoặc không thể truy cập sẽ làm chậm mọi lần tải lên của mọi danh mục trên backend này.', + 'storage.mirror.mirroredTo': 'Được nhân bản đến: {targets}', + 'storage.mirror.replicaOf': 'Bản sao của: {primaries}', + 'storage.mirror.cacheWarning': + 'Không khuyến nghị: danh mục này chứa nội dung có thể lấy lại được — sao chép nó thường là lãng phí.', + 'storage.mirror.degenerate.duplicate-mirror': + 'Một bản sao gương thứ hai bao quanh {primary} — bảng điều khiển chỉ quản lý bản đầu tiên; hãy xóa mục này để quản lý việc nhân bản từ {primary}.', + 'storage.mirror.degenerate.env-primary': + 'Bao quanh một backend được xác định bởi biến môi trường — không thể chỉnh sửa tại đây.', + 'storage.mirror.degenerate.missing-primary': 'Tham chiếu đến một backend không còn tồn tại.', + 'storage.remove.usedAsReplicaBy': 'Được dùng làm bản sao bởi: {primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': 'Đồng bộ ngay', + 'storage.sync.running': 'Đang đồng bộ… {done}/{total}', + 'storage.sync.counts': '{copied} đã sao chép · {skipped} đã bỏ qua · {failed} thất bại', + 'storage.sync.cancel': 'Hủy đồng bộ', + 'storage.sync.done': 'Đồng bộ hoàn tất: {copied} đã sao chép, {deleted} đã xóa, {failed} thất bại', + 'storage.sync.cancelled': 'Đã hủy đồng bộ', + 'storage.sync.error': 'Đồng bộ thất bại: {error}', + 'storage.sync.prompt': 'Các đối tượng hiện có chưa được nhân bản — đồng bộ ngay?', + 'storage.sync.dismiss': 'Bỏ qua', + 'storage.usage.line': '{objects} đối tượng · {size}', + 'storage.usage.computed': 'Đã tính dung lượng sử dụng {age}', + 'storage.usage.never': 'Chưa tính dung lượng sử dụng', + 'storage.usage.refresh': 'Làm mới', + 'storage.usage.compute': 'Tính ngay', + 'storage.usage.legacyNote': 'bao gồm thư viện ảnh cũ', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': 'Di chuyển các đối tượng hiện có sang backend mới?', + 'storage.migrate.promptLine': '{category}: {objects} đối tượng ({size}) từ {from} sang {to}', + 'storage.migrate.promptLineUnknown': '{category}: không rõ dung lượng (chưa quét mức sử dụng) từ {from} sang {to}', + 'storage.migrate.move': 'Di chuyển các đối tượng hiện có', + 'storage.migrate.routeOnly': 'Chỉ định tuyến các ghi mới', + 'storage.migrate.running': 'Đang di chuyển {category}… {done}/{total}', + 'storage.migrate.done': 'Di chuyển hoàn tất: {copied} đã sao chép, {skipped} đã bỏ qua', + 'storage.migrate.doneFailures': '{failed} thất bại — các đối tượng đó chưa được sao chép sang backend mới', + 'storage.migrate.failed': 'Di chuyển thất bại: {error} — danh mục chưa được chuyển đổi', + 'storage.migrate.cancelled': 'Đã hủy di chuyển — chưa có gì được chuyển đổi', + 'storage.migrate.reclaimable': '{objects} đối tượng ({size}) vẫn còn trên {from} — hãy thu hồi thủ công', + 'storage.migrate.cancel': 'Hủy di chuyển', + 'storage.migrate.promptCancel': 'Hủy', + 'storage.migrate.queued': 'Đang chờ: {categories}', + 'storage.migrate.queueDropped': 'Không thể bắt đầu di chuyển tiếp theo — hàng đợi còn lại đã bị xóa: {categories}', +}; +export default storage; diff --git a/shared/src/i18n/zh-TW/admin.ts b/shared/src/i18n/zh-TW/admin.ts index 3e12a6e15..62ba6336b 100644 --- a/shared/src/i18n/zh-TW/admin.ts +++ b/shared/src/i18n/zh-TW/admin.ts @@ -201,6 +201,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': '儲存失敗', 'admin.tabs.addons': '擴充套件', 'admin.tabs.plugins': '外掛', + 'admin.tabs.storage': '儲存', 'admin.plugins.rescan': '重新掃描', 'admin.plugins.rescanned': '已重新掃描外掛資料夾', 'admin.plugins.upload': '上傳外掛', diff --git a/shared/src/i18n/zh-TW/externalNotifications.ts b/shared/src/i18n/zh-TW/externalNotifications.ts index 416066206..26d4d43e6 100644 --- a/shared/src/i18n/zh-TW/externalNotifications.ts +++ b/shared/src/i18n/zh-TW/externalNotifications.ts @@ -52,6 +52,12 @@ const zhTW: NotificationLocale = { title: '新版 TREK 可用', body: `TREK ${p.version} 現已可用。請前往管理面板進行更新。`, }), + replica_failure: (p) => ({ + title: '儲存複本故障', + body: + `寫入複本 '${p.backend}' 失敗：${p.op} / ${p.key} — ${p.error}。` + + (p.suppressed !== '0' ? `自上次通知以來，還有 ${p.suppressed} 個失敗已被抑制。` : ''), + }), synology_session_cleared: () => ({ title: 'Synology 工作階段已清除', body: '您的 Synology 帳戶或 URL 已變更，您已登出 Synology Photos。', diff --git a/shared/src/i18n/zh-TW/index.ts b/shared/src/i18n/zh-TW/index.ts index 31882af05..d6370b596 100644 --- a/shared/src/i18n/zh-TW/index.ts +++ b/shared/src/i18n/zh-TW/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/zh-TW/notif.ts b/shared/src/i18n/zh-TW/notif.ts index ade8e2911..31a0f79ed 100644 --- a/shared/src/i18n/zh-TW/notif.ts +++ b/shared/src/i18n/zh-TW/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} 在 {trip} 中將 {category} 指派給你', 'notif.version_available.title': '有新版本可用', 'notif.version_available.text': 'TREK {version} 現已推出', + 'notif.replica_failure.title': '儲存複本故障', + 'notif.replica_failure.text': "寫入複本 '{backend}' 失敗：{op} / {key} — {error}", + 'notif.replica_failure.textSuppressed': "寫入複本 '{backend}' 失敗：{op} / {key} — {error}。自上次通知以來，還有 {suppressed} 個失敗已被抑制。", 'notif.action.view_trip': '查看旅行', 'notif.action.view_collab': '查看訊息', 'notif.action.view_packing': '查看打包', diff --git a/shared/src/i18n/zh-TW/storage.ts b/shared/src/i18n/zh-TW/storage.ts new file mode 100644 index 000000000..631742986 --- /dev/null +++ b/shared/src/i18n/zh-TW/storage.ts @@ -0,0 +1,130 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': '根目錄', + 'storage.help.root': '此後端儲存物件的伺服器絕對路徑。', + 'storage.field.endpoint': '端點 URL', + 'storage.help.endpoint': '相容 S3 服務的基礎 URL，例如 https://s3.example.com 或 http://127.0.0.1:9000。', + 'storage.field.bucket': '儲存貯體', + 'storage.field.accessKeyId': '存取金鑰 ID', + 'storage.field.secretAccessKey': '私密存取金鑰', + 'storage.field.region': '區域', + 'storage.help.region': '除非你的供應商要求特定區域，否則請保留預設值。', + 'storage.field.keyPrefix': '金鑰前綴', + 'storage.help.keyPrefix': '加在每個物件金鑰前的選用前綴，例如 trek/prod。', + 'storage.field.retries': '重試次數', + 'storage.field.timeoutMs': '逾時（毫秒）', + 'storage.field.primary': '主要後端', + 'storage.field.replicas': '複本', + 'storage.title': '儲存', + 'storage.description': 'TREK 存放上傳檔案、照片和備份的位置。儲存前不會有任何變更。', + 'storage.loading': '載入中…', + 'storage.saved': '儲存設定已儲存', + 'storage.save': '儲存變更', + 'storage.unsaved': '未儲存的變更', + 'storage.saveConflict': + '儲存設定自載入後已變更，因此您的變更未被儲存。請捨棄這些變更並重新載入已儲存的設定，然後重新開始。', + 'storage.discardAndReload': '捨棄我的變更並重新載入', + 'storage.configError.banner': '已儲存的儲存設定載入失敗 — 儲存將取代它們：{error}', + 'storage.backends.title': '後端', + 'storage.backends.add': '新增後端', + 'storage.backends.usedBy': '使用者：{categories}', + 'storage.backends.unused': '未指定任何分類', + 'storage.backends.envReadOnly': '由環境變數定義 — 唯讀', + 'storage.source.built-in': '內建', + 'storage.source.env': '環境', + 'storage.source.settings': '設定', + 'storage.type.local': '本機', + 'storage.type.s3': 'S3', + 'storage.type.mirror': '鏡像', + 'storage.actions.test': '測試', + 'storage.actions.edit': '編輯', + 'storage.actions.remove': '移除', + 'storage.test.running': '測試中…', + 'storage.test.ok': '連線正常', + 'storage.test.failed': '測試失敗', + 'storage.remove.title': '移除後端', + 'storage.remove.body': '要從設定中移除 {name} 嗎？如果仍有項目依賴它，伺服器將拒絕儲存。', + 'storage.remove.stillAssigned': '仍指定給：{categories}', + 'storage.form.addTitle': '新增後端', + 'storage.form.editTitle': '編輯後端', + 'storage.form.name': '名稱', + 'storage.form.type': '類型', + 'storage.form.apply': '套用', + 'storage.form.cancel': '取消', + 'storage.form.duplicateName': '名為 {name} 的後端已存在', + 'storage.categories.title': '分類', + 'storage.categories.default': '預設', + 'storage.categories.reassignWarning': '現有物件不會移動：新物件會進入新指定的後端，舊物件仍留在原處。', + 'storage.category.files': '行程文件', + 'storage.category.journey': '旅程相片', + 'storage.category.covers': '封面圖片', + 'storage.category.avatars': '個人大頭貼', + 'storage.category.places': '地點圖片', + 'storage.category.photos-google': 'Google 相片快取', + 'storage.category.photos-trek': 'TREK 相片快取', + 'storage.category.backups': '備份', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': '上傳到行程的檔案附件 — 車票、PDF、訂位確認單，以及在行程聊天中分享的檔案。', + 'storage.categoryDesc.journey': '附加到旅程項目的相片與縮圖。', + 'storage.categoryDesc.covers': '行程與收藏的封面圖片，包括從 Unsplash 取得的封面。', + 'storage.categoryDesc.avatars': '使用者帳戶的個人大頭貼。', + 'storage.categoryDesc.places': '附加到地點與收藏地點的圖片 — 上傳或匯入的。', + 'storage.categoryDesc.photos-google': 'Google Places 相片的快取副本 — 可重新取得，遺失也無妨。', + 'storage.categoryDesc.photos-trek': + '「照片」（Memories）功能所使用的 TREK 相片服務快取相片 — 可重新取得，遺失也無妨。', + 'storage.categoryDesc.backups': '由備份面板或排程建立的伺服器備份封存檔。', + 'storage.health.title': '健康狀態', + 'storage.health.allClear': '未記錄任何複本失敗。', + 'storage.health.seedFile': '存在 storage-config.json 種子檔案但已被忽略 — 設定資料列已存在。請在此管理儲存。', + 'storage.health.failureLine': '{backend} 上 {key} 的 {op} 失敗：{error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': '鏡像目標', + 'storage.mirror.targetsHelp': '對此後端的每次寫入也會複製到每個選定的目標。', + 'storage.mirror.latencyNote': '每次上傳時，複本會依序寫入 — 緩慢或無法連線的目標會拖慢此後端上每個分類的每次上傳。', + 'storage.mirror.mirroredTo': '已鏡像至：{targets}', + 'storage.mirror.replicaOf': '複本來源：{primaries}', + 'storage.mirror.cacheWarning': '不建議：此分類儲存的是可重新取得的內容 — 複製它通常是浪費的。', + 'storage.mirror.degenerate.duplicate-mirror': + '第二個鏡像包覆了 {primary} — 面板僅管理第一個；移除此項以從 {primary} 管理鏡像。', + 'storage.mirror.degenerate.env-primary': '包覆了一個由環境變數定義的後端 — 此處不可編輯。', + 'storage.mirror.degenerate.missing-primary': '參照了一個已不存在的後端。', + 'storage.remove.usedAsReplicaBy': '被用作複本，使用者：{primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': '立即同步', + 'storage.sync.running': '同步中… {done}/{total}', + 'storage.sync.counts': '已複製 {copied} · 已略過 {skipped} · 失敗 {failed}', + 'storage.sync.cancel': '取消同步', + 'storage.sync.done': '同步完成：已複製 {copied}，已刪除 {deleted}，失敗 {failed}', + 'storage.sync.cancelled': '已取消同步', + 'storage.sync.error': '同步失敗：{error}', + 'storage.sync.prompt': '現有物件尚未複製 — 要立即同步嗎？', + 'storage.sync.dismiss': '關閉', + 'storage.usage.line': '{objects} 個物件 · {size}', + 'storage.usage.computed': '用量已計算：{age}', + 'storage.usage.never': '尚未計算用量', + 'storage.usage.refresh': '重新整理', + 'storage.usage.compute': '立即計算', + 'storage.usage.legacyNote': '包含舊版相片庫', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': '要將現有物件移動到新的後端嗎？', + 'storage.migrate.promptLine': '{category}：從 {from} 到 {to} 共 {objects} 個物件（{size}）', + 'storage.migrate.promptLineUnknown': '{category}：大小未知（尚未掃描用量）從 {from} 到 {to}', + 'storage.migrate.move': '移動現有物件', + 'storage.migrate.routeOnly': '僅路由新寫入', + 'storage.migrate.running': '正在移動 {category}… {done}/{total}', + 'storage.migrate.done': '移動完成：已複製 {copied}，已略過 {skipped}', + 'storage.migrate.doneFailures': '{failed} 個失敗 — 這些物件未複製到新的後端', + 'storage.migrate.failed': '移動失敗：{error} — 該分類未切換', + 'storage.migrate.cancelled': '移動已取消 — 未切換任何內容', + 'storage.migrate.reclaimable': '{objects} 個物件（{size}）仍保留在 {from} 上 — 請手動回收', + 'storage.migrate.cancel': '取消移動', + 'storage.migrate.promptCancel': '取消', + 'storage.migrate.queued': '排隊中：{categories}', + 'storage.migrate.queueDropped': '無法啟動下一個遷移 — 剩餘佇列已清空：{categories}', +}; +export default storage; diff --git a/shared/src/i18n/zh/admin.ts b/shared/src/i18n/zh/admin.ts index c74308a66..c4789f07f 100644 --- a/shared/src/i18n/zh/admin.ts +++ b/shared/src/i18n/zh/admin.ts @@ -161,6 +161,7 @@ const admin: TranslationStrings = { 'admin.packingTemplates.saveError': '保存失败', 'admin.tabs.addons': '扩展', 'admin.tabs.plugins': '插件', + 'admin.tabs.storage': '存储', 'admin.plugins.rescan': '重新扫描', 'admin.plugins.rescanned': '已重新扫描插件文件夹', 'admin.plugins.upload': '上传插件', diff --git a/shared/src/i18n/zh/externalNotifications.ts b/shared/src/i18n/zh/externalNotifications.ts index 6fdec2485..1e56edca2 100644 --- a/shared/src/i18n/zh/externalNotifications.ts +++ b/shared/src/i18n/zh/externalNotifications.ts @@ -52,6 +52,12 @@ const zh: NotificationLocale = { title: '新版 TREK 可用', body: `TREK ${p.version} 现已可用。请前往管理面板进行更新。`, }), + replica_failure: (p) => ({ + title: '存储副本故障', + body: + `写入副本 '${p.backend}' 失败：${p.op} / ${p.key} — ${p.error}。` + + (p.suppressed !== '0' ? `自上次通知以来，还有 ${p.suppressed} 个失败被抑制。` : ''), + }), synology_session_cleared: () => ({ title: 'Synology 会话已清除', body: '您的 Synology 账户或 URL 已更改，您已退出 Synology Photos。', diff --git a/shared/src/i18n/zh/index.ts b/shared/src/i18n/zh/index.ts index fa29125e3..11ef2a989 100644 --- a/shared/src/i18n/zh/index.ts +++ b/shared/src/i18n/zh/index.ts @@ -42,6 +42,7 @@ import settings from './settings'; import share from './share'; import shared from './shared'; import stats from './stats'; +import storage from './storage'; import system_notice from './system_notice'; import todo from './todo'; import transport from './transport'; @@ -102,5 +103,6 @@ const locale = { ...mobileAdmin, ...mobileSettings, ...mobileCollections, + ...storage, }; export default locale; diff --git a/shared/src/i18n/zh/notif.ts b/shared/src/i18n/zh/notif.ts index 7f2cd5659..1c8a700df 100644 --- a/shared/src/i18n/zh/notif.ts +++ b/shared/src/i18n/zh/notif.ts @@ -27,6 +27,9 @@ const notif: TranslationStrings = { 'notif.packing_tagged.text': '{actor} 将您分配到 {trip} 中的 {category}', 'notif.version_available.title': '新版本可用', 'notif.version_available.text': 'TREK {version} 现已可用', + 'notif.replica_failure.title': '存储副本故障', + 'notif.replica_failure.text': "写入副本 '{backend}' 失败：{op} / {key} — {error}", + 'notif.replica_failure.textSuppressed': "写入副本 '{backend}' 失败：{op} / {key} — {error}。自上次通知以来，还有 {suppressed} 个失败被抑制。", 'notif.action.view_trip': '查看旅行', 'notif.action.view_collab': '查看消息', 'notif.action.view_packing': '查看行李', diff --git a/shared/src/i18n/zh/storage.ts b/shared/src/i18n/zh/storage.ts new file mode 100644 index 000000000..5caad1479 --- /dev/null +++ b/shared/src/i18n/zh/storage.ts @@ -0,0 +1,129 @@ +import type { TranslationStrings } from '../types'; + +const storage: TranslationStrings = { + 'storage.field.root': '根目录', + 'storage.help.root': '该后端存储对象的服务器绝对路径。', + 'storage.field.endpoint': '端点 URL', + 'storage.help.endpoint': '兼容 S3 服务的基础 URL，例如 https://s3.example.com 或 http://127.0.0.1:9000。', + 'storage.field.bucket': '存储桶', + 'storage.field.accessKeyId': '访问密钥 ID', + 'storage.field.secretAccessKey': '私密访问密钥', + 'storage.field.region': '区域', + 'storage.help.region': '除非你的提供商要求特定区域，否则保留默认值。', + 'storage.field.keyPrefix': '键前缀', + 'storage.help.keyPrefix': '添加到每个对象键的可选前缀，例如 trek/prod。', + 'storage.field.retries': '重试次数', + 'storage.field.timeoutMs': '超时（毫秒）', + 'storage.field.primary': '主后端', + 'storage.field.replicas': '副本', + 'storage.title': '存储', + 'storage.description': 'TREK 存放上传文件、照片和备份的位置。保存之前不会有任何更改。', + 'storage.loading': '加载中…', + 'storage.saved': '存储配置已保存', + 'storage.save': '保存更改', + 'storage.unsaved': '未保存的更改', + 'storage.saveConflict': + '存储配置自加载后已更改，因此您的更改未被保存。请放弃这些更改并重新加载已保存的设置，然后重新开始。', + 'storage.discardAndReload': '放弃我的更改并重新加载', + 'storage.configError.banner': '已保存的存储设置加载失败 — 保存将替换它们：{error}', + 'storage.backends.title': '后端', + 'storage.backends.add': '添加后端', + 'storage.backends.usedBy': '使用者：{categories}', + 'storage.backends.unused': '未分配到任何分类', + 'storage.backends.envReadOnly': '由环境变量定义 — 只读', + 'storage.source.built-in': '内置', + 'storage.source.env': '环境', + 'storage.source.settings': '设置', + 'storage.type.local': '本地', + 'storage.type.s3': 'S3', + 'storage.type.mirror': '镜像', + 'storage.actions.test': '测试', + 'storage.actions.edit': '编辑', + 'storage.actions.remove': '移除', + 'storage.test.running': '测试中…', + 'storage.test.ok': '连接正常', + 'storage.test.failed': '测试失败', + 'storage.remove.title': '移除后端', + 'storage.remove.body': '从配置中移除 {name}？如果仍有内容依赖它，服务器将拒绝保存。', + 'storage.remove.stillAssigned': '仍分配给：{categories}', + 'storage.form.addTitle': '添加后端', + 'storage.form.editTitle': '编辑后端', + 'storage.form.name': '名称', + 'storage.form.type': '类型', + 'storage.form.apply': '应用', + 'storage.form.cancel': '取消', + 'storage.form.duplicateName': '名为 {name} 的后端已存在', + 'storage.categories.title': '分类', + 'storage.categories.default': '默认', + 'storage.categories.reassignWarning': '现有对象不会移动：新对象会进入新分配的后端，旧对象保留在原处。', + 'storage.category.files': '行程文档', + 'storage.category.journey': '旅程照片', + 'storage.category.covers': '封面图片', + 'storage.category.avatars': '个人头像', + 'storage.category.places': '地点图片', + 'storage.category.photos-google': 'Google 照片缓存', + 'storage.category.photos-trek': 'TREK 照片缓存', + 'storage.category.backups': '备份', + + // What each category stores — rendered under the label in the category map. + 'storage.categoryDesc.files': '上传到行程的文件附件 — 车票、PDF、预订确认单，以及在行程聊天中共享的文件。', + 'storage.categoryDesc.journey': '附加到旅程条目的照片和缩略图。', + 'storage.categoryDesc.covers': '行程和收藏的封面图片，包括从 Unsplash 获取的封面。', + 'storage.categoryDesc.avatars': '用户账户的个人头像。', + 'storage.categoryDesc.places': '附加到地点和收藏地点的图片 — 上传或导入的。', + 'storage.categoryDesc.photos-google': 'Google Places 照片的缓存副本 — 可重新获取，丢失也无妨。', + 'storage.categoryDesc.photos-trek': '"照片"（Memories）功能使用的 TREK 照片服务缓存的照片 — 可重新获取，丢失也无妨。', + 'storage.categoryDesc.backups': '由备份面板或计划任务创建的服务器备份存档。', + 'storage.health.title': '健康状态', + 'storage.health.allClear': '未记录任何副本故障。', + 'storage.health.seedFile': '存在 storage-config.json 种子文件但已被忽略 — 配置行已存在。请在此管理存储。', + 'storage.health.failureLine': '{backend} 上 {key} 的 {op} 失败：{error}', + + // Replicas-on-primary mirror UX (2026-08-20 spec) + 'storage.mirror.targets': '镜像目标', + 'storage.mirror.targetsHelp': '对该后端的每次写入也会复制到每个选定的目标。', + 'storage.mirror.latencyNote': '每次上传时，副本会依次写入 — 缓慢或无法访问的目标会拖慢该后端上每个分类的每次上传。', + 'storage.mirror.mirroredTo': '已镜像至：{targets}', + 'storage.mirror.replicaOf': '副本来源：{primaries}', + 'storage.mirror.cacheWarning': '不建议：此分类保存的是可重新获取的内容 — 复制它通常是浪费的。', + 'storage.mirror.degenerate.duplicate-mirror': + '第二个镜像包裹了 {primary} — 面板仅管理第一个；移除此项以从 {primary} 管理镜像。', + 'storage.mirror.degenerate.env-primary': '包裹了一个由环境变量定义的后端 — 此处不可编辑。', + 'storage.mirror.degenerate.missing-primary': '引用了一个已不存在的后端。', + 'storage.remove.usedAsReplicaBy': '被用作副本，使用者：{primaries}', + + // Backfill + usage (backfill/stats/notifications spec) + 'storage.sync.now': '立即同步', + 'storage.sync.running': '同步中… {done}/{total}', + 'storage.sync.counts': '已复制 {copied} · 已跳过 {skipped} · 失败 {failed}', + 'storage.sync.cancel': '取消同步', + 'storage.sync.done': '同步完成：已复制 {copied}，已删除 {deleted}，失败 {failed}', + 'storage.sync.cancelled': '同步已取消', + 'storage.sync.error': '同步失败：{error}', + 'storage.sync.prompt': '现有对象尚未复制 — 立即同步？', + 'storage.sync.dismiss': '关闭', + 'storage.usage.line': '{objects} 个对象 · {size}', + 'storage.usage.computed': '用量已计算：{age}', + 'storage.usage.never': '尚未计算用量', + 'storage.usage.refresh': '刷新', + 'storage.usage.compute': '立即计算', + 'storage.usage.legacyNote': '包含旧版照片库', + + // Category migration (copy → flip → delta sweep) + 'storage.migrate.promptTitle': '将现有对象移动到新后端？', + 'storage.migrate.promptLine': '{category}：从 {from} 到 {to} 共 {objects} 个对象（{size}）', + 'storage.migrate.promptLineUnknown': '{category}：大小未知（尚未扫描用量）从 {from} 到 {to}', + 'storage.migrate.move': '移动现有对象', + 'storage.migrate.routeOnly': '仅路由新写入', + 'storage.migrate.running': '正在移动 {category}… {done}/{total}', + 'storage.migrate.done': '移动完成：已复制 {copied}，已跳过 {skipped}', + 'storage.migrate.doneFailures': '{failed} 个失败 — 这些对象未复制到新后端', + 'storage.migrate.failed': '移动失败：{error} — 该分类未切换', + 'storage.migrate.cancelled': '移动已取消 — 未切换任何内容', + 'storage.migrate.reclaimable': '{objects} 个对象（{size}）仍保留在 {from} 上 — 请手动回收', + 'storage.migrate.cancel': '取消移动', + 'storage.migrate.promptCancel': '取消', + 'storage.migrate.queued': '排队中：{categories}', + 'storage.migrate.queueDropped': '无法启动下一个迁移 — 剩余队列已清空：{categories}', +}; +export default storage; diff --git a/shared/src/index.ts b/shared/src/index.ts index 3eb02ab3a..8e1a2976a 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -44,6 +44,7 @@ export * from './book/book.schema'; export * from './book/journey-stats.schema'; export * from './book/book-store.schema'; export * from './share/share.schema'; +export * from './storage/storage.schema'; export * from './settings/settings.schema'; export * from './appearance/appearance.schema'; export * from './backup/backup.schema'; diff --git a/shared/src/storage/storage.schema.spec.ts b/shared/src/storage/storage.schema.spec.ts new file mode 100644 index 000000000..e9e6d5e18 --- /dev/null +++ b/shared/src/storage/storage.schema.spec.ts @@ -0,0 +1,185 @@ +import { + STORAGE_BACKEND_TYPES, + STORAGE_BACKEND_TYPE_IDS, + STORAGE_CATEGORIES, + storageAdminStateSchema, + storageBackfillStatusSchema, + storageConfigSchema, + storageSecretFields, + storageUsageSchema, + type StorageBackendTypeId, +} from './storage.schema'; + +import { describe, expect, it } from 'vitest'; + +/** + * Per-type samples with EVERY declared field present. The parity suite pins + * that these key sets and the generated schemas cannot drift from the field + * definitions — the registry's whole extensibility story rests on that. + */ +const SAMPLES: Record> = { + local: { root: '/data/uploads' }, + s3: { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk', + region: 'auto', + keyPrefix: 'trek/prod', + retries: 0, + timeoutMs: 5000, + }, + mirror: { primary: 'backups-local', replicas: ['nas-backups'] }, +}; + +describe('STORAGE_BACKEND_TYPES generation parity', () => { + it.each(STORAGE_BACKEND_TYPE_IDS)('%s: sample keys equal the declared field keys exactly', (type) => { + const declared = STORAGE_BACKEND_TYPES[type].fields.map((f) => f.key).sort(); + expect(Object.keys(SAMPLES[type]).sort()).toEqual(declared); + }); + + it.each(STORAGE_BACKEND_TYPE_IDS)('%s: every declared field round-trips through the generated schema', (type) => { + const parsed = STORAGE_BACKEND_TYPES[type].optionsSchema.parse(SAMPLES[type]) as Record; + for (const field of STORAGE_BACKEND_TYPES[type].fields) { + expect(parsed).toHaveProperty(field.key); + } + }); + + it.each(STORAGE_BACKEND_TYPE_IDS)('%s: unknown option keys are rejected (strict at every level)', (type) => { + const result = STORAGE_BACKEND_TYPES[type].optionsSchema.safeParse({ ...SAMPLES[type], rogue: 'x' }); + expect(result.success).toBe(false); + }); + + it('secret fields are required strings (the encrypt/mask machinery depends on it)', () => { + expect(storageSecretFields('s3')).toEqual(['secretAccessKey']); + expect(storageSecretFields('local')).toEqual([]); + expect(storageSecretFields('mirror')).toEqual([]); + for (const type of STORAGE_BACKEND_TYPE_IDS) { + for (const field of STORAGE_BACKEND_TYPES[type].fields) { + if (field.kind !== 'secret') continue; + expect(field.required).toBe(true); + const parsed = STORAGE_BACKEND_TYPES[type].optionsSchema.parse(SAMPLES[type]) as Record; + expect(typeof parsed[field.key]).toBe('string'); + } + } + }); + + it('every non-required field carries a defaultValue, so parsed options are always total', () => { + for (const type of STORAGE_BACKEND_TYPE_IDS) { + for (const field of STORAGE_BACKEND_TYPES[type].fields) { + if (!field.required) expect(field.defaultValue).toBeDefined(); + } + } + }); + + it('defaults fill omitted optional s3 fields', () => { + const parsed = STORAGE_BACKEND_TYPES.s3.optionsSchema.parse({ + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk', + }); + expect(parsed).toMatchObject({ region: 'us-east-1', keyPrefix: '', retries: 1, timeoutMs: 30000 }); + }); + + it('field refinements layer onto the base kind (endpoint URL, non-negative retries, positive timeout)', () => { + const base = SAMPLES.s3; + expect(STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse({ ...base, endpoint: 'not a url' }).success).toBe(false); + expect(STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse({ ...base, retries: -1 }).success).toBe(false); + expect(STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse({ ...base, timeoutMs: 0 }).success).toBe(false); + }); +}); + +describe('storageConfigSchema wire compatibility', () => { + it('accepts the hand-written S3-slice row shapes byte-for-byte', () => { + // The exact local→NAS mirror example the S3 spec documents as raw SQL. + const doc = { + backends: [ + { name: 'nas-backups', type: 'local', options: { root: '/mnt/nas/trek-backups' } }, + { name: 'backup-mirror', type: 'mirror', options: { primary: 'backups-local', replicas: ['nas-backups'] } }, + ], + categories: { backups: 'backup-mirror' }, + }; + expect(storageConfigSchema.parse(doc)).toEqual(doc); + }); + + it('accepts a settings-declared s3 backend (the v2 acceptance this slice ships)', () => { + const doc = { + backends: [{ name: 'off-box', type: 's3', options: SAMPLES.s3 }], + categories: { backups: 'off-box' }, + }; + expect(storageConfigSchema.safeParse(doc).success).toBe(true); + }); + + it('rejects the reserved readOnly top-level key — loud until a file-override mode is implemented', () => { + const result = storageConfigSchema.safeParse({ backends: [], categories: {}, readOnly: true }); + expect(result.success).toBe(false); + }); + + it('rejects unknown categories, unknown types, empty names, and empty backend refs', () => { + expect(storageConfigSchema.safeParse({ backends: [], categories: { nope: 'uploads-local' } }).success).toBe(false); + expect( + storageConfigSchema.safeParse({ backends: [{ name: 'x', type: 'ftp', options: {} }], categories: {} }).success, + ).toBe(false); + expect( + storageConfigSchema.safeParse({ + backends: [{ name: '', type: 'local', options: { root: '/x' } }], + categories: {}, + }).success, + ).toBe(false); + expect(storageConfigSchema.safeParse({ backends: [], categories: { backups: '' } }).success).toBe(false); + }); + + it('exposes the canonical eight-category list — photos is served-legacy, retired from config', () => { + expect(STORAGE_CATEGORIES).toEqual([ + 'files', + 'journey', + 'covers', + 'avatars', + 'places', + 'photos-google', + 'photos-trek', + 'backups', + ]); + }); + + it('rejects the retired photos category in a config document', () => { + const result = storageConfigSchema.safeParse({ backends: [], categories: { photos: 'uploads-local' } }); + expect(result.success).toBe(false); + }); +}); + +describe('backfill + usage wire shapes', () => { + it('storageBackfillStatusSchema round-trips a running and a finished status', () => { + const running = { + backend: 'backups-local-mirror', + status: 'running', + done: 3, + total: 10, + copied: 2, + skipped: 1, + failed: 0, + deleted: 1, + startedAt: 1_700_000_000_000, + }; + expect(storageBackfillStatusSchema.parse(running)).toEqual(running); + const done = { ...running, status: 'done', done: 10, finishedAt: 1_700_000_100_000 }; + expect(storageBackfillStatusSchema.parse(done)).toEqual(done); + expect(storageBackfillStatusSchema.safeParse({ ...running, status: 'paused' }).success).toBe(false); + }); + + it('storageUsageSchema carries all configurable categories plus the legacy photos bucket', () => { + const perCategory = { objects: 1, bytes: 2 }; + const usage = { + computedAt: 1_700_000_000_000, + categories: Object.fromEntries(STORAGE_CATEGORIES.map((c) => [c, perCategory])), + legacyPhotos: { objects: 0, bytes: 0 }, + }; + expect(storageUsageSchema.parse(usage)).toEqual(usage); + }); + + it('the admin state embeds usage (nullable) and backfills', () => { + const stateShape = storageAdminStateSchema.pick({ usage: true, backfills: true }); + expect(stateShape.parse({ usage: null, backfills: [] })).toEqual({ usage: null, backfills: [] }); + }); +}); diff --git a/shared/src/storage/storage.schema.ts b/shared/src/storage/storage.schema.ts new file mode 100644 index 000000000..020776650 --- /dev/null +++ b/shared/src/storage/storage.schema.ts @@ -0,0 +1,326 @@ +import { z } from 'zod'; + +/** + * Storage admin contract (spec: docs/superpowers/specs/2026-08-19-storage-admin-config-design.md). + * + * STORAGE_BACKEND_TYPES is the single schema-driven registry: each backend + * type declares its fields once, and both the Zod options schema (generated + * below) and the admin form's field metadata render from that one definition. + * Adding a backend type = one entry here + one driver class + one registry + * construction line. The registry is deliberately consumer-agnostic — the + * admin panel is its first renderer and a standalone seed-file generator is + * the planned second; do not specialize it toward either. + * + * Shared Zod validates SHAPE only. Semantics (name references, mirror + * composition, mirror-only-on-backups, no nesting, key rules) stay in the + * server registry's validators — the settled two-layer pattern. + */ + +/** + * Canonical CONFIGURABLE category list — relocated here from server + * storage.types.ts (which re-exports it) because the wire schema needs it and + * a second copy is forbidden. The legacy `photos` directory is served and + * backed up but not configurable — the server's SERVED_CATEGORIES extension + * (storage.types.ts) carries it. + */ +export const STORAGE_CATEGORIES = [ + 'files', + 'journey', + 'covers', + 'avatars', + 'places', + 'photos-google', + 'photos-trek', + 'backups', +] as const; +export type StorageCategory = (typeof STORAGE_CATEGORIES)[number]; +export const storageCategorySchema = z.enum(STORAGE_CATEGORIES); + +export const STORAGE_BACKEND_TYPE_IDS = ['local', 's3', 'mirror'] as const; +export type StorageBackendTypeId = (typeof STORAGE_BACKEND_TYPE_IDS)[number]; + +export interface StorageBackendFieldDef { + key: string; + kind: 'text' | 'path' | 'secret' | 'number' | 'backend-ref' | 'backend-ref-list'; + /** i18n key under the `storage` domain (locale files land with the client slice). */ + labelKey: string; + helpKey?: string; + required: boolean; + defaultValue?: string | number; + /** Optional per-field refinement piped onto the kind's base Zod type. */ + refine?: { schema: z.ZodTypeAny }; +} + +function fieldSchema(field: StorageBackendFieldDef): z.ZodTypeAny { + let base: z.ZodTypeAny; + switch (field.kind) { + case 'number': + base = z.number(); + break; + case 'backend-ref': + base = z.string().min(1); + break; + case 'backend-ref-list': + base = z.array(z.string().min(1)); + break; + default: // text | path | secret + base = field.required ? z.string().min(1) : z.string(); + } + if (field.defaultValue !== undefined) base = base.default(field.defaultValue); + if (field.refine) base = base.pipe(field.refine.schema); + return base; +} + +type FieldValue = F['kind'] extends 'number' + ? number + : F['kind'] extends 'backend-ref-list' + ? string[] + : string; +/** Parsed options are total: non-required fields must declare a defaultValue (spec pin). */ +export type StorageOptionsOf = { + [F in Fs[number] as F['key']]: FieldValue; +}; + +function optionsSchemaFor( + fields: Fs, +): z.ZodType> { + const shape: Record = {}; + for (const field of fields) shape[field.key] = fieldSchema(field); + // The one sanctioned cast: the runtime schema is generated from the same + // field defs the mapped type reads, and the parity spec pins the two. + return z.strictObject(shape) as unknown as z.ZodType>; +} + +const LOCAL_FIELDS = [ + { + key: 'root', + kind: 'path', + labelKey: 'storage.field.root', + helpKey: 'storage.help.root', + required: true, + }, +] as const satisfies readonly StorageBackendFieldDef[]; + +const S3_FIELDS = [ + { + key: 'endpoint', + kind: 'text', + labelKey: 'storage.field.endpoint', + helpKey: 'storage.help.endpoint', + required: true, + refine: { schema: z.url() }, + }, + { key: 'bucket', kind: 'text', labelKey: 'storage.field.bucket', required: true }, + { key: 'accessKeyId', kind: 'text', labelKey: 'storage.field.accessKeyId', required: true }, + { key: 'secretAccessKey', kind: 'secret', labelKey: 'storage.field.secretAccessKey', required: true }, + { + key: 'region', + kind: 'text', + labelKey: 'storage.field.region', + helpKey: 'storage.help.region', + required: false, + defaultValue: 'us-east-1', + }, + { + key: 'keyPrefix', + kind: 'text', + labelKey: 'storage.field.keyPrefix', + helpKey: 'storage.help.keyPrefix', + required: false, + defaultValue: '', + }, + { + key: 'retries', + kind: 'number', + labelKey: 'storage.field.retries', + required: false, + defaultValue: 1, + refine: { schema: z.number().int().min(0) }, + }, + { + key: 'timeoutMs', + kind: 'number', + labelKey: 'storage.field.timeoutMs', + required: false, + defaultValue: 30000, + refine: { schema: z.number().int().positive() }, + }, +] as const satisfies readonly StorageBackendFieldDef[]; + +const MIRROR_FIELDS = [ + { key: 'primary', kind: 'backend-ref', labelKey: 'storage.field.primary', required: true }, + { key: 'replicas', kind: 'backend-ref-list', labelKey: 'storage.field.replicas', required: true }, +] as const satisfies readonly StorageBackendFieldDef[]; + +export const STORAGE_BACKEND_TYPES = { + local: { fields: LOCAL_FIELDS, optionsSchema: optionsSchemaFor(LOCAL_FIELDS) }, + s3: { fields: S3_FIELDS, optionsSchema: optionsSchemaFor(S3_FIELDS) }, + mirror: { fields: MIRROR_FIELDS, optionsSchema: optionsSchemaFor(MIRROR_FIELDS) }, +} as const; + +export type StorageLocalOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['local']['optionsSchema']>; +export type StorageS3Options = z.infer<(typeof STORAGE_BACKEND_TYPES)['s3']['optionsSchema']>; +export type StorageMirrorOptions = z.infer<(typeof STORAGE_BACKEND_TYPES)['mirror']['optionsSchema']>; + +/** Field keys the type marks `secret` — the exact set encrypted at rest and masked on GET. */ +export function storageSecretFields(type: StorageBackendTypeId): readonly string[] { + return STORAGE_BACKEND_TYPES[type].fields.filter((f) => f.kind === 'secret').map((f) => f.key); +} + +export const storageBackendSchema = z.discriminatedUnion('type', [ + z.strictObject({ + name: z.string().min(1), + type: z.literal('local'), + options: STORAGE_BACKEND_TYPES.local.optionsSchema, + }), + z.strictObject({ + name: z.string().min(1), + type: z.literal('s3'), + options: STORAGE_BACKEND_TYPES.s3.optionsSchema, + }), + z.strictObject({ + name: z.string().min(1), + type: z.literal('mirror'), + options: STORAGE_BACKEND_TYPES.mirror.optionsSchema, + }), +]); +export type StorageBackend = z.infer; + +/** + * The two app_settings rows as one document — the PUT body and the seed-file + * shape. `.strict()` at every level: unknown keys are rejected, including the + * reserved top-level `readOnly` seed key (loud until a standing file-override + * mode is actually implemented). Byte-compatible with rows hand-written + * during the S3 slice (pinned in the sibling spec file). + */ +export const storageConfigSchema = z.strictObject({ + backends: z.array(storageBackendSchema), + categories: z.partialRecord(storageCategorySchema, z.string().min(1)), +}); +export type StorageConfig = z.infer; + +/** + * The PUT wire shape: the config document plus the optimistic-concurrency + * version the client read it at (audit #7 — a stale PUT can otherwise silently + * undo a category migration's just-flipped assignment). `storageConfigSchema` + * itself stays untouched — it doubles as the seed-file shape, which has no + * version to compare against. + */ +export const storageConfigPutSchema = storageConfigSchema.extend({ + version: z.number().int().nonnegative(), +}); +export type StorageConfigPut = z.infer; + +// ── GET /api/admin/storage state + POST /test wire shapes ───────────────────── + +export const storageBackendSourceSchema = z.enum(['built-in', 'env', 'settings']); +export const storageCategorySourceSchema = z.enum(['default', 'settings']); + +export const storageReplicaFailureSchema = z.object({ + backend: z.string(), + key: z.string(), + /** 'list' is the backfill sweep phase's replica.list() failing. */ + op: z.enum(['put', 'delete', 'stat', 'list']), + error: z.string(), + at: z.number(), +}); + +// ── Backfill + usage (backfill/stats/notifications spec) ───────────────────── + +export const storageBackfillStatusSchema = z.object({ + /** The mirror's wire name — user-facing copy says "Sync", never this. */ + backend: z.string(), + status: z.enum(['running', 'done', 'error', 'cancelled']), + /** Objects examined in the copy phase; total is 0 while enumerating. */ + done: z.number(), + total: z.number(), + copied: z.number(), + skipped: z.number(), + failed: z.number(), + /** Replica objects removed by the sweep phase — absent from the primary and not raced back. */ + deleted: z.number(), + startedAt: z.number(), + finishedAt: z.number().optional(), + /** status 'error' only: the aborting (primary-side) error. */ + error: z.string().optional(), +}); +export type StorageBackfillStatus = z.infer; + +// ── Category migration (copy → flip → delta sweep) ──────────────────────────── + +export const storageMigrationStatusSchema = z.object({ + category: storageCategorySchema, + from: z.string(), + to: z.string(), + status: z.enum(['running', 'done', 'failed', 'cancelled']), + /** Keys examined in the copy phase; total is 0 while enumerating. */ + done: z.number(), + total: z.number(), + copied: z.number(), + skipped: z.number(), + failed: z.number(), + startedAt: z.number(), + finishedAt: z.number().optional(), + /** status 'failed' only: the aborting error (copy failures also set counts). */ + error: z.string().optional(), + /** Terminal 'done' only: objects/bytes left on the old backend. */ + reclaimable: z.object({ objects: z.number(), bytes: z.number() }).optional(), +}); +export type StorageMigrationStatus = z.infer; + +export const storageUsageSchema = z.object({ + computedAt: z.number(), + categories: z.record(storageCategorySchema, z.object({ objects: z.number(), bytes: z.number() })), + /** The served-legacy photos directory — attributed to whatever backend `photos` resolves to. */ + legacyPhotos: z.object({ objects: z.number(), bytes: z.number() }), +}); +export type StorageUsage = z.infer; + +export const storageAdminStateSchema = z.object({ + backends: z.array( + z.object({ + name: z.string(), + type: z.enum(STORAGE_BACKEND_TYPE_IDS), + source: storageBackendSourceSchema, + /** Secret-kind fields carry MASKED_SETTING_VALUE — never a stored value. */ + options: z.record(z.string(), z.union([z.string(), z.number(), z.array(z.string())])), + categories: z.array(storageCategorySchema), + }), + ), + // Exhaustive record: the effective world always maps all eight configurable categories. + categories: z.record( + storageCategorySchema, + z.object({ backend: z.string().min(1), source: storageCategorySourceSchema }), + ), + health: z.object({ replicaFailures: z.array(storageReplicaFailureSchema) }), + seedFilePresent: z.boolean(), + usage: storageUsageSchema.nullable(), + backfills: z.array(storageBackfillStatusSchema), + migrations: z.array(storageMigrationStatusSchema), + /** Optimistic-concurrency counter for the two settings rows (audit #7) — echo it back on the next PUT. */ + version: z.number().int().nonnegative(), + /** + * Non-null when the stored storage settings failed to parse/validate and + * the registry fell back to the last-good config (or built-in defaults) — + * the admin panel renders this as a warning banner so a save is never a + * silent replacement of a config the operator never actually saw (audit + * minor). Null once the stored settings load cleanly. + */ + configError: z.string().nullable(), +}); +export type StorageAdminState = z.infer; + +export const storageTestRequestSchema = z.strictObject({ backend: storageBackendSchema }); +export type StorageTestRequest = z.infer; + +export const storageMigrationRequestSchema = z.strictObject({ + category: storageCategorySchema, + to: z.string().min(1), +}); +export type StorageMigrationRequest = z.infer; + +export const storageTestResponseSchema = z.object({ + ok: z.boolean(), + targets: z.array(z.object({ name: z.string(), ok: z.boolean(), error: z.string().optional() })), +}); +export type StorageTestResponse = z.infer; diff --git a/wiki/Admin-Panel-Overview.md b/wiki/Admin-Panel-Overview.md index 9fddadb3c..088f99f22 100644 --- a/wiki/Admin-Panel-Overview.md +++ b/wiki/Admin-Panel-Overview.md @@ -19,12 +19,13 @@ The Admin Panel is divided into tabs. Most tabs are always visible; a few appear | **User Defaults** | Default settings applied to new users | No | | **Addons** | Enable or disable optional features instance-wide | No | | **Plugins** | Install, update, and manage plugins; rescan the plugins folder; review each plugin's capability audit. See [Admin-Plugins](Admin-Plugins) | No | +| **Storage** | Storage backends, category assignment, replication, health | Hidden on managed instances | | **Settings** | Authentication methods, MFA, allowed file types, API keys, OIDC/SSO configuration, and JWT secret rotation | No | | **Notifications** | SMTP, webhook, ntfy, and push notification channel configuration; trip reminder toggle; admin notification preferences | No | -| **Backup** | Manual and scheduled database backups | No | +| **Backup** | Manual and scheduled database backups | Hidden on managed instances | | **Audit** | Chronological activity log | No | | **MCP Access** | OAuth sessions and static API tokens | Only when the MCP addon is enabled | -| **GitHub** | Release timeline and support links | No | +| **GitHub** | Release timeline and support links | Hidden on managed instances | | **Dev: Notifications** | Test notification dispatch | Only in development mode (`NODE_ENV=development`) | ![Admin panel on the User Defaults tab, setting instance-wide defaults for colour mode, temperature unit, distance unit, time format, currency and blurred booking codes](assets/AdminUserDefaults.png) @@ -45,5 +46,6 @@ See [Audit-Log](Audit-Log) for details on the hash chain and how the two logs di - [Admin-Categories](Admin-Categories) - [Admin-Packing-Templates](Admin-Packing-Templates) - [Admin-Permissions](Admin-Permissions) +- [[Admin: Storage|Admin-Storage]] - [Admin-MCP-Tokens](Admin-MCP-Tokens) - [Admin-GitHub-Releases](Admin-GitHub-Releases) diff --git a/wiki/Admin-Storage.md b/wiki/Admin-Storage.md new file mode 100644 index 000000000..e9f473fdb --- /dev/null +++ b/wiki/Admin-Storage.md @@ -0,0 +1,165 @@ +# Admin: Storage + +The **Storage** tab (Admin Panel → Storage) controls where TREK stores each +kind of content: which storage backends exist, which content category writes +to which backend, and whether writes are replicated to additional backends. +On managed/hosted instances this tab is hidden — storage is the operator's +concern. + +## Backends + +A backend is a named place bytes can live. Two exist out of the box: + +| Backend | What it is | +|---------|------------| +| `uploads-local` | The local `uploads/` directory (the `/app/uploads` volume in Docker) | +| `backups-local` | The local `data/backups` directory | + +You can **edit** a built-in to relocate its root directory, and **add** +backends of type: + +- **Local** — a directory on a disk or mount reachable by the server. +- **S3** — any S3-compatible service (AWS S3, Cloudflare R2, Backblaze B2, + Garage, MinIO/AIStor). Endpoint URL, bucket, credentials, and optionally + region, key prefix, retries, and timeout. For self-hosted endpoints, use an + IP address or `localhost` in the URL unless your server is configured for + virtual-hosted bucket addressing. + +If `TREK_PLACE_PHOTO_DIR` is set, a read-only `place-photos-local` backend +appears too — it is defined by that environment variable, not by this panel. + +### Replication (Mirror targets) + +Edit any backend and tick one or more **Mirror targets**: every write to that +backend is then also copied to each selected target. Removing all targets +turns replication off again. + +- A backend can serve as a **mirror target or as a category target, not + both** — only backends no category is stored on are offered in the list, and + saving a configuration that mixes the two is refused. A sync makes a target + match its backend (see below), so anything else stored there would be + deleted. +- Writes go to the backend itself first (it stays the source of truth), then + to each target in order. A slow or unreachable target slows every upload of + every category on that backend — ideal for backups, worth weighing before + replicating hot categories like trip documents. +- Replica failures **never fail the original request**. They are recorded and + shown in the **Health** strip at the top of the tab, with the backend, the + object, the operation, and how long ago it happened. An empty strip means + every replicated write landed. Replica failures also notify admins — in-app + and over the configured webhook/ntfy/email channels (first failure + immediately, then at most one summary per backend per hour). Preferences + live in the admin Notifications tab. +- Replication starts with the next write. To copy objects that existed before + you added a target, press **Sync now** on the backend's row — one sync runs + at a time, with progress and a cancel button; objects already present with + matching sizes are skipped, and per-object failures land in the Health + strip without stopping the run. A sync makes the target **match** the + backend, not just catch up: it also deletes target objects that no longer + exist on the backend — it's not a trash can, so run it only when you want + the target pruned to match, and stray deletions land in the Health strip + like any other replica failure. + +### Test + +Every backend row has a **Test** button: the server writes, checks, and +deletes a small probe object (`trek-probe/…`) on the backend — for a +replicated backend, on the backend itself and each target individually — and +reports per-target results. "Connection OK" means credentials, bucket, and +reachability all check out. Targets must be **saved** before Test can probe +them; a just-added, unsaved backend used as a target reports an error until +you save. + +## Categories + +Every kind of content TREK stores belongs to one of eight categories, each +assigned to exactly one backend: + +| Category | Id | What it stores | +|----------|----|----------------| +| Trip documents | `files` | File attachments uploaded to trips — tickets, PDFs, booking confirmations, and files shared in trip chat | +| Journey photos | `journey` | Photos and thumbnails attached to journey entries | +| Cover images | `covers` | Trip and collection cover images, including covers fetched from Unsplash | +| Profile pictures | `avatars` | User account profile pictures | +| Place images | `places` | Images attached to places and collection places — uploaded or imported | +| Google photo cache | `photos-google` | Cached copies of Google Places photos — re-fetchable, safe to lose | +| TREK photo cache | `photos-trek` | Cached photos from the TREK photo service used by Memories — re-fetchable, safe to lose | +| Backups | `backups` | Server backup archives created by the Backup panel or schedule | + +Reassigning a category changes where **new** objects go. If the category +already holds objects, the panel asks first: **Move existing objects** copies +everything to the new backend, switches the category, then sweeps any writes +that raced the copy; **Just route new writes** keeps the previous behavior — +new objects go to the new backend, existing ones stay put and keep being +served from wherever they already are. Replicating the Google photo cache or +Place images is flagged as not recommended — both hold content that is +re-fetchable or provider-derived. + +The legacy `/uploads/photos` directory written by older TREK versions is not +a category: its files are still served and included in backups, but nothing +writes there anymore and it cannot be reassigned. + +### Moving a category's objects + +A category migration shares its one-at-a-time slot with mirror syncs (see +*Replication* above) — starting a move while a sync (or another move) is +running is refused, and vice versa. Progress (done/total, copied/skipped/ +failed) shows inline with a cancel button. + +Cancelling before the category switches is completely safe: nothing has +changed yet, the copy just stops. A failed copy behaves the same way — the +category is never switched if any object failed to copy. Once the switch has +happened, the delta sweep (catching writes that landed on the old backend +while the copy was running) always runs to completion, even if you cancel. + +The old objects are **not** deleted when a move finishes — that's what the +reclaimable line means (`N objects (size) remain on — reclaim +manually`). It's left behind on purpose so you can confirm the new backend +looks right before removing it yourself. + +## Usage + +The Backends section shows per-backend and per-category object counts and +sizes, scanned nightly (04:15 server time) and on demand via **Refresh**. The +stamp shows when the numbers were computed — reassigning categories does not +retroactively move them until the next scan. The uploads backend's total +includes the legacy photo library. + +## Secrets and encryption + +Backend credentials (the S3 secret access key) are stored encrypted. Setting +the `ENCRYPTION_KEY` environment variable explicitly is recommended — without +it the implicit key persisted in the data directory is used, which works but +rides inside backup archives (someone holding a backup can decrypt the +secrets it contains). If you later move from the implicit key to an explicit +one, follow [[Encryption Key Rotation|Encryption-Key-Rotation]] — setting a +new key directly would orphan already-stored secrets. Saved secrets display +as a masked placeholder; leaving the mask untouched when editing keeps the +stored value. + +## Provisioning at first boot (seed file) + +For scripted deployments, place a `storage-config.json` in the data directory +(in Docker: mount it at `/app/data/storage-config.json`). It is imported +exactly once — on the first boot that has no stored storage configuration — +then ignored (a log line says so). An invalid file aborts boot with the exact +validation error. Example and the compose mount line: see the +[README Storage section](https://github.com/liketrek/TREK#storage). + +To reset storage configuration (or re-import a seed file): stop the server, +`sqlite3 data/travel.db "DELETE FROM app_settings WHERE key LIKE 'storage.%';"`, +start it again — it boots on the built-in defaults, or re-imports the seed +file if present. + +**Restore chicken-and-egg:** backups include the storage configuration. If +your only backup lives on S3 and the credentials are inside it: start a fresh +instance, enter the S3 credentials here (or mount a seed file), then restore +from the Backup panel. + +## Related pages + +- [[Backups|Backups]] — what a backup contains, restore procedure +- [[Environment Variables|Environment-Variables]] +- [[Encryption Key Rotation|Encryption-Key-Rotation]] +- [[Admin Panel Overview|Admin-Panel-Overview]] +- [[Troubleshooting|Troubleshooting]] diff --git a/wiki/Backups.md b/wiki/Backups.md index 71c368f06..f9a78ea3b 100644 --- a/wiki/Backups.md +++ b/wiki/Backups.md @@ -1,6 +1,6 @@ # Backups -TREK stores all data in a single SQLite database (`travel.db`) plus an `uploads/` directory of attachments, cover photos, and avatars. The Backup panel lets you create, download, restore, and schedule backups of both. +TREK stores all data in a single SQLite database (`travel.db`) plus an `uploads/` directory of attachments, cover photos, and avatars — by default; storage backends and replication are configured in [[Admin: Storage|Admin-Storage]]. The Backup panel lets you create, download, restore, and schedule backups of both. ## Where to find it @@ -15,7 +15,7 @@ A backup is a ZIP archive with these entries: | Entry | Contents | |---|---| | `travel.db` | The full SQLite database | -| `uploads/` | All uploaded attachments, covers, and avatars | +| `uploads/` | All uploaded attachments, covers, and avatars (default location — see [[Admin: Storage|Admin-Storage]]) | | `plugins-data/` | Each installed plugin's own database + files (present only if plugins are installed) | | `plugins-code/` | The installed plugin code, so a restore is self-contained (dev-linked plugins are skipped) | @@ -84,6 +84,7 @@ The following actions are recorded in the [Audit-Log](Audit-Log): ## See also +- [[Admin: Storage|Admin-Storage]] — replicate backups to S3-compatible storage - [Encryption-Key-Rotation](Encryption-Key-Rotation) - [Admin-Panel-Overview](Admin-Panel-Overview) - [Security-Hardening](Security-Hardening) diff --git a/wiki/Encryption-Key-Rotation.md b/wiki/Encryption-Key-Rotation.md index 847f459d7..b0d558cf9 100644 --- a/wiki/Encryption-Key-Rotation.md +++ b/wiki/Encryption-Key-Rotation.md @@ -16,6 +16,7 @@ TREK encrypts sensitive settings at rest using AES-256-GCM. The following values - Admin webhook URL and admin ntfy token (global, in `app_settings`) - MFA (TOTP) secrets for all users - Photo passphrases for Synology shared-link photos (in `trek_photos`) +- S3 storage backend secret access keys (in `app_settings['storage.backends']`; see [Admin-Storage](Admin-Storage)) The encryption derives a key from `ENCRYPTION_KEY` using SHA-256 (with a domain suffix per secret type), so the raw `ENCRYPTION_KEY` value is never stored in the database. @@ -61,6 +62,7 @@ The script: 3. Creates a timestamped backup of the database (e.g. `travel.db.backup-1713484800000`) before modifying anything. 4. Re-encrypts all stored secrets across all tables: - `app_settings`: `oidc_client_secret`, `smtp_pass`, `admin_webhook_url`, `admin_ntfy_token`, `maps_api_key`, `unsplash_api_key` + - `app_settings['storage.backends']`: the `secretAccessKey` of every S3 storage backend - `users` (per user): `maps_api_key`, `unsplash_api_key`, `openweather_api_key`, `immich_api_key`, `synology_password`, `synology_sid`, `synology_did`, `mfa_secret` - `settings` (per user): `webhook_url`, `ntfy_token`, `mapbox_access_token` - `trip_album_links`: `passphrase` diff --git a/wiki/Environment-Variables.md b/wiki/Environment-Variables.md index c01addc4e..a32a23dcf 100644 --- a/wiki/Environment-Variables.md +++ b/wiki/Environment-Variables.md @@ -275,6 +275,10 @@ To get a key: create a free account at [unsplash.com/developers](https://unsplas ## Storage & Paths +Storage backends and category assignment are configured in +[[Admin: Storage|Admin-Storage]] (or a seed-once `storage-config.json`), not +through environment variables. `TREK_PLACE_PHOTO_DIR` below is unaffected. + | Variable | Description | Default | |--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------| | `TREK_PLACE_PHOTO_DIR` | Directory where cached Google place photos are stored. Created recursively on boot. Set this to point photo storage at a dedicated mounted volume. | `uploads/photos/google` | diff --git a/wiki/_Sidebar.md b/wiki/_Sidebar.md index 8be0791db..332544376 100644 --- a/wiki/_Sidebar.md +++ b/wiki/_Sidebar.md @@ -94,6 +94,7 @@ - [[Admin: Packing Templates|Admin-Packing-Templates]] - [[Admin: Permissions|Admin-Permissions]] - [[Admin: Plugins|Admin-Plugins]] +- [[Admin: Storage|Admin-Storage]] - [[Admin: MCP Tokens|Admin-MCP-Tokens]] - [[Admin: GitHub Releases|Admin-GitHub-Releases]]