From 5758471e4e381a8514f4a585667311cdb4953c76 Mon Sep 17 00:00:00 2001 From: jubnl Date: Thu, 20 Aug 2026 00:30:47 +0200 Subject: [PATCH] =?UTF-8?q?feat(server):=20accept=20settings-declared=20s3?= =?UTF-8?q?=20backends=20=E2=80=94=20shared-schema=20options,=20decrypt-at?= =?UTF-8?q?-build,=20registry=20preview()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nest/storage/storage-registry.service.ts | 50 ++++++- .../storage/storage-registry.service.test.ts | 122 ++++++++++++++++-- 2 files changed, 158 insertions(+), 14 deletions(-) diff --git a/server/src/nest/storage/storage-registry.service.ts b/server/src/nest/storage/storage-registry.service.ts index 315ee2bed..e3a8bc18e 100644 --- a/server/src/nest/storage/storage-registry.service.ts +++ b/server/src/nest/storage/storage-registry.service.ts @@ -1,7 +1,9 @@ import fs from 'node:fs'; import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { STORAGE_BACKEND_TYPES } from '@trek/shared'; import { DatabaseService } from '../database/database.service'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { LocalDriver } from './drivers/local.driver'; import { MirrorDriver, type ReplicaFailure } from './drivers/mirror.driver'; import { S3Driver } from './drivers/s3.driver'; @@ -108,6 +110,19 @@ export class StorageRegistryService implements OnModuleInit { this.load(false); } + /** + * Validate a candidate config by running the real build() — merge over + * built-ins → parse → validateConfig → driver construction (network-free) — + * and discard the result. Throws exactly what a failing load would log; + * never touches this.state (admin writes must not take the silent boot-time + * fallback). Shares one side effect with any successful save: local roots + * and prefix dirs are created — an uncreatable root is exactly the error to + * surface before persisting. cleanSpool stays boot-only (boot=false here). + */ + preview(candidate: { backends: unknown; categories: unknown }): void { + void this.build(candidate, false); + } + resolve(category: StorageCategory): ResolvedCategory { if (!this.state) throw new StorageBackendError('storage registry not initialized'); const assignment = this.state.categories.get(category); @@ -227,7 +242,10 @@ export class StorageRegistryService implements OnModuleInit { } for (const config of backends.values()) { if (config.type !== 's3') continue; - drivers.set(config.name, new S3Driver({ id: config.name, ...config.options })); + drivers.set( + config.name, + new S3Driver({ id: config.name, ...config.options, secretAccessKey: decryptedSecret(config) }), + ); } for (const config of backends.values()) { if (config.type !== 'mirror') continue; @@ -277,10 +295,17 @@ function parseBackendList(raw: unknown): BackendConfig[] { return { name: entry.name, type: 'mirror', options: { primary: options.primary, replicas: replicas as string[] } }; } if (entry.type === 's3') { - throw new StorageBackendError( - `s3 backend '${entry.name}' cannot be declared in '${BACKENDS_KEY}' — s3 backends are ` + - 'env-declared only in v1 (TREK_S3_*; credentials never live in the database)', - ); + const parsed = STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse(options); + if (!parsed.success) { + const first = parsed.error.issues[0]; + throw new StorageBackendError( + `s3 backend '${entry.name}' has invalid options` + + (first ? ` — ${first.path.join('.') || '(options)'}: ${first.message}` : ''), + ); + } + // secretAccessKey stays as stored (usually enc:v1:) — decrypted only at + // driver construction, so config maps and snapshots never hold plaintext. + return { name: entry.name, type: 's3', options: parsed.data }; } throw new StorageBackendError(`backend '${entry.name}' has unknown type '${String(entry.type)}'`); }); @@ -300,6 +325,21 @@ function parseCategoryMap(raw: unknown): Array<[StorageCategory, string]> { }); } +/** + * Secrets live encrypted inside the storage.backends JSON (admin-config spec); + * plaintext passthrough is tolerated as belt-and-braces (decrypt_api_key's + * legacy-plaintext rule), though seed and PUT always store encrypted. + */ +function decryptedSecret(config: S3BackendConfig): string { + const plain = decrypt_api_key(config.options.secretAccessKey); + if (plain === null) { + throw new StorageBackendError( + `s3 backend '${config.name}': could not decrypt 'secretAccessKey' — was ENCRYPTION_KEY changed or the row edited by hand?`, + ); + } + return plain; +} + function validateConfig(backends: Map, categories: Map): void { for (const config of backends.values()) { if (config.type !== 'mirror') continue; diff --git a/server/tests/unit/nest/storage/storage-registry.service.test.ts b/server/tests/unit/nest/storage/storage-registry.service.test.ts index 396556a3f..e90f325d0 100644 --- a/server/tests/unit/nest/storage/storage-registry.service.test.ts +++ b/server/tests/unit/nest/storage/storage-registry.service.test.ts @@ -14,6 +14,7 @@ const { testDb, dbMock } = vi.hoisted(() => { }); vi.mock('../../../../src/db/database', () => dbMock); +vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-registry-test-key' })); import fs from 'node:fs'; import os from 'node:os'; @@ -24,6 +25,7 @@ import { runMigrations } from '../../../../src/db/migrations'; import { DatabaseService } from '../../../../src/nest/database/database.service'; import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; import type { deriveS3 } from '../../../../src/app-config/derive'; +import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto'; import { StorageRegistryService } from '../../../../src/nest/storage/storage-registry.service'; import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver'; import { MirrorDriver, type ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver'; @@ -204,7 +206,7 @@ describe('StorageRegistryService settings', () => { JSON.stringify({ backups: 'm2' }), ], [ - 'settings-declared s3 backend (env-only in v1)', + 'settings s3 backend with invalid options', JSON.stringify([{ name: 'x', type: 's3', options: {} }]), undefined, ], @@ -338,16 +340,118 @@ describe('s3-main (env-declared)', () => { const { registry } = makeRegistry({}, { s3: S3_ON }); expect(registry.resolve('backups').backendName).toBe('backup-mirror'); }); +}); - it('rejects a settings-declared s3 backend with the intentional message', () => { +// ── settings-declared s3 backends (admin-config slice: acceptance) ─────────── + +describe('settings-declared s3 backends', () => { + const s3Options = { + endpoint: 'http://127.0.0.1:9000', + bucket: 'trek', + accessKeyId: 'ak', + secretAccessKey: 'sk-plain', + region: 'us-east-1', + keyPrefix: '', + retries: 1, + timeoutMs: 30000, + }; + + it('accepts a settings s3 backend and assigns it to a category', () => { + setSetting('storage.backends', JSON.stringify([{ name: 'off-box', type: 's3', options: s3Options }])); + setSetting('storage.categories', JSON.stringify({ covers: 'off-box' })); + const { registry } = makeRegistry(); + const resolved = registry.resolve('covers'); + expect(resolved.backendName).toBe('off-box'); + expect(resolved.driver).toBeInstanceOf(S3Driver); + expect(resolved.keyPrefix).toBe('covers/'); + }); + + it('decrypts an enc:v1: secretAccessKey at build (decrypt-at-build)', () => { + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: encrypt_api_key('sk-secret') } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'off-box' })); + const { registry } = makeRegistry(); + expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver); + }); + + it('applies the shared-schema defaults for omitted optional fields', () => { + const { endpoint, bucket, accessKeyId, secretAccessKey } = s3Options; + setSetting( + 'storage.backends', + JSON.stringify([{ name: 'off-box', type: 's3', options: { endpoint, bucket, accessKeyId, secretAccessKey } }]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'off-box' })); + const { registry } = makeRegistry(); + expect(registry.resolve('backups').backendName).toBe('off-box'); + }); + + it('serves as a mirror replica (local primary + settings s3 replica on backups)', () => { + const nasRoot = makeTmpDir(); + setSetting( + 'storage.backends', + JSON.stringify([ + { name: 'off-box', type: 's3', options: s3Options }, + { name: 'nas-backups', type: 'local', options: { root: nasRoot } }, + { name: 'backup-mirror', type: 'mirror', options: { primary: 'nas-backups', replicas: ['off-box'] } }, + ]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'backup-mirror' })); + const { registry } = makeRegistry(); + expect(registry.resolve('backups').backendName).toBe('backup-mirror'); + expect(registry.resolve('backups').driver).toBeInstanceOf(MirrorDriver); + }); + + it('falls back with the pinned message when s3 options fail the shared schema', () => { const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); - setSetting('storage.backends', JSON.stringify([{ name: 'rogue', type: 's3', options: {} }])); - const { registry } = makeRegistry({}, { s3: S3_ON }); - expect(registry.resolve('backups').backendName).toBe('backups-local'); // defaults kept - + setSetting( + 'storage.backends', + JSON.stringify([{ name: 'off-box', type: 's3', options: { ...s3Options, endpoint: 'not a url' } }]), + ); + const { registry } = makeRegistry(); + expect(registry.resolve('backups').backendName).toBe('backups-local'); const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n'); - expect(logged).toContain("s3 backend 'rogue' cannot be declared in 'storage.backends'"); - expect(logged).toContain('env-declared only in v1'); - expect(logged).toContain('credentials never live in the database'); + expect(logged).toContain("s3 backend 'off-box' has invalid options — endpoint:"); + }); + + it('falls back with the pinned message when the ciphertext cannot be decrypted', () => { + const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + setSetting( + 'storage.backends', + JSON.stringify([{ name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: 'enc:v1:AAAA' } }]), + ); + setSetting('storage.categories', JSON.stringify({ backups: 'off-box' })); + const { registry } = makeRegistry(); + expect(registry.resolve('backups').backendName).toBe('backups-local'); + const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n'); + expect(logged).toContain("s3 backend 'off-box': could not decrypt 'secretAccessKey'"); + }); +}); + +// ── preview() — admin writes never take the silent fallback ────────────────── + +describe('preview()', () => { + it('throws the exact registry error for an invalid candidate and mutates nothing', () => { + const { registry } = makeRegistry(); + const before = registry.resolve('backups').driver; + expect(() => + registry.preview({ backends: [], categories: { backups: 'nope' } }), + ).toThrow("category 'backups' maps to unknown backend 'nope'"); + expect(registry.resolve('backups').driver).toBe(before); // same instance — state untouched + }); + + it('accepts a valid candidate without changing resolution', () => { + const nasRoot = makeTmpDir(); + const { registry } = makeRegistry(); + const before = registry.resolve('backups').driver; + registry.preview({ + backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }], + categories: { backups: 'nas-backups' }, + }); + expect(registry.resolve('backups').driver).toBe(before); + expect(registry.resolve('backups').backendName).toBe('backups-local'); }); });