feat(server): accept settings-declared s3 backends — shared-schema options, decrypt-at-build, registry preview()

2026-10-21 22:46:32 +00:00 · 2026-08-20 00:30:47 +02:00
parent 8bfe2d2e52
commit 5758471e4e
2 changed files with 158 additions and 14 deletions
@@ -1,7 +1,9 @@
 import fs from 'node:fs';
 import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';
+import { STORAGE_BACKEND_TYPES } from '@trek/shared';
 import { DatabaseService } from '../database/database.service';
 import { RuntimeEnvService } from '../app-config/runtime-env.service';
+import { decrypt_api_key } from '../common/crypto/apiKeyCrypto';
 import { LocalDriver } from './drivers/local.driver';
 import { MirrorDriver, type ReplicaFailure } from './drivers/mirror.driver';
 import { S3Driver } from './drivers/s3.driver';
@@ -108,6 +110,19 @@ export class StorageRegistryService implements OnModuleInit {
    this.load(false);
  }

+  /**
+   * Validate a candidate config by running the real build() — merge over
+   * built-ins → parse → validateConfig → driver construction (network-free) —
+   * and discard the result. Throws exactly what a failing load would log;
+   * never touches this.state (admin writes must not take the silent boot-time
+   * fallback). Shares one side effect with any successful save: local roots
+   * and prefix dirs are created — an uncreatable root is exactly the error to
+   * surface before persisting. cleanSpool stays boot-only (boot=false here).
+   */
+  preview(candidate: { backends: unknown; categories: unknown }): void {
+    void this.build(candidate, false);
+  }
+
  resolve(category: StorageCategory): ResolvedCategory {
    if (!this.state) throw new StorageBackendError('storage registry not initialized');
    const assignment = this.state.categories.get(category);
@@ -227,7 +242,10 @@ export class StorageRegistryService implements OnModuleInit {
    }
    for (const config of backends.values()) {
      if (config.type !== 's3') continue;
-      drivers.set(config.name, new S3Driver({ id: config.name, ...config.options }));
+      drivers.set(
+        config.name,
+        new S3Driver({ id: config.name, ...config.options, secretAccessKey: decryptedSecret(config) }),
+      );
    }
    for (const config of backends.values()) {
      if (config.type !== 'mirror') continue;
@@ -277,10 +295,17 @@ function parseBackendList(raw: unknown): BackendConfig[] {
      return { name: entry.name, type: 'mirror', options: { primary: options.primary, replicas: replicas as string[] } };
    }
    if (entry.type === 's3') {
-      throw new StorageBackendError(
-        `s3 backend '${entry.name}' cannot be declared in '${BACKENDS_KEY}' — s3 backends are ` +
-          'env-declared only in v1 (TREK_S3_*; credentials never live in the database)',
-      );
+      const parsed = STORAGE_BACKEND_TYPES.s3.optionsSchema.safeParse(options);
+      if (!parsed.success) {
+        const first = parsed.error.issues[0];
+        throw new StorageBackendError(
+          `s3 backend '${entry.name}' has invalid options` +
+            (first ? ` — ${first.path.join('.') || '(options)'}: ${first.message}` : ''),
+        );
+      }
+      // secretAccessKey stays as stored (usually enc:v1:) — decrypted only at
+      // driver construction, so config maps and snapshots never hold plaintext.
+      return { name: entry.name, type: 's3', options: parsed.data };
    }
    throw new StorageBackendError(`backend '${entry.name}' has unknown type '${String(entry.type)}'`);
  });
@@ -300,6 +325,21 @@ function parseCategoryMap(raw: unknown): Array<[StorageCategory, string]> {
  });
 }

+/**
+ * Secrets live encrypted inside the storage.backends JSON (admin-config spec);
+ * plaintext passthrough is tolerated as belt-and-braces (decrypt_api_key's
+ * legacy-plaintext rule), though seed and PUT always store encrypted.
+ */
+function decryptedSecret(config: S3BackendConfig): string {
+  const plain = decrypt_api_key(config.options.secretAccessKey);
+  if (plain === null) {
+    throw new StorageBackendError(
+      `s3 backend '${config.name}': could not decrypt 'secretAccessKey' — was ENCRYPTION_KEY changed or the row edited by hand?`,
+    );
+  }
+  return plain;
+}
+
 function validateConfig(backends: Map<string, BackendConfig>, categories: Map<StorageCategory, string>): void {
  for (const config of backends.values()) {
    if (config.type !== 'mirror') continue;
@@ -14,6 +14,7 @@ const { testDb, dbMock } = vi.hoisted(() => {
 });

 vi.mock('../../../../src/db/database', () => dbMock);
+vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-registry-test-key' }));

 import fs from 'node:fs';
 import os from 'node:os';
@@ -24,6 +25,7 @@ import { runMigrations } from '../../../../src/db/migrations';
 import { DatabaseService } from '../../../../src/nest/database/database.service';
 import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
 import type { deriveS3 } from '../../../../src/app-config/derive';
+import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto';
 import { StorageRegistryService } from '../../../../src/nest/storage/storage-registry.service';
 import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver';
 import { MirrorDriver, type ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver';
@@ -204,7 +206,7 @@ describe('StorageRegistryService settings', () => {
      JSON.stringify({ backups: 'm2' }),
    ],
    [
-      'settings-declared s3 backend (env-only in v1)',
+      'settings s3 backend with invalid options',
      JSON.stringify([{ name: 'x', type: 's3', options: {} }]),
      undefined,
    ],
@@ -338,16 +340,118 @@ describe('s3-main (env-declared)', () => {
    const { registry } = makeRegistry({}, { s3: S3_ON });
    expect(registry.resolve('backups').backendName).toBe('backup-mirror');
  });
+});

-  it('rejects a settings-declared s3 backend with the intentional message', () => {
+// ── settings-declared s3 backends (admin-config slice: acceptance) ───────────
+
+describe('settings-declared s3 backends', () => {
+  const s3Options = {
+    endpoint: 'http://127.0.0.1:9000',
+    bucket: 'trek',
+    accessKeyId: 'ak',
+    secretAccessKey: 'sk-plain',
+    region: 'us-east-1',
+    keyPrefix: '',
+    retries: 1,
+    timeoutMs: 30000,
+  };
+
+  it('accepts a settings s3 backend and assigns it to a category', () => {
+    setSetting('storage.backends', JSON.stringify([{ name: 'off-box', type: 's3', options: s3Options }]));
+    setSetting('storage.categories', JSON.stringify({ covers: 'off-box' }));
+    const { registry } = makeRegistry();
+    const resolved = registry.resolve('covers');
+    expect(resolved.backendName).toBe('off-box');
+    expect(resolved.driver).toBeInstanceOf(S3Driver);
+    expect(resolved.keyPrefix).toBe('covers/');
+  });
+
+  it('decrypts an enc:v1: secretAccessKey at build (decrypt-at-build)', () => {
+    setSetting(
+      'storage.backends',
+      JSON.stringify([
+        { name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: encrypt_api_key('sk-secret') } },
+      ]),
+    );
+    setSetting('storage.categories', JSON.stringify({ backups: 'off-box' }));
+    const { registry } = makeRegistry();
+    expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver);
+  });
+
+  it('applies the shared-schema defaults for omitted optional fields', () => {
+    const { endpoint, bucket, accessKeyId, secretAccessKey } = s3Options;
+    setSetting(
+      'storage.backends',
+      JSON.stringify([{ name: 'off-box', type: 's3', options: { endpoint, bucket, accessKeyId, secretAccessKey } }]),
+    );
+    setSetting('storage.categories', JSON.stringify({ backups: 'off-box' }));
+    const { registry } = makeRegistry();
+    expect(registry.resolve('backups').backendName).toBe('off-box');
+  });
+
+  it('serves as a mirror replica (local primary + settings s3 replica on backups)', () => {
+    const nasRoot = makeTmpDir();
+    setSetting(
+      'storage.backends',
+      JSON.stringify([
+        { name: 'off-box', type: 's3', options: s3Options },
+        { name: 'nas-backups', type: 'local', options: { root: nasRoot } },
+        { name: 'backup-mirror', type: 'mirror', options: { primary: 'nas-backups', replicas: ['off-box'] } },
+      ]),
+    );
+    setSetting('storage.categories', JSON.stringify({ backups: 'backup-mirror' }));
+    const { registry } = makeRegistry();
+    expect(registry.resolve('backups').backendName).toBe('backup-mirror');
+    expect(registry.resolve('backups').driver).toBeInstanceOf(MirrorDriver);
+  });
+
+  it('falls back with the pinned message when s3 options fail the shared schema', () => {
    const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
-    setSetting('storage.backends', JSON.stringify([{ name: 'rogue', type: 's3', options: {} }]));
-    const { registry } = makeRegistry({}, { s3: S3_ON });
-    expect(registry.resolve('backups').backendName).toBe('backups-local'); // defaults kept
-
+    setSetting(
+      'storage.backends',
+      JSON.stringify([{ name: 'off-box', type: 's3', options: { ...s3Options, endpoint: 'not a url' } }]),
+    );
+    const { registry } = makeRegistry();
+    expect(registry.resolve('backups').backendName).toBe('backups-local');
    const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n');
-    expect(logged).toContain("s3 backend 'rogue' cannot be declared in 'storage.backends'");
-    expect(logged).toContain('env-declared only in v1');
-    expect(logged).toContain('credentials never live in the database');
+    expect(logged).toContain("s3 backend 'off-box' has invalid options — endpoint:");
+  });
+
+  it('falls back with the pinned message when the ciphertext cannot be decrypted', () => {
+    const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
+    setSetting(
+      'storage.backends',
+      JSON.stringify([{ name: 'off-box', type: 's3', options: { ...s3Options, secretAccessKey: 'enc:v1:AAAA' } }]),
+    );
+    setSetting('storage.categories', JSON.stringify({ backups: 'off-box' }));
+    const { registry } = makeRegistry();
+    expect(registry.resolve('backups').backendName).toBe('backups-local');
+    const logged = errorSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    expect(logged).toContain("s3 backend 'off-box': could not decrypt 'secretAccessKey'");
+  });
+});
+
+// ── preview() — admin writes never take the silent fallback ──────────────────
+
+describe('preview()', () => {
+  it('throws the exact registry error for an invalid candidate and mutates nothing', () => {
+    const { registry } = makeRegistry();
+    const before = registry.resolve('backups').driver;
+    expect(() =>
+      registry.preview({ backends: [], categories: { backups: 'nope' } }),
+    ).toThrow("category 'backups' maps to unknown backend 'nope'");
+    expect(registry.resolve('backups').driver).toBe(before); // same instance — state untouched
+  });
+
+  it('accepts a valid candidate without changing resolution', () => {
+    const nasRoot = makeTmpDir();
+    const { registry } = makeRegistry();
+    const before = registry.resolve('backups').driver;
+    registry.preview({
+      backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }],
+      categories: { backups: 'nas-backups' },
+    });
+    expect(registry.resolve('backups').driver).toBe(before);
+    expect(registry.resolve('backups').backendName).toBe('backups-local');
  });
 });