feat(server): api/admin/storage GET/PUT — masked secrets, preview-gated saves, wired reload, audits

This commit is contained in:
jubnl
2026-08-20 08:06:02 +02:00
parent 5dc78dd5cf
commit 7439bcd33c
9 changed files with 485 additions and 15 deletions
+1
View File
@@ -18,6 +18,7 @@ const ACTION_LABELS: Record<string, string> = {
'admin.user_delete': 'deleted user',
'admin.plugin_retrust': "re-trusted a plugin's author signing key",
'admin.invite_create': 'created invite',
'admin.storage_update': 'updated storage configuration',
'immich.private_ip_configured': 'configured Immich with private IP',
};
@@ -79,6 +79,8 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [
'PluginsController.link',
'PluginsController.reload',
'PluginsController.upload',
'StorageAdminController.get',
'StorageAdminController.update',
];
/**
@@ -0,0 +1,51 @@
import { Body, Controller, Get, HttpException, Put, Req, UseGuards } from '@nestjs/common';
import type { Request } from 'express';
import { redactStorageSecrets } from './storage-secrets';
import type { User } from '../../types';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { AdminGuard } from '../auth/admin.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { AuditService } from '../audit/audit.service';
import { getClientIp } from '../audit/client-ip';
import { ManagedForbidden } from '../common/managed';
import { StorageAdminService } from './storage-admin.service';
import { StorageConfigDto } from './storage-admin.dto';
/**
* /api/admin/storage — the admin surface over the storage registry (spec:
* docs/superpowers/specs/2026-08-19-storage-admin-config-design.md). Backends,
* category assignment, and mirror composition are managed here; secrets are
* stored encrypted and only ever rendered as the mask. Pipeline refusals are
* operator-grade registry messages, surfaced verbatim in the 400 envelope.
*/
@Controller('api/admin/storage')
@UseGuards(JwtAuthGuard, AdminGuard)
@ManagedForbidden('storage backends and their credentials are hoster-level configuration')
export class StorageAdminController {
constructor(
private readonly service: StorageAdminService,
private readonly audit: AuditService,
) {}
@Get()
get() {
return this.service.state();
}
@Put()
update(@CurrentUser() user: User, @Body() body: StorageConfigDto, @Req() req: Request) {
try {
this.service.applyConfig(body);
} catch (err) {
throw new HttpException({ error: err instanceof Error ? err.message : String(err) }, 400);
}
this.audit.writeAudit({
userId: user.id,
action: 'admin.storage_update',
ip: getClientIp(req),
details: redactStorageSecrets(body) as unknown as Record<string, unknown>,
});
// The defaults-tab contract: never echo the request — answer the fresh effective world.
return this.service.state();
}
}
@@ -0,0 +1,10 @@
import { createZodDto } from 'nestjs-zod';
import { storageConfigSchema, storageTestRequestSchema } from '@trek/shared';
/**
* createZodDto wrappers over the @trek/shared storage contracts — the global
* ZodValidationPipe (APP_PIPE) validates @Body() params typed with these, and
* validate-body-contracts.ts refuses boot for any unwrapped mutation body.
*/
export class StorageConfigDto extends createZodDto(storageConfigSchema) {}
export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {}
@@ -0,0 +1,89 @@
import fs from 'node:fs';
import { Injectable } from '@nestjs/common';
import type { StorageAdminState, StorageConfig } from '@trek/shared';
import { DatabaseService } from '../database/database.service';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import {
BACKENDS_KEY,
CATEGORIES_KEY,
StorageRegistryService,
} from './storage-registry.service';
import { StorageService } from './storage.service';
import { SEED_CONFIG_PATH } from './storage-paths';
import {
encryptionGateError,
encryptStorageSecrets,
listPlaintextSecrets,
maskBackendOptions,
unmaskStorageConfig,
} from './storage-secrets';
import { StorageBackendError } from './storage.types';
/**
* Owner of the api/admin/storage read/write pipelines (spec:
* docs/superpowers/specs/2026-08-19-storage-admin-config-design.md, Server).
* Reads render from the registry's live snapshot; writes run
* unmask → encryption gate → preview → encrypt → persist → reload, so an
* admin save either fully applies or changes nothing — never the boot-time
* silent fallback.
*/
@Injectable()
export class StorageAdminService {
constructor(
private readonly db: DatabaseService,
private readonly registry: StorageRegistryService,
private readonly storage: StorageService,
private readonly env: RuntimeEnvService,
) {}
/** The effective world — secrets masked, categories cross-referenced per backend. */
state(): StorageAdminState {
const snapshot = this.registry.snapshot();
const assignments = Object.entries(snapshot.categories) as Array<
[keyof typeof snapshot.categories, { backend: string; source: 'default' | 'settings' }]
>;
return {
backends: snapshot.backends.map((backend) => ({
name: backend.name,
type: backend.type,
source: backend.source,
options: maskBackendOptions(backend.type, backend.options),
categories: assignments.filter(([, a]) => a.backend === backend.name).map(([category]) => category),
})),
categories: snapshot.categories,
health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) },
encryptionReady: this.env.env().security.encryptionKeySet,
seedFilePresent: fs.existsSync(SEED_CONFIG_PATH),
};
}
/** Full-document replace of the two settings rows. Throws StorageBackendError on any refusal. */
applyConfig(config: StorageConfig): void {
const unmasked = unmaskStorageConfig(config, this.storedBackendsRow());
const plaintext = listPlaintextSecrets(unmasked);
if (plaintext.length > 0 && !this.env.env().security.encryptionKeySet) {
throw new StorageBackendError(encryptionGateError(plaintext[0]!));
}
this.registry.preview({ backends: unmasked.backends, categories: unmasked.categories });
const encrypted = encryptStorageSecrets(unmasked);
this.db.transaction(() => {
const upsert = this.db.prepare(
'INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
);
upsert.run(BACKENDS_KEY, JSON.stringify(encrypted.backends));
upsert.run(CATEGORIES_KEY, JSON.stringify(encrypted.categories));
});
this.registry.reload();
}
/** The raw stored backends row — the unmask source (tolerates absent/garbage rows). */
private storedBackendsRow(): unknown {
const row = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', BACKENDS_KEY);
if (!row?.value) return [];
try {
return JSON.parse(row.value) as unknown;
} catch {
return [];
}
}
}
@@ -8,7 +8,12 @@ import { LocalDriver } from './drivers/local.driver';
import { MirrorDriver, type ReplicaFailure } from './drivers/mirror.driver';
import { S3Driver } from './drivers/s3.driver';
import { DEFAULT_BACKUPS_ROOT, DEFAULT_UPLOADS_ROOT, GLOBAL_TEMP_DIR, SEED_CONFIG_PATH } from './storage-paths';
import { assertNoMaskSentinels, encryptStorageSecrets, encryptionGateError, listPlaintextSecrets } from './storage-secrets';
import {
assertNoMaskSentinels,
encryptStorageSecrets,
encryptionGateError,
listPlaintextSecrets,
} from './storage-secrets';
import {
STORAGE_CATEGORIES,
StorageBackendError,
@@ -67,8 +72,8 @@ interface RegistryState {
snapshot: RegistrySnapshot;
}
const BACKENDS_KEY = 'storage.backends';
const CATEGORIES_KEY = 'storage.categories';
export const BACKENDS_KEY = 'storage.backends';
export const CATEGORIES_KEY = 'storage.categories';
const REPLICA_FAILURE_RING_SIZE = 50;
/**
+17 -12
View File
@@ -1,23 +1,28 @@
import { Module } from '@nestjs/common';
import { AppConfigModule } from '../app-config/app-config.module';
import { AuditModule } from '../audit/audit.module';
import { StorageRegistryService } from './storage-registry.service';
import { StorageService } from './storage.service';
import { StorageAdminService } from './storage-admin.service';
import { StorageAdminController } from './storage-admin.controller';
/**
* The storage core (spec:
* docs/superpowers/specs/2026-07-20-storage-backend-abstraction-design.md).
* Consuming modules import this explicitly (repo convention — SchedulingModule
* is deliberately not consumed via @Global either); AppConfigModule is listed
* even though it is @Global so an e2e TestingModule built around one domain
* still resolves RuntimeEnvService (the memories.module precedent).
* DatabaseModule IS consumed via @Global, like every other domain.
*
* No controller: v1 has no HTTP surface — the registry's reload() is wired to
* an admin route only when the admin UI lands (Deferred).
* Storage container: registry (config), facade (byte-paths), admin surface.
* AuditModule feeds the write audits. AuthModule is deliberately NOT imported
* here — AuthModule itself imports StorageModule (avatar uploads), so the
* reverse import is a real module cycle (Nest resolves it as `imports[1] is
* undefined`, not a clean forwardRef case). JwtAuthGuard/AdminGuard need no
* provider from AuthModule to begin with — they carry no constructor
* dependencies, so `@UseGuards(JwtAuthGuard, AdminGuard)` instantiates them
* directly, the same as BackupModule/BackupController (which also sits behind
* StorageModule) already does. StorageRegistryService stays UNEXPORTED — the
* admin controller reaches it as a same-module provider, and nothing outside
* may cache drivers or trigger reloads.
*/
@Module({
imports: [AppConfigModule],
providers: [StorageRegistryService, StorageService],
imports: [AppConfigModule, AuditModule],
controllers: [StorageAdminController],
providers: [StorageRegistryService, StorageService, StorageAdminService],
exports: [StorageService],
})
export class StorageModule {}
@@ -0,0 +1,90 @@
import { describe, it, expect, vi } from 'vitest';
import { HttpException } from '@nestjs/common';
import type { Request } from 'express';
import type { User } from '../../../../src/types';
import { StorageAdminController } from '../../../../src/nest/storage/storage-admin.controller';
import type { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service';
import type { AuditService } from '../../../../src/nest/audit/audit.service';
import type { StorageConfigDto } from '../../../../src/nest/storage/storage-admin.dto';
import { StorageModule } from '../../../../src/nest/storage/storage.module';
import { StorageBackendError } from '../../../../src/nest/storage/storage.types';
import { expectRegisteredController } from '../../../helpers/module-providers';
const user = { id: 1 } as User;
const req = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } as unknown as Request;
const FRESH_STATE = { backends: [], categories: {}, health: { replicaFailures: [] }, encryptionReady: true, seedFilePresent: false };
function makeController(over: Partial<Record<keyof StorageAdminService, unknown>> = {}) {
const writeAudit = vi.fn();
const service = {
state: vi.fn(() => FRESH_STATE),
applyConfig: vi.fn(),
...over,
} as unknown as StorageAdminService;
const controller = new StorageAdminController(service, { writeAudit } as unknown as AuditService);
return { controller, service, writeAudit };
}
const CONFIG = {
backends: [
{
name: 'off-box',
type: 's3' as const,
options: {
endpoint: 'http://127.0.0.1:9000',
bucket: 'trek',
accessKeyId: 'ak',
secretAccessKey: 'sk-plain',
region: 'us-east-1',
keyPrefix: '',
retries: 1,
timeoutMs: 30000,
},
},
],
categories: { backups: 'off-box' },
} as StorageConfigDto;
describe('StorageAdminController', () => {
it('STORCTL-001 GET returns the service state untouched', () => {
const { controller, service } = makeController();
expect(controller.get()).toBe(FRESH_STATE);
expect(service.state).toHaveBeenCalledTimes(1);
});
it('STORCTL-002 PUT applies, audits with secrets redacted, and answers the fresh state', () => {
const { controller, service, writeAudit } = makeController();
const result = controller.update(user, CONFIG, req);
expect(service.applyConfig).toHaveBeenCalledWith(CONFIG);
expect(result).toBe(FRESH_STATE); // never echoes the request
expect(writeAudit).toHaveBeenCalledWith(
expect.objectContaining({ userId: 1, action: 'admin.storage_update' }),
);
const details = (writeAudit.mock.calls[0]![0] as { details: { backends: Array<{ options: Record<string, unknown> }> } }).details;
expect(details.backends[0]!.options.secretAccessKey).toBe('***');
expect(details.backends[0]!.options.accessKeyId).toBe('ak'); // names/shape survive redaction
});
it('STORCTL-003 PUT maps pipeline refusals to a 400 with the message verbatim, no audit', () => {
const { controller, writeAudit } = makeController({
applyConfig: vi.fn(() => {
throw new StorageBackendError("category 'backups' maps to unknown backend 'nope'");
}),
});
try {
controller.update(user, CONFIG, req);
expect.unreachable('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(HttpException);
expect((err as HttpException).getStatus()).toBe(400);
expect((err as HttpException).getResponse()).toEqual({
error: "category 'backups' maps to unknown backend 'nope'",
});
}
expect(writeAudit).not.toHaveBeenCalled();
});
it('STORCTL-004 the controller is registered in StorageModule', () => {
expectRegisteredController(StorageModule, StorageAdminController);
});
});
@@ -0,0 +1,217 @@
import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from 'vitest';
const { testDb, dbMock } = vi.hoisted(() => {
const Database = require('better-sqlite3');
const db = new Database(':memory:');
db.exec('PRAGMA journal_mode = WAL');
db.exec('PRAGMA foreign_keys = ON');
return { testDb: db, dbMock: { db, closeDb: () => {}, reinitialize: () => {} } };
});
vi.mock('../../../../src/db/database', () => dbMock);
vi.mock('../../../../src/config', () => ({ ENCRYPTION_KEY: 'storage-admin-test-key' }));
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { MASKED_SETTING_VALUE, type StorageConfig } from '@trek/shared';
import { createTables } from '../../../../src/db/schema';
import { runMigrations } from '../../../../src/db/migrations';
import { DatabaseService } from '../../../../src/nest/database/database.service';
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto';
import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service';
import { StorageRegistryService, BACKENDS_KEY, CATEGORIES_KEY } from '../../../../src/nest/storage/storage-registry.service';
import { StorageService } from '../../../../src/nest/storage/storage.service';
const db = new DatabaseService(testDb);
beforeAll(() => {
createTables(testDb);
runMigrations(testDb);
});
const tmpDirs: string[] = [];
function makeTmpDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-admin-'));
tmpDirs.push(dir);
return dir;
}
function setSetting(key: string, value: string): void {
testDb.prepare('INSERT OR REPLACE INTO app_settings (key, value) VALUES (?, ?)').run(key, value);
}
function readRow(key: string): string | undefined {
const row = testDb.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as { value: string } | undefined;
return row?.value;
}
const S3_OPTIONS = {
endpoint: 'http://127.0.0.1:9000',
bucket: 'trek',
accessKeyId: 'ak',
secretAccessKey: 'sk-plain',
region: 'us-east-1',
keyPrefix: '',
retries: 1,
timeoutMs: 30000,
};
/** Real registry + real service over the in-memory DB; env stub toggles the key. */
function makeService(opts: { encryptionKeySet?: boolean; uploadsRoot?: string } = {}) {
const uploadsRoot = opts.uploadsRoot ?? makeTmpDir();
setSetting(BACKENDS_KEY, JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }]));
const env = {
env: () => ({ paths: {}, security: { encryptionKeySet: opts.encryptionKeySet ?? true } }),
} as unknown as RuntimeEnvService;
const registry = new StorageRegistryService(db, env);
registry.onModuleInit();
const service = new StorageAdminService(db, registry, new StorageService(registry), env);
return { service, registry, uploadsRoot };
}
/** The settings-owned document the service persists (uploads override + extras). */
function configWith(uploadsRoot: string, extra: Partial<StorageConfig> = {}): StorageConfig {
return {
backends: [
{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } },
...(extra.backends ?? []),
],
categories: extra.categories ?? {},
};
}
beforeEach(() => {
testDb.prepare("DELETE FROM app_settings WHERE key LIKE 'storage.%'").run();
});
afterEach(() => {
vi.restoreAllMocks();
while (tmpDirs.length) fs.rmSync(tmpDirs.pop()!, { recursive: true, force: true });
});
describe('StorageAdminService.state', () => {
it('STORADM-001 renders the effective world: sources, categories-per-backend, flags', () => {
const { service, uploadsRoot } = makeService();
const state = service.state();
const uploads = state.backends.find((b) => b.name === 'uploads-local')!;
expect(uploads).toMatchObject({ type: 'local', source: 'settings', options: { root: uploadsRoot } });
expect(uploads.categories).toContain('files');
expect(uploads.categories).not.toContain('backups');
expect(state.backends.find((b) => b.name === 'backups-local')).toMatchObject({ source: 'built-in' });
expect(state.categories.backups).toEqual({ backend: 'backups-local', source: 'default' });
expect(state.encryptionReady).toBe(true);
expect(state.seedFilePresent).toBe(false);
expect(state.health).toEqual({ replicaFailures: [] });
});
it('STORADM-002 masks exactly the secret fields (accessKeyId stays visible)', () => {
const { service, uploadsRoot } = makeService();
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }],
categories: { backups: 'off-box' },
}));
const offBox = service.state().backends.find((b) => b.name === 'off-box')!;
expect(offBox.options.secretAccessKey).toBe(MASKED_SETTING_VALUE);
expect(offBox.options.accessKeyId).toBe('ak');
});
it('STORADM-003 surfaces replica failures through StorageService.health()', () => {
const { service, registry } = makeService();
registry.recordReplicaFailure({ backend: 'nas', key: 'backup-1.zip', op: 'put', error: 'disk full', at: 123 });
expect(service.state().health.replicaFailures).toEqual([
{ backend: 'nas', key: 'backup-1.zip', op: 'put', error: 'disk full', at: 123 },
]);
});
});
describe('StorageAdminService.applyConfig', () => {
it('STORADM-010 happy path: persists both rows, reloads, new config is live', () => {
const { service, registry, uploadsRoot } = makeService();
const nasRoot = makeTmpDir();
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'nas-backups', type: 'local', options: { root: nasRoot } }],
categories: { backups: 'nas-backups' },
}));
expect(registry.resolve('backups').backendName).toBe('nas-backups'); // reload() ran
expect(JSON.parse(readRow(CATEGORIES_KEY)!)).toEqual({ backups: 'nas-backups' });
});
it('STORADM-011 encrypts plaintext secrets at rest', () => {
const { service, uploadsRoot } = makeService();
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }],
categories: { backups: 'off-box' },
}));
const stored = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record<string, unknown> }>;
const secret = String(stored.find((b) => b.name === 'off-box')!.options.secretAccessKey);
expect(secret.startsWith('enc:v1:')).toBe(true);
});
it('STORADM-012 mask echo preserves the stored ciphertext byte-for-byte', () => {
const { service, uploadsRoot } = makeService();
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }],
categories: { backups: 'off-box' },
}));
const before = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record<string, unknown> }>;
const cipherBefore = before.find((b) => b.name === 'off-box')!.options.secretAccessKey;
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: MASKED_SETTING_VALUE } }],
categories: { backups: 'off-box' },
}));
const after = JSON.parse(readRow(BACKENDS_KEY)!) as Array<{ name: string; options: Record<string, unknown> }>;
expect(after.find((b) => b.name === 'off-box')!.options.secretAccessKey).toBe(cipherBefore);
});
it('STORADM-013 a mask on a renamed/new backend throws the re-enter error, persists nothing', () => {
const { service, uploadsRoot } = makeService();
const before = readRow(BACKENDS_KEY);
expect(() =>
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'brand-new', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: MASKED_SETTING_VALUE } }],
categories: {},
})),
).toThrow("re-enter the secret 'secretAccessKey' for 'brand-new'");
expect(readRow(BACKENDS_KEY)).toBe(before);
});
it('STORADM-014 plaintext secret without ENCRYPTION_KEY → error naming the variable, nothing persisted', () => {
const { service, uploadsRoot } = makeService({ encryptionKeySet: false });
const before = readRow(BACKENDS_KEY);
expect(() =>
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: S3_OPTIONS }],
categories: {},
})),
).toThrow(/ENCRYPTION_KEY/);
expect(readRow(BACKENDS_KEY)).toBe(before);
});
it('STORADM-015 an encrypted (mask-echoed or enc:v1:) secret saves fine WITHOUT ENCRYPTION_KEY set', () => {
// The gate is about NEW plaintext material only — resaving stored ciphertext must not lock admins out.
const { service, uploadsRoot } = makeService({ encryptionKeySet: false });
service.applyConfig(configWith(uploadsRoot, {
backends: [{ name: 'off-box', type: 's3', options: { ...S3_OPTIONS, secretAccessKey: encrypt_api_key('sk') } }],
categories: { backups: 'off-box' },
}));
expect(readRow(BACKENDS_KEY)).toBeDefined();
});
it('STORADM-016 preview() refusals surface verbatim and persist nothing', () => {
const { service, uploadsRoot, registry } = makeService();
const before = readRow(CATEGORIES_KEY);
expect(() =>
service.applyConfig(configWith(uploadsRoot, { categories: { backups: 'nope' } })),
).toThrow("category 'backups' maps to unknown backend 'nope'");
expect(readRow(CATEGORIES_KEY)).toBe(before);
expect(registry.resolve('backups').backendName).toBe('backups-local'); // live state untouched
});
it('STORADM-017 persists both rows in ONE transaction and reloads once', () => {
const { service, registry, uploadsRoot } = makeService();
const txSpy = vi.spyOn(db, 'transaction');
const reloadSpy = vi.spyOn(registry, 'reload');
service.applyConfig(configWith(uploadsRoot));
expect(txSpy).toHaveBeenCalledTimes(1);
expect(reloadSpy).toHaveBeenCalledTimes(1);
});
});