mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-19 05:26:47 +00:00
feat(server): seed-once storage-config.json boot import — fail closed on any invalid seed
This commit is contained in:
@@ -18,3 +18,5 @@ export const DATA_ROOT = path.resolve(__dirname, '..', '..', '..', 'data');
|
||||
export const DEFAULT_BACKUPS_ROOT = path.join(DATA_ROOT, 'backups');
|
||||
/** Driver-agnostic global scratch space (`data/tmp`) — see StorageService.tempDir(). */
|
||||
export const GLOBAL_TEMP_DIR = path.join(DATA_ROOT, 'tmp');
|
||||
/** Seed-once boot provisioning file — imported only when no storage.* row exists. */
|
||||
export const SEED_CONFIG_PATH = path.join(DATA_ROOT, 'storage-config.json');
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import fs from 'node:fs';
|
||||
import { Injectable, Logger, type OnModuleInit } from '@nestjs/common';
|
||||
import { STORAGE_BACKEND_TYPES } from '@trek/shared';
|
||||
import { STORAGE_BACKEND_TYPES, storageConfigSchema } from '@trek/shared';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { decrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
import { LocalDriver } from './drivers/local.driver';
|
||||
import { MirrorDriver, type ReplicaFailure } from './drivers/mirror.driver';
|
||||
import { S3Driver } from './drivers/s3.driver';
|
||||
import { DEFAULT_BACKUPS_ROOT, DEFAULT_UPLOADS_ROOT, GLOBAL_TEMP_DIR } from './storage-paths';
|
||||
import { DEFAULT_BACKUPS_ROOT, DEFAULT_UPLOADS_ROOT, GLOBAL_TEMP_DIR, SEED_CONFIG_PATH } from './storage-paths';
|
||||
import { assertNoMaskSentinels, encryptStorageSecrets, encryptionGateError, listPlaintextSecrets } from './storage-secrets';
|
||||
import {
|
||||
STORAGE_CATEGORIES,
|
||||
StorageBackendError,
|
||||
@@ -102,6 +103,7 @@ export class StorageRegistryService implements OnModuleInit {
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
this.seedFromFileOnce();
|
||||
this.load(true);
|
||||
}
|
||||
|
||||
@@ -150,6 +152,70 @@ export class StorageRegistryService implements OnModuleInit {
|
||||
return this.failures;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed-once boot provisioning (spec: 2026-08-19-storage-admin-config-design.md).
|
||||
* Imported only when NO storage.* row exists; runs the same validation the
|
||||
* PUT pipeline runs (encryption gate, then preview); secrets are encrypted
|
||||
* on import; the file is ignored (loudly) afterward. Every failure aborts
|
||||
* boot with the exact error — an actively-provisioning operator must see
|
||||
* it, so this deliberately runs OUTSIDE load()'s last-good safety net.
|
||||
* Recovery: stop the server, DELETE FROM app_settings WHERE key LIKE
|
||||
* 'storage.%', restart (documented in the README with slice 3).
|
||||
*/
|
||||
private seedFromFileOnce(): void {
|
||||
const rowCount = this.db.get<{ n: number }>(
|
||||
'SELECT COUNT(*) AS n FROM app_settings WHERE key IN (?, ?)',
|
||||
BACKENDS_KEY,
|
||||
CATEGORIES_KEY,
|
||||
);
|
||||
const filePresent = fs.existsSync(SEED_CONFIG_PATH);
|
||||
if ((rowCount?.n ?? 0) > 0) {
|
||||
if (filePresent) {
|
||||
this.logger.log(`storage config rows exist — ignoring ${SEED_CONFIG_PATH}; manage storage in the admin UI`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!filePresent) return;
|
||||
|
||||
const fail = (detail: string): never => {
|
||||
throw new Error(`invalid storage seed file ${SEED_CONFIG_PATH}: ${detail}`);
|
||||
};
|
||||
let json: unknown;
|
||||
try {
|
||||
json = JSON.parse(fs.readFileSync(SEED_CONFIG_PATH, 'utf8'));
|
||||
} catch (err) {
|
||||
return fail(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
const parsed = storageConfigSchema.safeParse(json);
|
||||
if (!parsed.success) {
|
||||
return fail(
|
||||
parsed.error.issues.map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`).join('; '),
|
||||
);
|
||||
}
|
||||
const config = parsed.data;
|
||||
try {
|
||||
assertNoMaskSentinels(config);
|
||||
const plaintext = listPlaintextSecrets(config);
|
||||
if (plaintext.length > 0 && !this.env.env().security.encryptionKeySet) {
|
||||
throw new StorageBackendError(encryptionGateError(plaintext[0]!));
|
||||
}
|
||||
this.preview({ backends: config.backends, categories: config.categories });
|
||||
} catch (err) {
|
||||
return fail(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
const encrypted = encryptStorageSecrets(config);
|
||||
this.db.transaction(() => {
|
||||
const upsert = this.db.prepare(
|
||||
'INSERT INTO app_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value',
|
||||
);
|
||||
upsert.run(BACKENDS_KEY, JSON.stringify(encrypted.backends));
|
||||
upsert.run(CATEGORIES_KEY, JSON.stringify(encrypted.categories));
|
||||
});
|
||||
this.logger.log(
|
||||
`storage config seeded from ${SEED_CONFIG_PATH} — the file is now ignored; manage storage in the admin UI`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalid settings never take the server down: on any failure the previous
|
||||
* state is kept (at boot, when there is no previous state, the built-in
|
||||
|
||||
@@ -29,7 +29,12 @@ import { StorageRegistryService } from '../../../../src/nest/storage/storage-reg
|
||||
import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver';
|
||||
import { MirrorDriver, type ReplicaFailure } from '../../../../src/nest/storage/drivers/mirror.driver';
|
||||
import { S3Driver } from '../../../../src/nest/storage/drivers/s3.driver';
|
||||
import { GLOBAL_TEMP_DIR, DEFAULT_BACKUPS_ROOT, DEFAULT_UPLOADS_ROOT } from '../../../../src/nest/storage/storage-paths';
|
||||
import {
|
||||
GLOBAL_TEMP_DIR,
|
||||
DEFAULT_BACKUPS_ROOT,
|
||||
DEFAULT_UPLOADS_ROOT,
|
||||
SEED_CONFIG_PATH,
|
||||
} from '../../../../src/nest/storage/storage-paths';
|
||||
import { STORAGE_CATEGORIES } from '../../../../src/nest/storage/storage.types';
|
||||
|
||||
const db = new DatabaseService(testDb);
|
||||
@@ -52,13 +57,13 @@ interface EnvPaths {
|
||||
placePhotoDir?: string;
|
||||
}
|
||||
|
||||
function makeEnvStub(initial: EnvPaths): { env: RuntimeEnvService; setPaths: (p: EnvPaths) => void } {
|
||||
let paths = initial;
|
||||
function makeEnvStub(
|
||||
initial: EnvPaths,
|
||||
security: { encryptionKeySet: boolean } = { encryptionKeySet: true },
|
||||
): { env: RuntimeEnvService } {
|
||||
const paths = initial;
|
||||
return {
|
||||
env: { env: () => ({ paths }) } as unknown as RuntimeEnvService,
|
||||
setPaths: (p) => {
|
||||
paths = p;
|
||||
},
|
||||
env: { env: () => ({ paths, security }) } as unknown as RuntimeEnvService,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -77,6 +82,7 @@ interface RegistryOpts {
|
||||
/** Extra storage.backends entries, appended after the uploads-local override. */
|
||||
backends?: unknown[];
|
||||
categories?: Record<string, string>;
|
||||
encryptionKeySet?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -89,10 +95,13 @@ function makeRegistry(opts: RegistryOpts = {}) {
|
||||
const uploadsRoot = opts.uploadsRoot ?? makeTmpDir();
|
||||
setSetting('storage.backends', JSON.stringify([uploadsOverride(uploadsRoot), ...(opts.backends ?? [])]));
|
||||
if (opts.categories) setSetting('storage.categories', JSON.stringify(opts.categories));
|
||||
const stub = makeEnvStub({ placePhotoDir: opts.placePhotoDir });
|
||||
const stub = makeEnvStub(
|
||||
{ placePhotoDir: opts.placePhotoDir },
|
||||
{ encryptionKeySet: opts.encryptionKeySet ?? true },
|
||||
);
|
||||
const registry = new StorageRegistryService(db, stub.env);
|
||||
if (opts.boot !== false) registry.onModuleInit();
|
||||
return { registry, uploadsRoot, setPaths: stub.setPaths, setUploadsRoot: (root: string) => rewriteUploadsOverride(root) };
|
||||
return { registry, uploadsRoot, setUploadsRoot: (root: string) => rewriteUploadsOverride(root) };
|
||||
}
|
||||
|
||||
/** For reload tests: swap the override row's root in place. */
|
||||
@@ -425,3 +434,159 @@ describe('preview()', () => {
|
||||
expect(registry.resolve('backups').backendName).toBe('backups-local');
|
||||
});
|
||||
});
|
||||
|
||||
// ── seed-once boot import ─────────────────────────────────────────────────────
|
||||
|
||||
describe('seed-once storage-config.json import', () => {
|
||||
const nasRoot = () => makeTmpDir();
|
||||
|
||||
function writeSeed(content: string): void {
|
||||
fs.mkdirSync(path.dirname(SEED_CONFIG_PATH), { recursive: true });
|
||||
fs.writeFileSync(SEED_CONFIG_PATH, content);
|
||||
}
|
||||
|
||||
/** A registry with NO storage.* rows (makeRegistry seeds an override row, so build raw). */
|
||||
function makeUnseededRegistry(opts: { encryptionKeySet?: boolean } = {}) {
|
||||
const stub = makeEnvStub({}, { encryptionKeySet: opts.encryptionKeySet ?? true });
|
||||
return new StorageRegistryService(db, stub.env);
|
||||
}
|
||||
|
||||
function readRow(key: string): string | undefined {
|
||||
const row = testDb.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as
|
||||
| { value: string }
|
||||
| undefined;
|
||||
return row?.value;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(SEED_CONFIG_PATH, { force: true });
|
||||
});
|
||||
|
||||
it('SEED-001 imports a valid file when no rows exist: encrypted rows, live drivers, pinned log', () => {
|
||||
const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
|
||||
const root = nasRoot();
|
||||
writeSeed(
|
||||
JSON.stringify({
|
||||
backends: [
|
||||
{ name: 'nas-backups', type: 'local', options: { root } },
|
||||
{
|
||||
name: 'off-box',
|
||||
type: 's3',
|
||||
options: {
|
||||
endpoint: 'http://127.0.0.1:9000',
|
||||
bucket: 'trek',
|
||||
accessKeyId: 'ak',
|
||||
secretAccessKey: 'sk-seed',
|
||||
},
|
||||
},
|
||||
],
|
||||
categories: { backups: 'off-box' },
|
||||
}),
|
||||
);
|
||||
const registry = makeUnseededRegistry();
|
||||
registry.onModuleInit();
|
||||
|
||||
expect(registry.resolve('backups').driver).toBeInstanceOf(S3Driver);
|
||||
const storedBackends = JSON.parse(readRow('storage.backends')!) as Array<{
|
||||
name: string;
|
||||
options: Record<string, unknown>;
|
||||
}>;
|
||||
const offBox = storedBackends.find((b) => b.name === 'off-box')!;
|
||||
expect(String(offBox.options.secretAccessKey).startsWith('enc:v1:')).toBe(true); // never plaintext at rest
|
||||
expect(readRow('storage.categories')).toBe(JSON.stringify({ backups: 'off-box' }));
|
||||
const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n');
|
||||
expect(logged).toContain('storage config seeded from');
|
||||
expect(logged).toContain('the file is now ignored; manage storage in the admin UI');
|
||||
});
|
||||
|
||||
it('SEED-002 rows exist → file not read, pinned ignore log, rows unchanged', () => {
|
||||
const logSpy = vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
|
||||
writeSeed('{ this would explode if parsed');
|
||||
const { registry } = makeRegistry(); // helper writes the uploads override row
|
||||
expect(registry.resolve('files').backendName).toBe('uploads-local');
|
||||
const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n');
|
||||
expect(logged).toContain('storage config rows exist — ignoring');
|
||||
});
|
||||
|
||||
it('SEED-003 no rows, no file → boots on built-in defaults (no seed side effects)', () => {
|
||||
const registry = makeUnseededRegistry();
|
||||
registry.onModuleInit();
|
||||
expect(registry.resolve('backups').backendName).toBe('backups-local');
|
||||
expect(readRow('storage.backends')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('SEED-004 unparseable JSON aborts boot with the exact error', () => {
|
||||
writeSeed('not json at all');
|
||||
const registry = makeUnseededRegistry();
|
||||
expect(() => registry.onModuleInit()).toThrow(/invalid storage seed file .*storage-config\.json: /);
|
||||
});
|
||||
|
||||
it('SEED-005 schema violation aborts boot — including the reserved readOnly key', () => {
|
||||
writeSeed(JSON.stringify({ backends: [], categories: {}, readOnly: true }));
|
||||
expect(() => makeUnseededRegistry().onModuleInit()).toThrow(/invalid storage seed file/);
|
||||
});
|
||||
|
||||
it('SEED-006 semantic violation aborts boot with the registry error verbatim', () => {
|
||||
writeSeed(JSON.stringify({ backends: [], categories: { backups: 'nope' } }));
|
||||
expect(() => makeUnseededRegistry().onModuleInit()).toThrow(
|
||||
/category 'backups' maps to unknown backend 'nope'/,
|
||||
);
|
||||
});
|
||||
|
||||
it('SEED-007 plaintext secrets without ENCRYPTION_KEY abort boot naming the variable', () => {
|
||||
writeSeed(
|
||||
JSON.stringify({
|
||||
backends: [
|
||||
{
|
||||
name: 'off-box',
|
||||
type: 's3',
|
||||
options: {
|
||||
endpoint: 'http://127.0.0.1:9000',
|
||||
bucket: 'trek',
|
||||
accessKeyId: 'ak',
|
||||
secretAccessKey: 'sk-seed',
|
||||
},
|
||||
},
|
||||
],
|
||||
categories: {},
|
||||
}),
|
||||
);
|
||||
const registry = makeUnseededRegistry({ encryptionKeySet: false });
|
||||
expect(() => registry.onModuleInit()).toThrow(/ENCRYPTION_KEY/);
|
||||
expect(readRow('storage.backends')).toBeUndefined(); // nothing persisted on abort
|
||||
});
|
||||
|
||||
it('SEED-008 a mask sentinel in the seed file aborts boot', () => {
|
||||
writeSeed(
|
||||
JSON.stringify({
|
||||
backends: [
|
||||
{
|
||||
name: 'off-box',
|
||||
type: 's3',
|
||||
options: {
|
||||
endpoint: 'http://127.0.0.1:9000',
|
||||
bucket: 'trek',
|
||||
accessKeyId: 'ak',
|
||||
secretAccessKey: '••••••••',
|
||||
},
|
||||
},
|
||||
],
|
||||
categories: {},
|
||||
}),
|
||||
);
|
||||
expect(() => makeUnseededRegistry().onModuleInit()).toThrow(/mask/);
|
||||
});
|
||||
|
||||
it('SEED-009 a second boot after a successful seed ignores the file (seed-once)', () => {
|
||||
const root = nasRoot();
|
||||
writeSeed(JSON.stringify({ backends: [{ name: 'nas', type: 'local', options: { root } }], categories: { backups: 'nas' } }));
|
||||
makeUnseededRegistry().onModuleInit();
|
||||
const firstRow = readRow('storage.backends');
|
||||
|
||||
writeSeed(JSON.stringify({ backends: [], categories: { backups: 'other' } })); // would fail preview if read
|
||||
const second = makeUnseededRegistry();
|
||||
second.onModuleInit(); // must not throw — rows exist, file ignored
|
||||
expect(readRow('storage.backends')).toBe(firstRow);
|
||||
expect(second.resolve('backups').backendName).toBe('nas');
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user