mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-21 22:46:32 +00:00
feat(storage): storage key validation, driver contract suite, LocalDriver
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { Readable } from 'node:stream';
|
||||
import { assertValidKey, assertValidPrefix } from '../storage-keys';
|
||||
import {
|
||||
isLocalTempFile,
|
||||
StorageBackendError,
|
||||
StorageInvalidKeyError,
|
||||
StorageNotFoundError,
|
||||
type ByteRange,
|
||||
type LocalTempFile,
|
||||
type ObjectStat,
|
||||
type StorageDriver,
|
||||
} from '../storage.types';
|
||||
|
||||
const SPOOL_DIR_NAME = '.tmp';
|
||||
|
||||
function errnoCode(err: unknown): string | undefined {
|
||||
return (err as NodeJS.ErrnoException).code;
|
||||
}
|
||||
|
||||
/** ENOENT on the path or any parent segment being a file — both are a miss. */
|
||||
function isMissing(err: unknown): boolean {
|
||||
const code = errnoCode(err);
|
||||
return code === 'ENOENT' || code === 'ENOTDIR';
|
||||
}
|
||||
|
||||
/**
|
||||
* The only real driver type in v1: plain files under a root directory.
|
||||
*
|
||||
* `put` is atomic — stream sources spool into the hidden `<root>/.tmp` and
|
||||
* commit with a single same-volume `rename()`; caller-owned temp files rename
|
||||
* directly with an EXDEV copy-fallback for the cross-volume edge case. The
|
||||
* root is realpath'd at `init()` because in Docker both storage anchors are
|
||||
* symlinks (Dockerfile: `/app/server/uploads → /app/uploads`,
|
||||
* `/app/server/data → /app/data`) and a naive `startsWith` containment check
|
||||
* misbehaves on symlinked roots — the trap backup.impl.ts:423 and
|
||||
* nest/plugins/paths.ts already work around with `realpathSync`.
|
||||
*/
|
||||
export class LocalDriver implements StorageDriver {
|
||||
readonly id: string;
|
||||
private readonly configuredRoot: string;
|
||||
private realRoot: string | null = null;
|
||||
|
||||
constructor(opts: { id: string; root: string }) {
|
||||
this.id = opts.id;
|
||||
this.configuredRoot = opts.root;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure root + spool + category prefix dirs exist and resolve the real
|
||||
* root. Runs on every registry load (boot and reload()) so a newly
|
||||
* configured backend is usable immediately; `cleanSpool` is passed at boot
|
||||
* only — on a reload it could delete an in-flight upload's spool file.
|
||||
*/
|
||||
init(opts: { ensurePrefixes?: string[]; cleanSpool?: boolean } = {}): void {
|
||||
fs.mkdirSync(this.configuredRoot, { recursive: true });
|
||||
this.realRoot = fs.realpathSync(this.configuredRoot);
|
||||
fs.mkdirSync(this.spoolDir(), { recursive: true });
|
||||
for (const prefix of opts.ensurePrefixes ?? []) {
|
||||
assertValidPrefix(prefix);
|
||||
if (prefix) fs.mkdirSync(path.join(this.realRoot, prefix), { recursive: true });
|
||||
}
|
||||
if (opts.cleanSpool) {
|
||||
for (const entry of fs.readdirSync(this.spoolDir())) {
|
||||
fs.rmSync(path.join(this.spoolDir(), entry), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private root(): string {
|
||||
if (!this.realRoot) {
|
||||
throw new StorageBackendError(`LocalDriver '${this.id}' used before init()`);
|
||||
}
|
||||
return this.realRoot;
|
||||
}
|
||||
|
||||
private spoolDir(): string {
|
||||
return path.join(this.root(), SPOOL_DIR_NAME);
|
||||
}
|
||||
|
||||
getSpoolDir(): string {
|
||||
return this.spoolDir();
|
||||
}
|
||||
|
||||
/**
|
||||
* Central key validation plus defense-in-depth containment: even a key that
|
||||
* somehow passed validation must resolve inside the real root.
|
||||
*/
|
||||
private resolvePath(key: string): string {
|
||||
assertValidKey(key);
|
||||
const resolved = path.resolve(this.root(), key);
|
||||
if (!resolved.startsWith(this.root() + path.sep)) {
|
||||
throw new StorageInvalidKeyError(key);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
getLocalPath(key: string): string {
|
||||
return this.resolvePath(key);
|
||||
}
|
||||
|
||||
async put(key: string, source: Readable | LocalTempFile): Promise<void> {
|
||||
const dest = this.resolvePath(key);
|
||||
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
|
||||
|
||||
if (isLocalTempFile(source)) {
|
||||
try {
|
||||
await fs.promises.rename(source.tmpPath, dest);
|
||||
} catch (err) {
|
||||
if (errnoCode(err) !== 'EXDEV') {
|
||||
throw new StorageBackendError(`put failed for '${key}' on '${this.id}'`, err);
|
||||
}
|
||||
await fs.promises.copyFile(source.tmpPath, dest);
|
||||
await fs.promises.unlink(source.tmpPath);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const spool = path.join(this.spoolDir(), randomUUID());
|
||||
try {
|
||||
await pipeline(source, fs.createWriteStream(spool));
|
||||
await fs.promises.rename(spool, dest);
|
||||
} catch (err) {
|
||||
await fs.promises.rm(spool, { force: true });
|
||||
throw err instanceof Error && !errnoCode(err)
|
||||
? err // source-stream failure: surface the caller's own error untouched
|
||||
: new StorageBackendError(`put failed for '${key}' on '${this.id}'`, err);
|
||||
}
|
||||
}
|
||||
|
||||
async getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }> {
|
||||
const resolved = this.resolvePath(key);
|
||||
const stat = await this.stat(key);
|
||||
if (!stat) throw new StorageNotFoundError(key);
|
||||
const stream = fs.createReadStream(
|
||||
resolved,
|
||||
range ? { start: range.start, end: range.end } : undefined,
|
||||
);
|
||||
return { stream, stat };
|
||||
}
|
||||
|
||||
async stat(key: string): Promise<ObjectStat | null> {
|
||||
const resolved = this.resolvePath(key);
|
||||
try {
|
||||
const st = await fs.promises.stat(resolved);
|
||||
if (!st.isFile()) return null;
|
||||
return { key, size: st.size, mtimeMs: st.mtimeMs };
|
||||
} catch (err) {
|
||||
if (isMissing(err)) return null;
|
||||
throw new StorageBackendError(`stat failed for '${key}' on '${this.id}'`, err);
|
||||
}
|
||||
}
|
||||
|
||||
async delete(key: string): Promise<void> {
|
||||
const resolved = this.resolvePath(key);
|
||||
try {
|
||||
await fs.promises.unlink(resolved);
|
||||
} catch (err) {
|
||||
if (isMissing(err)) return; // idempotent
|
||||
throw new StorageBackendError(`delete failed for '${key}' on '${this.id}'`, err);
|
||||
}
|
||||
}
|
||||
|
||||
async *list(prefix: string): AsyncIterable<ObjectStat> {
|
||||
assertValidPrefix(prefix);
|
||||
const root = this.root();
|
||||
yield* this.walk(prefix ? path.resolve(root, prefix) : root, root);
|
||||
}
|
||||
|
||||
private async *walk(dir: string, root: string): AsyncIterable<ObjectStat> {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
if (isMissing(err)) return; // unpopulated prefix — empty, not an error
|
||||
throw new StorageBackendError(`list failed under '${dir}' on '${this.id}'`, err);
|
||||
}
|
||||
for (const entry of entries) {
|
||||
// Skips the .tmp spool (and any dotfile) defensively — key validation
|
||||
// already makes dot segments unreachable, but list walks the real disk.
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
const entryPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
yield* this.walk(entryPath, root);
|
||||
} else if (entry.isFile()) {
|
||||
const st = await fs.promises.stat(entryPath);
|
||||
yield {
|
||||
key: path.relative(root, entryPath).split(path.sep).join('/'),
|
||||
size: st.size,
|
||||
mtimeMs: st.mtimeMs,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { StorageInvalidKeyError } from './storage.types';
|
||||
|
||||
/**
|
||||
* Central key validation — the ONE place storage keys are checked, replacing
|
||||
* the scattered per-route guards (`path.basename` + `startsWith` shapes in
|
||||
* files.service.ts / platform.routes.ts / journey-public.controller.ts) as
|
||||
* call sites migrate. Drivers still confine resolved paths to their root as
|
||||
* defense in depth, but no driver or route re-implements these rules.
|
||||
*
|
||||
* Rejected: `..` segments, absolute paths, backslashes, empty segments,
|
||||
* leading-dot segments (matches `express.static`'s default
|
||||
* `dotfiles: 'ignore'` and keeps the `.tmp` spool unreachable), and control
|
||||
* characters. Spaces are valid — legacy `photos/<filename>` keys are
|
||||
* user-named flat files.
|
||||
*/
|
||||
|
||||
const MAX_KEY_LENGTH = 1024;
|
||||
|
||||
function hasControlChars(value: string): boolean {
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const code = value.charCodeAt(i);
|
||||
if (code < 0x20 || code === 0x7f) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function isValidKey(key: string): boolean {
|
||||
if (!key || key.length > MAX_KEY_LENGTH) return false;
|
||||
if (key.includes('\\') || hasControlChars(key)) return false;
|
||||
for (const segment of key.split('/')) {
|
||||
if (segment === '' || segment.startsWith('.')) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function assertValidKey(key: string): void {
|
||||
if (!isValidKey(key)) throw new StorageInvalidKeyError(key);
|
||||
}
|
||||
|
||||
/** Prefixes additionally allow `''` (list everything) and a trailing `/`. */
|
||||
export function isValidPrefix(prefix: string): boolean {
|
||||
if (prefix === '') return true;
|
||||
return isValidKey(prefix.endsWith('/') ? prefix.slice(0, -1) : prefix);
|
||||
}
|
||||
|
||||
export function assertValidPrefix(prefix: string): void {
|
||||
if (!isValidPrefix(prefix)) throw new StorageInvalidKeyError(prefix);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* The storage domain's one `__dirname` anchor (the uploads-root.ts idiom, and
|
||||
* eventually its replacement: the registry's defaults make this the single
|
||||
* source of truth the other 16 copies collapse onto in later slices).
|
||||
*
|
||||
* `__dirname` moves with the file — `server/{src,dist}/nest/storage` sits
|
||||
* three levels under `server/`, so three `..` hops resolve to `<server>/`
|
||||
* under both the src (vitest) and dist (runtime) layouts. The depth is stated
|
||||
* rather than counted by hand and pinned by storage-keys.test.ts, exactly
|
||||
* like uploads-root.test.ts pins UPLOADS_ROOT. In Docker both anchors are
|
||||
* symlinks (`/app/server/uploads → /app/uploads`, `/app/server/data →
|
||||
* /app/data`), which is why LocalDriver realpaths its root at init.
|
||||
*/
|
||||
export const DEFAULT_UPLOADS_ROOT = path.resolve(__dirname, '..', '..', '..', 'uploads');
|
||||
export const DATA_ROOT = path.resolve(__dirname, '..', '..', '..', 'data');
|
||||
export const DEFAULT_BACKUPS_ROOT = path.join(DATA_ROOT, 'backups');
|
||||
/** Driver-agnostic global scratch space (`data/tmp`) — see StorageService.tempDir(). */
|
||||
export const GLOBAL_TEMP_DIR = path.join(DATA_ROOT, 'tmp');
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
|
||||
/**
|
||||
* The storage abstraction's contract surface (spec:
|
||||
* docs/superpowers/specs/2026-07-20-storage-backend-abstraction-design.md).
|
||||
*
|
||||
* S3 constraints are baked into the driver contract, not the implementation:
|
||||
* keys are opaque POSIX-style strings, no rename/append/partial writes, no
|
||||
* directory semantics. Adding a backend later means one driver file that
|
||||
* passes the contract suite (tests/unit/nest/storage/storage-driver.contract.ts)
|
||||
* — zero changes to controllers or services.
|
||||
*/
|
||||
|
||||
export const STORAGE_CATEGORIES = [
|
||||
'files',
|
||||
'journey',
|
||||
'covers',
|
||||
'avatars',
|
||||
'places',
|
||||
'photos',
|
||||
'photos-google',
|
||||
'photos-trek',
|
||||
'backups',
|
||||
] as const;
|
||||
export type StorageCategory = (typeof STORAGE_CATEGORIES)[number];
|
||||
|
||||
export interface ObjectStat {
|
||||
key: string;
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
}
|
||||
|
||||
/** Inclusive byte range, `fs.createReadStream` semantics; `end` omitted = to EOF. */
|
||||
export interface ByteRange {
|
||||
start: number;
|
||||
end?: number;
|
||||
}
|
||||
|
||||
/** Ignored by local drivers; a future S3 driver needs it for the PUT. */
|
||||
export interface PutOptions {
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A temp file whose OWNERSHIP transfers to `put()`: the driver moves or
|
||||
* consumes it (rename into place locally; upload-then-delete remotely).
|
||||
* Callers must not touch the path afterwards.
|
||||
*/
|
||||
export interface LocalTempFile {
|
||||
tmpPath: string;
|
||||
}
|
||||
|
||||
export function isLocalTempFile(source: Readable | LocalTempFile): source is LocalTempFile {
|
||||
return typeof (source as LocalTempFile).tmpPath === 'string';
|
||||
}
|
||||
|
||||
export interface StorageDriver {
|
||||
/** Backend instance name, e.g. 'uploads-local'. */
|
||||
readonly id: string;
|
||||
/** Atomic: spool + single rename locally; single/multipart PUT remotely. */
|
||||
put(key: string, source: Readable | LocalTempFile, opts?: PutOptions): Promise<void>;
|
||||
getStream(key: string, range?: ByteRange): Promise<{ stream: Readable; stat: ObjectStat }>;
|
||||
stat(key: string): Promise<ObjectStat | null>;
|
||||
/** Idempotent; resolving on a missing key is success. */
|
||||
delete(key: string): Promise<void>;
|
||||
list(prefix: string): AsyncIterable<ObjectStat>;
|
||||
/** Fast-path for local drivers; `null`/absent is exactly the remote-driver branch every caller must handle. */
|
||||
getLocalPath?(key: string): string | null;
|
||||
/** Same-filesystem spool dir (LocalDriver: `<root>/.tmp`), or null for drivers with no local filesystem. */
|
||||
getSpoolDir?(): string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Typed error set so route handlers map storage failures to their existing
|
||||
* bespoke envelopes without string-matching driver internals.
|
||||
*/
|
||||
export class StorageBackendError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly cause?: unknown,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'StorageBackendError';
|
||||
}
|
||||
}
|
||||
|
||||
export class StorageNotFoundError extends Error {
|
||||
constructor(readonly key: string) {
|
||||
super(`storage object not found: ${key}`);
|
||||
this.name = 'StorageNotFoundError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Serving slices map this to a miss (404/next()), never a 500. */
|
||||
export class StorageInvalidKeyError extends Error {
|
||||
constructor(readonly key: string) {
|
||||
super(`invalid storage key: ${key}`);
|
||||
this.name = 'StorageInvalidKeyError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver';
|
||||
import { StorageInvalidKeyError } from '../../../../src/nest/storage/storage.types';
|
||||
import { describeStorageDriver, type DriverHarness } from './storage-driver.contract';
|
||||
|
||||
function makeTmpDir(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-'));
|
||||
}
|
||||
|
||||
async function makeHarness(): Promise<DriverHarness> {
|
||||
const root = makeTmpDir();
|
||||
const tmpFiles = makeTmpDir();
|
||||
const driver = new LocalDriver({ id: 'test-local', root });
|
||||
driver.init({ cleanSpool: true });
|
||||
let counter = 0;
|
||||
return {
|
||||
driver,
|
||||
makeTempFile: async (contents) => {
|
||||
const p = path.join(tmpFiles, `src-${counter++}.tmp`);
|
||||
fs.writeFileSync(p, contents);
|
||||
return p;
|
||||
},
|
||||
cleanup: async () => {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
fs.rmSync(tmpFiles, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describeStorageDriver('LocalDriver', makeHarness);
|
||||
|
||||
describe('LocalDriver specifics', () => {
|
||||
const dirs: string[] = [];
|
||||
const tmp = (): string => {
|
||||
const d = makeTmpDir();
|
||||
dirs.push(d);
|
||||
return d;
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('realpaths a symlinked root and confines keys to the real directory', async () => {
|
||||
// Docker symlinks BOTH storage anchors (/app/server/uploads → /app/uploads,
|
||||
// /app/server/data → /app/data), so this is the deployed shape, not an edge.
|
||||
const realdir = tmp();
|
||||
const link = path.join(tmp(), 'uploads-link');
|
||||
fs.symlinkSync(realdir, link);
|
||||
|
||||
const driver = new LocalDriver({ id: 'sym', root: link });
|
||||
driver.init();
|
||||
await driver.put('files/x.bin', Readable.from('sym'));
|
||||
|
||||
const local = driver.getLocalPath('files/x.bin');
|
||||
expect(local.startsWith(fs.realpathSync(realdir) + path.sep)).toBe(true);
|
||||
expect(fs.readFileSync(local, 'utf8')).toBe('sym');
|
||||
expect(() => driver.getLocalPath('../escape')).toThrow(StorageInvalidKeyError);
|
||||
});
|
||||
|
||||
it('cleans spool leftovers only at boot, never on reload-style init', () => {
|
||||
const driver = new LocalDriver({ id: 'spool', root: tmp() });
|
||||
driver.init();
|
||||
const stray = path.join(driver.getSpoolDir(), 'leftover.part');
|
||||
fs.writeFileSync(stray, 'in-flight upload');
|
||||
|
||||
driver.init(); // reload(): must NOT delete an in-flight upload's spool file
|
||||
expect(fs.existsSync(stray)).toBe(true);
|
||||
|
||||
driver.init({ cleanSpool: true }); // boot: crash leftovers are reclaimed
|
||||
expect(fs.existsSync(stray)).toBe(false);
|
||||
});
|
||||
|
||||
it('creates its root, spool, and category prefix dirs at init', () => {
|
||||
const root = path.join(tmp(), 'nested', 'uploads');
|
||||
const driver = new LocalDriver({ id: 'pfx', root });
|
||||
driver.init({ ensurePrefixes: ['files', 'photos/trek'] });
|
||||
|
||||
expect(fs.statSync(path.join(root, '.tmp')).isDirectory()).toBe(true);
|
||||
expect(fs.statSync(path.join(root, 'files')).isDirectory()).toBe(true);
|
||||
expect(fs.statSync(path.join(root, 'photos/trek')).isDirectory()).toBe(true);
|
||||
});
|
||||
|
||||
it('falls back to copy+unlink when a temp-file rename crosses volumes (EXDEV)', async () => {
|
||||
const root = tmp();
|
||||
const driver = new LocalDriver({ id: 'exdev', root });
|
||||
driver.init();
|
||||
const src = path.join(tmp(), 'other-volume.bin');
|
||||
fs.writeFileSync(src, 'exdev payload');
|
||||
|
||||
vi.spyOn(fs.promises, 'rename').mockRejectedValueOnce(
|
||||
Object.assign(new Error('EXDEV: cross-device link not permitted'), { code: 'EXDEV' }),
|
||||
);
|
||||
|
||||
await driver.put('files/moved.bin', { tmpPath: src });
|
||||
expect(fs.readFileSync(path.join(root, 'files/moved.bin'), 'utf8')).toBe('exdev payload');
|
||||
expect(fs.existsSync(src)).toBe(false);
|
||||
});
|
||||
|
||||
it('commits stream puts via a same-volume spool rename', async () => {
|
||||
const root = tmp();
|
||||
const driver = new LocalDriver({ id: 'samevol', root });
|
||||
driver.init();
|
||||
|
||||
const renameSpy = vi.spyOn(fs.promises, 'rename');
|
||||
await driver.put('files/spooled.bin', Readable.from('data'));
|
||||
|
||||
const [from, to] = renameSpy.mock.calls.at(-1)!;
|
||||
expect(String(from).startsWith(path.join(fs.realpathSync(root), '.tmp') + path.sep)).toBe(true);
|
||||
expect(String(to)).toBe(path.join(fs.realpathSync(root), 'files/spooled.bin'));
|
||||
});
|
||||
|
||||
it('lists a missing prefix as empty', async () => {
|
||||
const driver = new LocalDriver({ id: 'empty', root: tmp() });
|
||||
driver.init();
|
||||
const seen: unknown[] = [];
|
||||
for await (const stat of driver.list('nope/')) seen.push(stat);
|
||||
expect(seen).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { Readable } from 'node:stream';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
StorageInvalidKeyError,
|
||||
StorageNotFoundError,
|
||||
type ObjectStat,
|
||||
type StorageDriver,
|
||||
} from '../../../../src/nest/storage/storage.types';
|
||||
|
||||
/**
|
||||
* The parameterized driver contract suite — the groundwork's real product
|
||||
* (spec, Testing). Every driver must pass it: LocalDriver and MirrorDriver in
|
||||
* v1; a future S3 driver imports this same suite (against MinIO/localstack)
|
||||
* and must pass before touching TREK code.
|
||||
*
|
||||
* Not a `.test.ts` on purpose: vitest only collects files matching the
|
||||
* `*.test.ts` include, so this suite runs exclusively through a driver's own
|
||||
* spec file.
|
||||
*/
|
||||
|
||||
export interface DriverHarness {
|
||||
driver: StorageDriver;
|
||||
/** Create a temp file whose ownership put() may take (LocalTempFile source). */
|
||||
makeTempFile(contents: string | Buffer): Promise<string>;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
const INVALID_KEYS = ['../x', '/abs', 'a\\b', 'a//b', '.tmp/x', 'a/.h', 'a\u0000b'];
|
||||
|
||||
export function describeStorageDriver(name: string, makeHarness: () => Promise<DriverHarness>): void {
|
||||
describe(`storage driver contract: ${name}`, () => {
|
||||
let h: DriverHarness;
|
||||
|
||||
beforeEach(async () => {
|
||||
h = await makeHarness();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await h.cleanup();
|
||||
});
|
||||
|
||||
async function readAll(key: string, range?: { start: number; end?: number }): Promise<Buffer> {
|
||||
const { stream } = await h.driver.getStream(key, range);
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) chunks.push(Buffer.from(chunk as Buffer));
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function listAll(prefix: string): Promise<ObjectStat[]> {
|
||||
const out: ObjectStat[] = [];
|
||||
for await (const stat of h.driver.list(prefix)) out.push(stat);
|
||||
return out;
|
||||
}
|
||||
|
||||
it('roundtrips a stream source and stats its size', async () => {
|
||||
await h.driver.put('a/stream.bin', Readable.from(Buffer.from('hello storage')));
|
||||
expect((await readAll('a/stream.bin')).toString()).toBe('hello storage');
|
||||
const stat = await h.driver.stat('a/stream.bin');
|
||||
expect(stat).toMatchObject({ key: 'a/stream.bin', size: 13 });
|
||||
expect(stat!.mtimeMs).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('roundtrips a LocalTempFile source and consumes the temp file', async () => {
|
||||
const tmp = await h.makeTempFile('from tmp file');
|
||||
await h.driver.put('a/tmpfile.bin', { tmpPath: tmp });
|
||||
expect((await readAll('a/tmpfile.bin')).toString()).toBe('from tmp file');
|
||||
expect(fs.existsSync(tmp)).toBe(false); // ownership transferred
|
||||
});
|
||||
|
||||
it('overwrites an existing key', async () => {
|
||||
await h.driver.put('a/over.bin', Readable.from('first'));
|
||||
await h.driver.put('a/over.bin', Readable.from('second!'));
|
||||
expect((await readAll('a/over.bin')).toString()).toBe('second!');
|
||||
expect((await h.driver.stat('a/over.bin'))!.size).toBe(7);
|
||||
});
|
||||
|
||||
it('serves closed and open-ended byte ranges', async () => {
|
||||
await h.driver.put('a/range.bin', Readable.from('0123456789'));
|
||||
expect((await readAll('a/range.bin', { start: 2, end: 5 })).toString()).toBe('2345');
|
||||
expect((await readAll('a/range.bin', { start: 7 })).toString()).toBe('789');
|
||||
});
|
||||
|
||||
it('rejects getStream on a missing key with StorageNotFoundError', async () => {
|
||||
await expect(h.driver.getStream('a/nope.bin')).rejects.toBeInstanceOf(StorageNotFoundError);
|
||||
});
|
||||
|
||||
it('stats a missing key as null', async () => {
|
||||
expect(await h.driver.stat('a/nope.bin')).toBeNull();
|
||||
});
|
||||
|
||||
it('deletes idempotently', async () => {
|
||||
await h.driver.put('a/gone.bin', Readable.from('x'));
|
||||
await h.driver.delete('a/gone.bin');
|
||||
expect(await h.driver.stat('a/gone.bin')).toBeNull();
|
||||
await expect(h.driver.delete('a/gone.bin')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('lists by prefix with full keys and sizes', async () => {
|
||||
await h.driver.put('a/one.bin', Readable.from('11'));
|
||||
await h.driver.put('a/sub/two.bin', Readable.from('222'));
|
||||
await h.driver.put('b/three.bin', Readable.from('3'));
|
||||
|
||||
const underA = await listAll('a/');
|
||||
expect(underA.map((s) => s.key).sort()).toEqual(['a/one.bin', 'a/sub/two.bin']);
|
||||
expect(underA.find((s) => s.key === 'a/sub/two.bin')!.size).toBe(3);
|
||||
|
||||
const everything = await listAll('');
|
||||
expect(everything.map((s) => s.key).sort()).toEqual(['a/one.bin', 'a/sub/two.bin', 'b/three.bin']);
|
||||
});
|
||||
|
||||
it.each(INVALID_KEYS)('rejects invalid key %j on every method', async (key) => {
|
||||
await expect(h.driver.put(key, Readable.from('x'))).rejects.toBeInstanceOf(StorageInvalidKeyError);
|
||||
await expect(h.driver.getStream(key)).rejects.toBeInstanceOf(StorageInvalidKeyError);
|
||||
await expect(h.driver.stat(key)).rejects.toBeInstanceOf(StorageInvalidKeyError);
|
||||
await expect(h.driver.delete(key)).rejects.toBeInstanceOf(StorageInvalidKeyError);
|
||||
});
|
||||
|
||||
it('leaves no object and no visible partial when the source stream errors', async () => {
|
||||
const boom = new Readable({
|
||||
read() {
|
||||
this.push('partial bytes');
|
||||
this.destroy(new Error('source exploded'));
|
||||
},
|
||||
});
|
||||
await expect(h.driver.put('a/broken.bin', boom)).rejects.toThrow();
|
||||
expect(await h.driver.stat('a/broken.bin')).toBeNull();
|
||||
expect((await listAll('')).map((s) => s.key)).toEqual([]);
|
||||
});
|
||||
|
||||
it('honors the getSpoolDir contract', async () => {
|
||||
const spool = h.driver.getSpoolDir?.() ?? null;
|
||||
if (spool === null) return; // drivers with no local filesystem
|
||||
expect(fs.existsSync(spool)).toBe(true);
|
||||
// Spool contents are invisible to the abstraction by construction.
|
||||
fs.writeFileSync(path.join(spool, 'leftover.part'), 'junk');
|
||||
expect((await listAll('')).map((s) => s.key)).toEqual([]);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
isValidKey,
|
||||
assertValidKey,
|
||||
isValidPrefix,
|
||||
assertValidPrefix,
|
||||
} from '../../../../src/nest/storage/storage-keys';
|
||||
import { StorageInvalidKeyError } from '../../../../src/nest/storage/storage.types';
|
||||
import {
|
||||
DEFAULT_UPLOADS_ROOT,
|
||||
DEFAULT_BACKUPS_ROOT,
|
||||
GLOBAL_TEMP_DIR,
|
||||
} from '../../../../src/nest/storage/storage-paths';
|
||||
|
||||
// Central key validation (spec: storage-keys.ts owns the rules; drivers and
|
||||
// routes never re-implement them). Every rejected shape here replaces one of
|
||||
// the scattered per-route guards the refactor retires.
|
||||
describe('storage keys', () => {
|
||||
const validKeys = [
|
||||
'files/a.pdf',
|
||||
'journey/thumbs/ab12.jpg',
|
||||
'backup-2026.zip', // backups category uses bare keys (spec rev 3.1)
|
||||
'photos/trek/deadbeef.bin',
|
||||
// legacy photos/<filename> keys are user-named flat files — spaces stay valid
|
||||
'photos/my holiday pic.jpg',
|
||||
];
|
||||
|
||||
const invalidKeys = [
|
||||
'', // empty
|
||||
'/abs', // absolute
|
||||
'a\\b', // backslash
|
||||
'a//b', // empty segment
|
||||
'a/', // trailing slash = empty segment
|
||||
'../x', // traversal
|
||||
'a/../b', // embedded traversal
|
||||
'.', // dot segment
|
||||
'..', // dot-dot segment
|
||||
'.tmp/x', // leading-dot segment (spool must be unreachable)
|
||||
'a/.hidden', // dotfile segment
|
||||
'a\u0000b', // control character
|
||||
`a/${'x'.repeat(1024)}`, // over length cap
|
||||
];
|
||||
|
||||
it.each(validKeys)('accepts %j', (key) => {
|
||||
expect(isValidKey(key)).toBe(true);
|
||||
expect(() => assertValidKey(key)).not.toThrow();
|
||||
});
|
||||
|
||||
it.each(invalidKeys)('rejects %j', (key) => {
|
||||
expect(isValidKey(key)).toBe(false);
|
||||
expect(() => assertValidKey(key)).toThrow(StorageInvalidKeyError);
|
||||
});
|
||||
|
||||
it('caps keys at 1024 chars (boundary)', () => {
|
||||
const exactly1024 = `a/${'x'.repeat(1022)}`;
|
||||
expect(exactly1024.length).toBe(1024);
|
||||
expect(isValidKey(exactly1024)).toBe(true);
|
||||
expect(isValidKey(`${exactly1024}x`)).toBe(false);
|
||||
});
|
||||
|
||||
describe('prefixes', () => {
|
||||
it.each(['', 'thumbs/', 'thumbs', 'photos/google/'])('accepts %j', (prefix) => {
|
||||
expect(isValidPrefix(prefix)).toBe(true);
|
||||
expect(() => assertValidPrefix(prefix)).not.toThrow();
|
||||
});
|
||||
|
||||
it.each(['../', '/abs', '.tmp/', 'a//b/', 'a\\b'])('rejects %j', (prefix) => {
|
||||
expect(isValidPrefix(prefix)).toBe(false);
|
||||
expect(() => assertValidPrefix(prefix)).toThrow(StorageInvalidKeyError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Depth contract for the storage domain's one __dirname anchor, same guard as
|
||||
// tests/unit/nest/uploads-root.test.ts: the resolved defaults must equal the
|
||||
// paths the server has always used, under both src (vitest) and dist layouts.
|
||||
describe('storage default paths', () => {
|
||||
it('anchors the default uploads root at <server>/uploads', () => {
|
||||
expect(DEFAULT_UPLOADS_ROOT).toBe(path.resolve(process.cwd(), 'uploads'));
|
||||
});
|
||||
|
||||
it('anchors the default backups root at <server>/data/backups', () => {
|
||||
expect(DEFAULT_BACKUPS_ROOT).toBe(path.resolve(process.cwd(), 'data/backups'));
|
||||
});
|
||||
|
||||
it('anchors the global temp dir at <server>/data/tmp', () => {
|
||||
expect(GLOBAL_TEMP_DIR).toBe(path.resolve(process.cwd(), 'data/tmp'));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user