Files
TREK/server/tests/unit/nest/storage/local.driver.test.ts
T
Maurice 790fccb92c fix(storage): judge unreferenced files without a brake anyone can trip
The 20 % brake skipped a category and wiped its marks every night once a
fifth of it looked unreferenced. Any user could push a small install over
that line, a backlog from #2924 started braked, and a second install on the
same bucket below the line was not caught at all. It is gone:

- Each pass leaves a trek-owner-<install id> marker at the root of every
  category it judges. A category holding another install's marker is
  skipped as foreign_install, one where the marker cannot be written as
  owner_unknown. Backups leave the markers out.
- The install id lives in app_settings outside the storage.* prefix. The
  switch is stored as that id, so a database from another install reads as
  off, and every reopen (a restore) writes this install's id back, turns
  deletion off and drops the marks, confirmations and reports.
- More than a fifth unreferenced still marks and reports, but deletes only
  after an admin confirmed the count shown (POST .../reclaim/confirm). A
  larger count later waits again.

A skip no longer touches the marks, so a failed listing or lookup does not
restart the week. Check now runs in the background and answers 202; the
admin state says running and keeps the last deleting report apart, so a
report-only check never resets what was freed. A pass that deletes writes a
system audit entry, a pass that stops or fails keeps its partial counts and
marks the rest interrupted, a backend error stops deleting in its category,
and deleting stops for the rest of the pass once the switch is off or a
backup is reading the uploads. The usage scan waits for a pass still
running.

Sources read each category once per pass and answer the pre-delete
re-check with a query narrowed to the keys' plain runs. Bare names keep
their raw spelling (a '#', '?' or trailing blank is part of an upload's
name), and local roots are resolved through their symlinks before the
overlap check. A pass lists only a category's root.
2026-11-01 16:18:08 +01:00

153 lines
5.9 KiB
TypeScript

import { LocalDriver } from '../../../../src/nest/storage/drivers/local.driver';
import { StorageInvalidKeyError } from '../../../../src/nest/storage/storage.types';
import { describeStorageDriver, type DriverHarness } from './storage-driver.contract';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Readable } from 'node:stream';
import { describe, it, expect, vi, afterEach } from 'vitest';
function makeTmpDir(): string {
return fs.mkdtempSync(path.join(os.tmpdir(), 'trek-storage-'));
}
async function makeHarness(): Promise<DriverHarness> {
const root = makeTmpDir();
const tmpFiles = makeTmpDir();
const driver = new LocalDriver({ id: 'test-local', root });
driver.init({ cleanSpool: true });
let counter = 0;
return {
driver,
makeTempFile: async (contents) => {
const p = path.join(tmpFiles, `src-${counter++}.tmp`);
fs.writeFileSync(p, contents);
return p;
},
cleanup: async () => {
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(tmpFiles, { recursive: true, force: true });
},
};
}
describeStorageDriver('LocalDriver', makeHarness);
describe('LocalDriver specifics', () => {
const dirs: string[] = [];
const tmp = (): string => {
const d = makeTmpDir();
dirs.push(d);
return d;
};
afterEach(() => {
vi.restoreAllMocks();
while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true });
});
it('realpaths a symlinked root and confines keys to the real directory', async () => {
// Docker symlinks BOTH storage anchors (/app/server/uploads → /app/uploads,
// /app/server/data → /app/data), so this is the deployed shape, not an edge.
const realdir = tmp();
const link = path.join(tmp(), 'uploads-link');
fs.symlinkSync(realdir, link);
const driver = new LocalDriver({ id: 'sym', root: link });
driver.init();
await driver.put('files/x.bin', Readable.from('sym'));
const local = driver.getLocalPath('files/x.bin');
expect(local.startsWith(fs.realpathSync(realdir) + path.sep)).toBe(true);
expect(fs.readFileSync(local, 'utf8')).toBe('sym');
expect(() => driver.getLocalPath('../escape')).toThrow(StorageInvalidKeyError);
});
it('cleans aged spool leftovers only at boot, never on reload-style init', () => {
const driver = new LocalDriver({ id: 'spool', root: tmp() });
driver.init();
const stray = path.join(driver.getSpoolDir(), 'leftover.part');
fs.writeFileSync(stray, 'in-flight upload');
// Crash leftover: old enough to clear the reap age gate.
const old = new Date(Date.now() - 2 * 60 * 60 * 1000);
fs.utimesSync(stray, old, old);
driver.init(); // reload(): must NOT delete an in-flight upload's spool file
expect(fs.existsSync(stray)).toBe(true);
driver.init({ cleanSpool: true }); // boot: crash leftovers are reclaimed
expect(fs.existsSync(stray)).toBe(false);
});
it('the boot spool sweep spares fresh entries (another process may be mid-upload)', () => {
const driver = new LocalDriver({ id: 'spool-fresh', root: tmp() });
driver.init();
const inflight = path.join(driver.getSpoolDir(), 'inflight.part');
fs.writeFileSync(inflight, 'spooling right now');
driver.init({ cleanSpool: true });
expect(fs.existsSync(inflight)).toBe(true);
});
it('creates its root, spool, and category prefix dirs at init', () => {
const root = path.join(tmp(), 'nested', 'uploads');
const driver = new LocalDriver({ id: 'pfx', root });
driver.init({ ensurePrefixes: ['files', 'photos/trek'] });
expect(fs.statSync(path.join(root, '.tmp')).isDirectory()).toBe(true);
expect(fs.statSync(path.join(root, 'files')).isDirectory()).toBe(true);
expect(fs.statSync(path.join(root, 'photos/trek')).isDirectory()).toBe(true);
});
it('falls back to copy+unlink when a temp-file rename crosses volumes (EXDEV)', async () => {
const root = tmp();
const driver = new LocalDriver({ id: 'exdev', root });
driver.init();
const src = path.join(tmp(), 'other-volume.bin');
fs.writeFileSync(src, 'exdev payload');
vi.spyOn(fs.promises, 'rename').mockRejectedValueOnce(
Object.assign(new Error('EXDEV: cross-device link not permitted'), { code: 'EXDEV' }),
);
await driver.put('files/moved.bin', { tmpPath: src });
expect(fs.readFileSync(path.join(root, 'files/moved.bin'), 'utf8')).toBe('exdev payload');
expect(fs.existsSync(src)).toBe(false);
});
it('commits stream puts via a same-volume spool rename', async () => {
const root = tmp();
const driver = new LocalDriver({ id: 'samevol', root });
driver.init();
const renameSpy = vi.spyOn(fs.promises, 'rename');
await driver.put('files/spooled.bin', Readable.from('data'));
const [from, to] = renameSpy.mock.calls.at(-1)!;
expect(String(from).startsWith(path.join(fs.realpathSync(root), '.tmp') + path.sep)).toBe(true);
expect(String(to)).toBe(path.join(fs.realpathSync(root), 'files/spooled.bin'));
});
it('lists a missing prefix as empty', async () => {
const driver = new LocalDriver({ id: 'empty', root: tmp() });
driver.init();
const seen: unknown[] = [];
for await (const stat of driver.list('nope/')) seen.push(stat);
expect(seen).toEqual([]);
});
it('a shallow list yields the objects at the prefix only, none of a folder below it', async () => {
const driver = new LocalDriver({ id: 'shallow', root: tmp() });
driver.init();
await driver.put('journey/top.jpg', Readable.from('a'));
await driver.put('journey/thumbs/deep.jpg', Readable.from('b'));
const shallow: string[] = [];
for await (const stat of driver.list('journey/', { shallow: true })) shallow.push(stat.key);
expect(shallow).toEqual(['journey/top.jpg']);
const deep: string[] = [];
for await (const stat of driver.list('journey/')) deep.push(stat.key);
expect(deep.sort()).toEqual(['journey/thumbs/deep.jpg', 'journey/top.jpg']);
});
});