Files
TREK/server/tests/e2e/storage-reclaim.e2e.test.ts
T
Maurice 790fccb92c fix(storage): judge unreferenced files without a brake anyone can trip
The 20 % brake skipped a category and wiped its marks every night once a
fifth of it looked unreferenced. Any user could push a small install over
that line, a backlog from #2924 started braked, and a second install on the
same bucket below the line was not caught at all. It is gone:

- Each pass leaves a trek-owner-<install id> marker at the root of every
  category it judges. A category holding another install's marker is
  skipped as foreign_install, one where the marker cannot be written as
  owner_unknown. Backups leave the markers out.
- The install id lives in app_settings outside the storage.* prefix. The
  switch is stored as that id, so a database from another install reads as
  off, and every reopen (a restore) writes this install's id back, turns
  deletion off and drops the marks, confirmations and reports.
- More than a fifth unreferenced still marks and reports, but deletes only
  after an admin confirmed the count shown (POST .../reclaim/confirm). A
  larger count later waits again.

A skip no longer touches the marks, so a failed listing or lookup does not
restart the week. Check now runs in the background and answers 202; the
admin state says running and keeps the last deleting report apart, so a
report-only check never resets what was freed. A pass that deletes writes a
system audit entry, a pass that stops or fails keeps its partial counts and
marks the rest interrupted, a backend error stops deleting in its category,
and deleting stops for the rest of the pass once the switch is off or a
backup is reading the uploads. The usage scan waits for a pass still
running.

Sources read each category once per pass and answer the pre-delete
re-check with a query narrowed to the keys' plain runs. Bare names keep
their raw spelling (a '#', '?' or trailing blank is part of an upload's
name), and local roots are resolved through their symlinks before the
overlap check. A pass lists only a category's root.
2026-11-01 16:18:08 +01:00

183 lines
8.1 KiB
TypeScript

/**
* Storage reclaim e2e: /api/admin/storage/reclaim on the whole app (buildApp
* against the worker's snapshot test db), so the real JwtAuthGuard,
* AdminGuard and ManagedGuard, the Zod pipe on both bodies and the response
* contracts the handlers declare all stand in front of it. Every owner
* domain registers its reference source here, so a pass looks at every
* reclaimable category; the checks below run in the background and only
* report. RCLE-001 to RCLE-007.
*/
import { buildApp } from '../../src/bootstrap';
import { AppSettings } from '../../src/db/entities/AppSettings.entity';
import { AuditLog } from '../../src/db/entities/AuditLog.entity';
import { StorageJobsService } from '../../src/nest/storage/storage-jobs.service';
import { STORAGE_RECLAIMABLE_CATEGORIES, storageOwnerMarkerKey } from '../../src/nest/storage/storage-reclaim.types';
import { StorageService } from '../../src/nest/storage/storage.service';
import { countRows, deleteRows, findRow } from '../helpers/factories/rows';
import { makeAdmin, makeUser } from '../helpers/factories/users';
import { sessionCookie } from './harness';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import type { Server } from 'http';
import request from 'supertest';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('../../src/nest/audit/audit-log.logger', () => ({
LOG_LEVEL: 'error',
logInfo: vi.fn(),
logDebug: vi.fn(),
logError: vi.fn(),
logWarn: vi.fn(),
}));
describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guard)', () => {
let orm: MikroORM;
let app: INestApplication;
let server: Server;
let adminCookie: string;
let userCookie: string;
beforeAll(async () => {
app = await buildApp();
orm = app.get(MikroORM);
server = app.getHttpServer();
userCookie = sessionCookie((await makeUser(orm, { email: 'e2e-reclaim@example.test' })).user.id);
adminCookie = sessionCookie((await makeAdmin(orm, { email: 'e2e-reclaim-admin@example.test' })).user.id);
});
beforeEach(async () => {
await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } });
await deleteRows(orm, AuditLog);
});
afterAll(async () => {
// The uploads folder is the test run's shared one: take back the owner markers this app's checks wrote.
const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value;
if (installId) {
const storage = app.get(StorageService);
await Promise.all(
STORAGE_RECLAIMABLE_CATEGORIES.map((category) =>
storage.delete(category, storageOwnerMarkerKey(installId)).catch(() => undefined),
),
);
}
await app?.close();
});
/** The admin state's reclaim part once a check started by hand has finished. */
async function settledState() {
await app.get(StorageJobsService).exclusiveIdle();
const res = await request(server).get('/api/admin/storage').set('Cookie', adminCookie);
expect(res.status).toBe(200);
return res.body.reclaim;
}
it('RCLE-001: 401 without a session on every route', async () => {
expect((await request(server).put('/api/admin/storage/reclaim').send({ deleteEnabled: true })).status).toBe(401);
expect((await request(server).post('/api/admin/storage/reclaim/run').send({})).status).toBe(401);
expect((await request(server).post('/api/admin/storage/reclaim/confirm').send({ category: 'covers' })).status).toBe(
401,
);
});
it('RCLE-002: 403 for a non-admin, and nothing is stored', async () => {
const res = await request(server)
.put('/api/admin/storage/reclaim')
.set('Cookie', userCookie)
.send({ deleteEnabled: true });
expect(res.status).toBe(403);
expect(res.body).toEqual({ error: 'Admin access required' });
expect(await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' })).toBeFalsy();
});
it('RCLE-003: PUT stores the switch for this install, answers the state, audits, and the admin state carries it', async () => {
const res = await request(server)
.put('/api/admin/storage/reclaim')
.set('Cookie', adminCookie)
.send({ deleteEnabled: true });
expect(res.status).toBe(200);
expect(res.body).toEqual({ deleteEnabled: true, running: false, lastRun: null, lastDeleteRun: null });
expect(await countRows(orm, AuditLog, { action: 'admin.storage_reclaim_update' })).toBe(1);
// Stored as the install id it was turned on for, so a database restored elsewhere reads as off.
const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value;
expect(installId).toMatch(/^[0-9a-f-]{36}$/);
expect((await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' }))?.value).toBe(installId);
expect((await settledState()).deleteEnabled).toBe(true);
const off = await request(server)
.put('/api/admin/storage/reclaim')
.set('Cookie', adminCookie)
.send({ deleteEnabled: false });
expect(off.body.deleteEnabled).toBe(false);
});
it('RCLE-004: PUT and confirm refuse a body outside the contract', async () => {
const refused = await Promise.all(
[{}, { deleteEnabled: 'yes' }, { deleteEnabled: true, extra: 1 }].map((body) =>
request(server).put('/api/admin/storage/reclaim').set('Cookie', adminCookie).send(body),
),
);
expect(refused.map((res) => res.status)).toEqual([400, 400, 400]);
const confirm = await Promise.all(
[{}, { category: 'nowhere' }, { category: 'covers', count: 3 }].map((body) =>
request(server).post('/api/admin/storage/reclaim/confirm').set('Cookie', adminCookie).send(body),
),
);
expect(confirm.map((res) => res.status)).toEqual([400, 400, 400]);
expect(await countRows(orm, AuditLog)).toBe(0);
});
it('RCLE-005: POST run answers 202 at once, the check asks every source, deletes nothing and lands on the state', async () => {
const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({});
expect(res.status).toBe(202);
expect(res.body).toEqual({ started: true });
const audit = await findRow(orm, AuditLog, { action: 'admin.storage_reclaim_run' });
expect(JSON.parse(String(audit?.details))).toEqual({ mode: 'report' });
const reclaim = await settledState();
expect(reclaim.running).toBe(false);
expect(reclaim.lastRun.mode).toBe('report');
expect(reclaim.lastDeleteRun).toBeNull();
for (const category of STORAGE_RECLAIMABLE_CATEGORIES) {
expect(reclaim.lastRun.categories[category].skipped).not.toBe('sources_missing');
expect(reclaim.lastRun.categories[category].deleted).toEqual({ objects: 0, bytes: 0 });
}
expect(reclaim.lastRun.categories.backups.skipped).toBe('not_reclaimed');
});
it('RCLE-006: 409 with a code while another storage job holds the slot, and while nothing waits for a confirmation', async () => {
const jobs = app.get(StorageJobsService);
expect(jobs.claimExclusive('a test job')).toBe(true);
try {
const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({});
expect(res.status).toBe(409);
expect(res.body).toEqual({
error: 'another storage job is running, one storage job at a time',
code: 'reclaim_busy',
});
expect(await countRows(orm, AuditLog)).toBe(0);
} finally {
jobs.releaseExclusive('a test job');
}
const confirm = await request(server)
.post('/api/admin/storage/reclaim/confirm')
.set('Cookie', adminCookie)
.send({ category: 'covers' });
expect(confirm.status).toBe(409);
expect(confirm.body.code).toBe('reclaim_nothing_to_confirm');
});
it('RCLE-007: managed mode refuses the surface with the standard body', async () => {
process.env.TREK_MANAGED = 'true';
try {
const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({});
expect(res.status).toBe(403);
expect(res.body).toEqual({
error: 'This is configured by the operator of this instance.',
code: 'MANAGED_FORBIDDEN',
});
} finally {
delete process.env.TREK_MANAGED;
}
});
});