diff --git a/server/src/nest/audit/audit.service.ts b/server/src/nest/audit/audit.service.ts index 0d1680eef..38cceab26 100644 --- a/server/src/nest/audit/audit.service.ts +++ b/server/src/nest/audit/audit.service.ts @@ -31,6 +31,8 @@ const ACTION_LABELS: Record = { 'admin.storage_backfill': 'started a storage sync', 'admin.storage_backfill_cancel': 'cancelled a storage sync', 'admin.storage_stats_refresh': 'refreshed storage usage stats', + 'admin.storage_reclaim_update': 'switched storage reclaim deletion', + 'admin.storage_reclaim_run': 'ran a storage reclaim pass', 'admin.places_contribute_decision': 'decided on sharing place details with TREK Places', 'admin.places_contribute_withdraw': 'withdrew the place details this installation shared', 'immich.private_ip_configured': 'configured Immich with private IP', diff --git a/server/src/nest/storage/storage-admin.dto.ts b/server/src/nest/storage/storage-admin.dto.ts index ea23ecf7c..834c5c91b 100644 --- a/server/src/nest/storage/storage-admin.dto.ts +++ b/server/src/nest/storage/storage-admin.dto.ts @@ -1,4 +1,10 @@ -import { storageConfigPutSchema, storageMigrationRequestSchema, storageTestRequestSchema } from '@trek/shared'; +import { + storageConfigPutSchema, + storageMigrationRequestSchema, + storageReclaimRunRequestSchema, + storageReclaimSettingsRequestSchema, + storageTestRequestSchema, +} from '@trek/shared'; import { createZodDto } from 'nestjs-zod'; @@ -12,3 +18,5 @@ import { createZodDto } from 'nestjs-zod'; export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {} export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {} export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {} +export class StorageReclaimSettingsDto extends createZodDto(storageReclaimSettingsRequestSchema) {} +export class StorageReclaimRunDto extends createZodDto(storageReclaimRunRequestSchema) {} diff --git a/server/src/nest/storage/storage-admin.service.ts b/server/src/nest/storage/storage-admin.service.ts index 9f0a28694..441e12493 100644 --- a/server/src/nest/storage/storage-admin.service.ts +++ b/server/src/nest/storage/storage-admin.service.ts @@ -4,6 +4,7 @@ import { UnitOfWork } from '../database/unit-of-work'; import { StorageJobsService } from './storage-jobs.service'; import { getSeedConfigPath } from './storage-paths'; import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe'; +import { readReclaimState } from './storage-reclaim.settings'; import { BACKENDS_KEY, CATEGORIES_KEY, VERSION_KEY, StorageRegistryService } from './storage-registry.service'; import { assertNoMaskSentinels, @@ -63,6 +64,7 @@ export class StorageAdminService { health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) }, seedFilePresent: fs.existsSync(getSeedConfigPath()), usage: await this.stats.readUsage(), + reclaim: await readReclaimState(this.appSettings), backfills: this.jobs.statuses(), migrations: this.jobs.migrationStatuses(), version: await this.registry.currentConfigVersion(), diff --git a/server/src/nest/storage/storage-jobs.service.ts b/server/src/nest/storage/storage-jobs.service.ts index 4e3698e99..e6249b33c 100644 --- a/server/src/nest/storage/storage-jobs.service.ts +++ b/server/src/nest/storage/storage-jobs.service.ts @@ -41,15 +41,38 @@ export class StorageJobsService { private readonly jobs = new Map(); private readonly migrations = new Map(); private ttlMs = 10 * 60_000; + /** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */ + private exclusive: string | null = null; - /** One storage job at a time, across both backfills and category migrations. */ + /** One storage job at a time, across backfills, category migrations and the reclaim pass. */ private anyJobRunning(): boolean { return ( + this.exclusive !== null || [...this.jobs.values()].some((j) => j.status.status === 'running') || [...this.migrations.values()].some((m) => m.status.status === 'running') ); } + /** The refusal a start meets while the slot is taken, naming what holds it. */ + private busyError(fallback: string): BackfillBusyError { + return new BackfillBusyError(this.exclusive ? `${this.exclusive} is running, one storage job at a time` : fallback); + } + + /** + * Takes the one-job slot for a pass that runs in the caller (the reclaim + * pass): false while a backfill, a migration or another such pass runs. + * The caller releases it in a finally. + */ + claimExclusive(name: string): boolean { + if (this.anyJobRunning()) return false; + this.exclusive = name; + return true; + } + + releaseExclusive(name: string): void { + if (this.exclusive === name) this.exclusive = null; + } + constructor(private readonly registry: StorageRegistryService) {} /** Test-only factory for a short TTL, since a second constructor param would confuse Nest DI. */ @@ -74,7 +97,7 @@ export class StorageJobsService { throw new BackfillTargetError(`'${mirrorName}' is not a mirror backend`); } if (this.anyJobRunning()) { - throw new BackfillBusyError('a sync is already running — one backfill at a time'); + throw this.busyError('a sync is already running — one backfill at a time'); } const prefixes = resolved.map((r) => r.keyPrefix); @@ -156,7 +179,7 @@ export class StorageJobsService { const target = this.registry.driverByName(to); if (!target) throw new MigrationTargetError(`no backend named '${to}'`); if (this.anyJobRunning()) { - throw new BackfillBusyError('a sync is already running — one storage job at a time'); + throw this.busyError('a sync is already running — one storage job at a time'); } // The DESTINATION prefix, computed for `to` before the flip — may differ // from `current.keyPrefix` (audit #8: a mode-A '' source migrating onto a diff --git a/server/src/nest/storage/storage-reclaim-registry.service.ts b/server/src/nest/storage/storage-reclaim-registry.service.ts new file mode 100644 index 000000000..4214d2904 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim-registry.service.ts @@ -0,0 +1,37 @@ +import { logWarn } from '../audit/audit-log.logger'; +import type { StorageReferenceSource, StorageReferenceSourceId } from './storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Where the domains whose rows name stored objects register their reference + * source (`storage-reclaim.types.ts`), the StorageEventsService shape: an owner + * imports StorageModule, injects this and calls `register` from its own + * `onApplicationBootstrap`, so storage never imports the owner back. + * + * Registering never throws: the e2e suites boot partial module graphs, and a + * graph without a source is no error here. The reclaim pass checks the + * registered ids against its ledger and skips every category that lacks one, + * and a full-app test pins that the app registers exactly what the ledger asks. + */ +@Injectable() +export class StorageReclaimRegistryService { + private readonly registered: StorageReferenceSource[] = []; + + register(source: StorageReferenceSource): void { + // A provider listed in two modules bootstraps twice: one entry is enough. + if (this.registered.includes(source)) return; + if (this.registered.some((existing) => existing.id === source.id)) { + // Two sources under one id are both asked; neither may hide the other. + logWarn(`[storage] a second reference source registered as '${source.id}', both are asked`); + } + this.registered.push(source); + } + + sources(): readonly StorageReferenceSource[] { + return this.registered; + } + + ids(): Set { + return new Set(this.registered.map((source) => source.id)); + } +} diff --git a/server/src/nest/storage/storage-reclaim.controller.ts b/server/src/nest/storage/storage-reclaim.controller.ts new file mode 100644 index 000000000..9a150af1e --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.controller.ts @@ -0,0 +1,76 @@ +import type { User } from '../../types'; +import { AuditService } from '../audit/audit.service'; +import { getClientIp } from '../audit/client-ip'; +import { AdminGuard } from '../auth-core/admin.guard'; +import { CurrentUser } from '../auth-core/current-user.decorator'; +import { JwtAuthGuard } from '../auth-core/jwt-auth.guard'; +import { ManagedForbidden } from '../common/managed'; +import { ResponseContract } from '../common/response-contract'; +import { StorageReclaimRunDto, StorageReclaimSettingsDto } from './storage-admin.dto'; +import { StorageReclaimService } from './storage-reclaim.service'; +import { Body, Controller, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common'; +import { + storageReclaimReportSchema, + storageReclaimStateSchema, + type StorageReclaimReport, + type StorageReclaimState, +} from '@trek/shared'; + +import type { Request } from 'express'; + +/** + * /api/admin/storage/reclaim: the admin's switch for the reclaim pass and a + * pass on demand. The state itself rides on GET /api/admin/storage. Deleting + * stays off until an admin turns it on here; a pass then still deletes only + * what has been unreferenced for a week (StorageReclaimService). + */ +@Controller('api/admin/storage/reclaim') +@UseGuards(JwtAuthGuard, AdminGuard) +@ManagedForbidden('storage backends and their contents are hoster-level configuration') +export class StorageReclaimController { + constructor( + private readonly reclaim: StorageReclaimService, + private readonly audit: AuditService, + ) {} + + @Put() + @ResponseContract(storageReclaimStateSchema) + async update( + @CurrentUser() user: User, + @Body() body: StorageReclaimSettingsDto, + @Req() req: Request, + ): Promise { + const state = await this.reclaim.setDeleteEnabled(body.deleteEnabled); + await this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_reclaim_update', + ip: getClientIp(req), + details: { deleteEnabled: body.deleteEnabled }, + }); + return state; + } + + /** One pass now, honouring the switch unless `reportOnly`. 409 on a demo or while another storage job runs. */ + @Post('run') + @HttpCode(200) + @ResponseContract(storageReclaimReportSchema) + async run( + @CurrentUser() user: User, + @Body() body: StorageReclaimRunDto, + @Req() req: Request, + ): Promise { + const report = await this.reclaim.run({ reportOnly: body.reportOnly === true }); + const rows = Object.values(report.categories); + await this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_reclaim_run', + ip: getClientIp(req), + details: { + mode: report.mode, + deleted: rows.reduce((sum, row) => sum + row.deleted.objects, 0), + bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0), + }, + }); + return report; + } +} diff --git a/server/src/nest/storage/storage-reclaim.job.ts b/server/src/nest/storage/storage-reclaim.job.ts new file mode 100644 index 000000000..406339247 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.job.ts @@ -0,0 +1,62 @@ +import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { logDebug, logError, logInfo } from '../audit/audit-log.logger'; +import { DomainError } from '../common/domain-error'; +import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { StorageReclaimService } from './storage-reclaim.service'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; +import type { StorageReclaimReport } from '@trek/shared'; + +/** + * The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00 and + * before the usage scan at 04:15, so the next usage figures already show what + * it freed. No pass at boot: a pass that may delete waits for its hour. + */ +@Injectable() +export class StorageReclaimJob implements OnApplicationBootstrap { + constructor( + private readonly registrar: CronRegistrarService, + private readonly reclaim: StorageReclaimService, + private readonly runtimeEnv: RuntimeEnvService, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + this.registrar.register('storage-reclaim', '5 4 * * *', () => this.tick()); + } + + /** + * @txIndependent a sweep in passes: each category's marks and each delete + * stand on their own (StorageReclaimService.run). + */ + async tick(now: Date = new Date()): Promise { + if (this.runtimeEnv.isDemoMode()) return; + try { + const report = await this.reclaim.run({ now }); + const line = summary(report); + if (line) logInfo(`Storage reclaim: ${line}`); + } catch (err: unknown) { + if (err instanceof DomainError && err.getStatus() === 409) { + logDebug(`Storage reclaim skipped: ${err.message}`); + return; + } + logError(`Storage reclaim: ${err instanceof Error ? err.message : String(err)}`); + } + } +} + +/** One line when the pass found, deleted or refused something; null when there was nothing to say. */ +function summary(report: StorageReclaimReport): string | null { + const rows = Object.values(report.categories); + const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0); + const deleted = rows.reduce((sum, row) => sum + row.deleted.objects, 0); + const bytes = rows.reduce((sum, row) => sum + row.deleted.bytes, 0); + const braked = Object.entries(report.categories) + .filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed') + .map(([category, row]) => `${category} (${row.skipped})`); + const parts: string[] = []; + if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`); + if (deleted > 0) parts.push(`deleted ${deleted} (${bytes} bytes)`); + if (braked.length > 0) parts.push(`skipped ${braked.join(', ')}`); + if (report.stopped) parts.push(`stopped: ${report.stopped}`); + return parts.length > 0 ? parts.join('; ') : null; +} diff --git a/server/src/nest/storage/storage-reclaim.ledger.ts b/server/src/nest/storage/storage-reclaim.ledger.ts new file mode 100644 index 000000000..bdde765c0 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.ledger.ts @@ -0,0 +1,153 @@ +import type { StorageReclaimableCategory, StorageReferenceSourceId } from './storage-reclaim.types'; +import type { BackendSnapshot } from './storage-registry.service'; +import type { ServedCategory } from './storage.types'; + +import path from 'node:path'; + +export type ReclaimRule = + | { readonly reclaim: false; readonly reason: string } + | { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] }; + +/** The owners of columns that hold a free-form upload path, which can point into any category. */ +const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos']; + +/** + * What the reclaim pass may do per category, and which reference sources must + * have registered before it may call one object of it unreferenced. A Record, + * so a new category does not compile until it is decided here (the + * KEPT_ON_DEMO precedent). A category missing one of its sources is skipped, + * never swept on a partial answer. + */ +export const RECLAIM_LEDGER: Readonly> = { + // trip_files.filename (trashed rows included) holds the bare name; collab + // note files, chat images and doc-sync copies are trip_files rows too. + files: { reclaim: true, requiredSources: ['files', ...FREE_FORM_SOURCES] }, + // trek_photos.file_path and thumbnail_path (posters), journeys.cover_image. + journey: { reclaim: true, requiredSources: FREE_FORM_SOURCES }, + // Trip, collection and journey covers; a trip copy and a journey share one object. + covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES }, + avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] }, + // places.image_url, collection_places.image_url and the contribution outbox. + places: { reclaim: true, requiredSources: FREE_FORM_SOURCES }, + photos: { + reclaim: false, + reason: + 'the legacy photo folder is served to every signed-in user by name, holds the place-photo caches in its ' + + 'google/ and trek/ subtrees, and nothing writes to it any more', + }, + 'photos-google': { reclaim: false, reason: 'a cache the place-photo sweep keeps against its meta rows' }, + 'photos-trek': { reclaim: false, reason: 'a cache with a one-hour lifetime and a sweep of its own' }, + backups: { reclaim: false, reason: 'no row names a backup; auto-backup rotates them by name' }, +}; + +/** Every source id some category of the ledger waits for: what a full app must register. */ +export function requiredReclaimSources(): Set { + const ids = new Set(); + for (const rule of Object.values(RECLAIM_LEDGER)) { + if (rule.reclaim) for (const id of rule.requiredSources) ids.add(id); + } + return ids; +} + +/** The categories the ledger lets a pass delete from. */ +export function reclaimableCategories(): StorageReclaimableCategory[] { + return (Object.keys(RECLAIM_LEDGER) as ServedCategory[]).filter( + (category): category is StorageReclaimableCategory => RECLAIM_LEDGER[category].reclaim, + ); +} + +/** + * Only the root level of a category is swept: `files/assistant/` belongs to the + * assistant's retention, `journey/thumbs/` to the thumbnail sweep, and a nested + * key elsewhere was not written by TREK. + */ +export function inReclaimScope(key: string): boolean { + return key !== '' && !key.includes('/'); +} + +export interface CategoryAssignment { + backendName: string; + keyPrefix: string; +} + +/** + * The categories whose objects may sit where another category's do, decided + * from the RESOLVED backends rather than their names: two backends can share a + * root or a bucket, `TREK_PLACE_PHOTO_DIR` can point inside the uploads folder, + * the legacy `photos/` folder holds the two photo caches, and `backups` on a + * shared backend is that backend's whole root. A category is excluded when its + * location, or a mirror member's copy of it, contains another category's (equal + * counts), or when a location cannot be resolved at all. + */ +export function sharedLocationCategories( + assignments: ReadonlyMap, + backends: readonly BackendSnapshot[], +): Set { + const byName = new Map(backends.map((backend) => [backend.name, backend])); + const locations = new Map(); + for (const [category, assignment] of assignments) { + locations.set(category, locationsOf(assignment, byName)); + } + const shared = new Set(); + for (const [category, own] of locations) { + if (own === null) { + shared.add(category); + continue; + } + for (const [other, theirs] of locations) { + if (other === category) continue; + if (theirs === null || theirs.some((location) => own.some((mine) => location.startsWith(mine)))) { + shared.add(category); + break; + } + } + } + return shared; +} + +function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap): string[] | null { + const backend = backends.get(assignment.backendName); + if (!backend) return null; + const members = + backend.type === 'mirror' + ? [backend.options.primary, ...asList(backend.options.replicas)].map((name) => backends.get(String(name))) + : [backend]; + const bases: string[] = []; + for (const member of members) { + const base = member ? baseOf(member) : null; + if (base === null) return null; + bases.push(base); + } + return bases.map((base) => base + assignment.keyPrefix.toLowerCase()); +} + +/** + * Where a backend keeps its objects, lowercased: comparing more paths as equal + * than the volume does only excludes more, never less. + */ +function baseOf(backend: BackendSnapshot): string | null { + const { options } = backend; + if (backend.type === 'local') { + if (typeof options.root !== 'string' || options.root === '') return null; + return `local:${trimSlashes(path.resolve(options.root).replace(/\\/g, '/'))}/`.toLowerCase(); + } + if (backend.type === 's3') { + if (typeof options.endpoint !== 'string' || typeof options.bucket !== 'string') return null; + const keyPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : ''; + const base = `s3:${trimSlashes(options.endpoint)}/${options.bucket}/`; + return (keyPrefix === '' ? base : `${base}${keyPrefix}/`).toLowerCase(); + } + return null; +} + +function asList(value: string | number | string[] | undefined): string[] { + return Array.isArray(value) ? value : []; +} + +function trimSlashes(value: string): string { + let start = 0; + let end = value.length; + while (start < end && value[start] === '/') start += 1; + while (end > start && value[end - 1] === '/') end -= 1; + return value.slice(start, end); +} diff --git a/server/src/nest/storage/storage-reclaim.service.ts b/server/src/nest/storage/storage-reclaim.service.ts new file mode 100644 index 000000000..6fce60fac --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.service.ts @@ -0,0 +1,419 @@ +import { AppSettings } from '../../db/entities/AppSettings.entity'; +import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity'; +import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; +import type { + ReclaimMarkRow, + StorageReclaimMarksRepository, +} from '../../db/repositories/StorageReclaimMarks.repository'; +import { dbNow, parseDbTimestamp } from '../../db/types'; +import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { logError } from '../audit/audit-log.logger'; +import { DomainError } from '../common/domain-error'; +import { DatabaseLifecycle } from '../database/database-lifecycle.service'; +import { StorageJobsService } from './storage-jobs.service'; +import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; +import { inReclaimScope, RECLAIM_LEDGER, sharedLocationCategories } from './storage-reclaim.ledger'; +import { + RECLAIM_DELETE_KEY, + RECLAIM_REPORT_KEY, + readReclaimDeleteEnabled, + readReclaimState, +} from './storage-reclaim.settings'; +import type { StorageReferenceSource } from './storage-reclaim.types'; +import { StorageRegistryService } from './storage-registry.service'; +import { StorageService } from './storage.service'; +import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable, type OnModuleInit } from '@nestjs/common'; +import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared'; + +const HOUR_MS = 60 * 60 * 1000; +/** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */ +const RECLAIM_GRACE_MS = 24 * HOUR_MS; +/** How long an object must stay unreferenced, on every pass, before a pass may delete it. */ +const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS; +/** More unreferenced than this share of a category points at a bucket another install shares: hands off. */ +const RECLAIM_BRAKE_SHARE = 0.2; +/** Keys per reference question, the page the sources are promised. */ +const SOURCE_PAGE = 500; +/** Marks per statement: four bound values each, well under SQLite's limit. */ +const MARK_BATCH = 100; +/** Deletes in flight at once. */ +const DELETE_BATCH = 20; +const JOB_NAME = 'a storage reclaim'; + +interface Candidate { + key: string; + size: number; +} + +interface PassContext { + now: number; + deleting: boolean; + sources: readonly StorageReferenceSource[]; + shared: ReadonlySet; + generation: number; +} + +interface ReclaimRunOptions { + /** Delete nothing, whatever the switch says. */ + reportOnly?: boolean; + now?: Date; +} + +/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */ +class ReclaimStopped extends Error {} + +/** + * The reclaim pass (#2924): objects in upload storage that no row names any + * more, found in two phases and, only when the admin switched deletion on, + * deleted. A pass lists each category the ledger allows, skips objects younger + * than a day (the bytes of an upload land before its row), asks every + * registered reference source which keys are still named, and marks the rest + * with the time it first saw them. An object goes only once its mark is a week + * old, after one more question to every source about that single key, through + * `StorageService.delete` (so replicas follow and a demo keeps its bytes). + * + * Brakes, each failing closed: a category whose sources are not all + * registered, whose objects share a location with another category's, whose + * listing or lookups fail, or where more than a fifth of the objects look + * unreferenced is skipped and its marks start over. A pass never runs beside + * a backfill or a category migration (the one-job slot of StorageJobsService), + * never on a demo instance, and stops when the database is reopened under it. + */ +@Injectable() +export class StorageReclaimService implements OnModuleInit { + private generation = 0; + + constructor( + private readonly storage: StorageService, + private readonly registry: StorageRegistryService, + private readonly references: StorageReclaimRegistryService, + private readonly jobs: StorageJobsService, + private readonly lifecycle: DatabaseLifecycle, + private readonly runtimeEnv: RuntimeEnvService, + @InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository, + @InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository, + ) {} + + onModuleInit(): void { + this.lifecycle.onReopened(async () => { + this.generation += 1; + }); + } + + async state(): Promise { + return await readReclaimState(this.appSettings); + } + + async setDeleteEnabled(enabled: boolean): Promise { + await this.appSettings.setValue(RECLAIM_DELETE_KEY, enabled ? 'true' : 'false'); + return await this.state(); + } + + /** + * One pass over every category, its report persisted and returned. Refuses + * with a 409 on a demo instance and while another storage job holds the slot. + */ + async run(options: ReclaimRunOptions = {}): Promise { + if (this.runtimeEnv.isDemoMode()) { + throw new DomainError(409, 'storage reclaim does not run on a demo instance', { code: 'reclaim_demo' }); + } + if (!this.jobs.claimExclusive(JOB_NAME)) { + throw new DomainError(409, 'another storage job is running, one storage job at a time', { + code: 'reclaim_busy', + }); + } + try { + return await this.pass(options.reportOnly === true, options.now ?? new Date()); + } finally { + this.jobs.releaseExclusive(JOB_NAME); + } + } + + /** + * @txIndependent a sweep in passes: each category's marks and each delete + * stand on their own, and a pass cut short leaves the work done so far valid. + */ + private async pass(reportOnly: boolean, now: Date): Promise { + const wallStart = Date.now(); + const ctx: PassContext = { + now: now.getTime(), + deleting: !reportOnly && (await readReclaimDeleteEnabled(this.appSettings)), + sources: [...this.references.sources()], + shared: this.sharedLocations(), + generation: this.generation, + }; + const categories = {} as Record; + let stopped: StorageReclaimReport['stopped'] = null; + await SERVED_CATEGORIES.reduce>(async (previous, category) => { + await previous; + if (stopped === null) { + try { + categories[category] = await this.reclaimCategory(category, ctx); + return; + } catch (err) { + if (!(err instanceof ReclaimStopped)) throw err; + stopped = 'database_reopened'; + } + } + categories[category] = emptyReport(null, 0); + }, Promise.resolve()); + const report: StorageReclaimReport = { + startedAt: ctx.now, + finishedAt: ctx.now + (Date.now() - wallStart), + mode: ctx.deleting ? 'delete' : 'report', + stopped, + categories, + }; + await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, JSON.stringify(report)); + return report; + } + + /** The categories whose resolved location overlaps another's, from the live registry. */ + private sharedLocations(): Set { + const assignments = new Map( + SERVED_CATEGORIES.map((category) => { + const { backendName, keyPrefix } = this.registry.resolve(category); + return [category, { backendName, keyPrefix }] as const; + }), + ); + return sharedLocationCategories(assignments, this.registry.snapshot().backends); + } + + /** + * @txIndependent a sweep in passes: marks are written per batch and every + * delete stands alone, so a failure part way keeps what came before. + */ + private async reclaimCategory(category: ServedCategory, ctx: PassContext): Promise { + const rule = RECLAIM_LEDGER[category]; + if (!rule.reclaim) return emptyReport('not_reclaimed', 0); + const sources = ctx.sources.filter((source) => claimsAny(source, category)); + const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id)); + if (rule.requiredSources.some((id) => !answering.has(id))) return this.skip(category, 'sources_missing', 0); + if (ctx.shared.has(category)) return this.skip(category, 'shared_location', 0); + + let listed: { objects: number; old: Candidate[] }; + try { + listed = await this.listOld(category, ctx.now); + } catch (err) { + logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`); + return this.skip(category, 'list_failed', 0); + } + let unreferenced: Candidate[]; + try { + unreferenced = await this.unreferencedOf(category, listed.old, sources); + } catch (err) { + logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`); + return this.skip(category, 'source_failed', listed.objects); + } + this.assertSameDatabase(ctx); + const unreferencedCount = countOf(unreferenced); + if (unreferenced.length > listed.objects * RECLAIM_BRAKE_SHARE) { + return { + ...(await this.skip(category, 'too_many_unreferenced', listed.objects)), + unreferenced: unreferencedCount, + }; + } + + const marks = new Map((await this.marks.marksOf(category)).map((mark) => [mark.key, mark])); + await this.writeMarks(category, unreferenced, marks, ctx.now); + const cutoff = ctx.now - RECLAIM_DELAY_MS; + const eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff)); + const deleted = ctx.deleting ? await this.deleteEligible(category, eligible, sources, ctx) : []; + return { + objects: listed.objects, + unreferenced: unreferencedCount, + eligible: countOf(eligible), + deleted: countOf(deleted), + skipped: null, + }; + } + + /** A category this pass cannot judge: nothing is deleted and its marks start over. */ + private async skip( + category: ServedCategory, + reason: NonNullable, + objects: number, + ): Promise { + await this.marks.deleteCategory(category); + return emptyReport(reason, objects); + } + + /** The root-level objects of a category, and of them the ones past the grace period. */ + private async listOld(category: ServedCategory, now: number): Promise<{ objects: number; old: Candidate[] }> { + let objects = 0; + const old: Candidate[] = []; + for await (const stat of this.storage.list(category)) { + if (!inReclaimScope(stat.key)) continue; + objects += 1; + // A missing or zero modification time reads as infinitely old: such an object is kept. + if (!(stat.mtimeMs > 0) || now - stat.mtimeMs < RECLAIM_GRACE_MS) continue; + old.push({ key: stat.key, size: stat.size }); + } + return { objects, old }; + } + + /** The candidates no source names, asked a page at a time. */ + private async unreferencedOf( + category: ServedCategory, + candidates: readonly Candidate[], + sources: readonly StorageReferenceSource[], + ): Promise { + return chunk(candidates, SOURCE_PAGE).reduce>(async (previous, page) => { + const done = await previous; + const referenced = await this.referencedIn( + category, + page.map((candidate) => candidate.key), + sources, + ); + done.push(...page.filter((candidate) => !referenced.has(candidate.key.toLowerCase()))); + return done; + }, Promise.resolve([])); + } + + /** + * The keys some source still names, lowercased. Every key is claimed: the + * ledger's required sources answer for the whole category, and a category + * missing one never gets this far. + */ + private async referencedIn( + category: ServedCategory, + keys: readonly string[], + sources: readonly StorageReferenceSource[], + ): Promise> { + const answers = await Promise.all( + sources.map(async (source) => { + const asked = keys.filter((key) => claimsKey(source, category, key)); + return asked.length === 0 ? new Set() : await source.referenced(category, asked); + }), + ); + const referenced = new Set(); + for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase()); + return referenced; + } + + /** + * @txIndependent a mark batch stands on its own: a pass that stops between + * two leaves marks that are each true. + */ + private async writeMarks( + category: ServedCategory, + unreferenced: readonly Candidate[], + marks: ReadonlyMap, + now: number, + ): Promise { + const seenAt = dbNow(new Date(now)); + const fresh = unreferenced.filter((candidate) => !marks.has(candidate.key)); + const still = new Set(unreferenced.map((candidate) => candidate.key)); + const stale = [...marks.keys()].filter((key) => !still.has(key)); + await chunk(fresh, MARK_BATCH).reduce>(async (previous, page) => { + await previous; + await this.marks.insertMissing(category, page, seenAt); + }, Promise.resolve()); + await chunk(stale, MARK_BATCH).reduce>(async (previous, page) => { + await previous; + await this.marks.deleteKeys(category, page); + }, Promise.resolve()); + } + + /** + * @txIndependent one delete per object, each with its own re-check, so a + * failure leaves the others' work standing. + */ + private async deleteEligible( + category: ServedCategory, + eligible: readonly Candidate[], + sources: readonly StorageReferenceSource[], + ctx: PassContext, + ): Promise { + return chunk(eligible, DELETE_BATCH).reduce>(async (previous, batch) => { + const done = await previous; + const results = await Promise.all(batch.map((candidate) => this.deleteOne(category, candidate, sources, ctx))); + done.push(...batch.filter((_candidate, i) => results[i])); + return done; + }, Promise.resolve([])); + } + + /** + * Asks every source about this one key again, checks the object is still the + * old one the pass judged, then deletes it and its mark. False keeps it. + * + * @txIndependent the object goes first and the mark after, so a mark never + * outlives a delete that failed. + */ + private async deleteOne( + category: ServedCategory, + candidate: Candidate, + sources: readonly StorageReferenceSource[], + ctx: PassContext, + ): Promise { + let referenced: Set; + try { + referenced = await this.referencedIn(category, [candidate.key], sources); + } catch (err) { + logError(`[storage] reclaim kept a '${category}' object, its re-check failed: ${messageOf(err)}`); + return false; + } + if (referenced.size > 0) { + await this.marks.deleteKeys(category, [candidate.key]); + return false; + } + const stat = await this.storage.stat(category, candidate.key); + if (stat === null || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS) { + // Gone, or written again since it was judged: the next pass looks at it afresh. + await this.marks.deleteKeys(category, [candidate.key]); + return false; + } + if (this.storage.keepsOnDelete(category, candidate.key)) return false; + this.assertSameDatabase(ctx); + try { + await this.storage.delete(category, candidate.key); + } catch (err) { + logError(`[storage] reclaim could not delete a '${category}' object: ${messageOf(err)}`); + return false; + } + await this.marks.deleteKeys(category, [candidate.key]); + return true; + } + + private assertSameDatabase(ctx: PassContext): void { + if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass'); + } +} + +function emptyReport(skipped: StorageReclaimCategoryReport['skipped'], objects: number): StorageReclaimCategoryReport { + const none = { objects: 0, bytes: 0 }; + return { objects, unreferenced: none, eligible: none, deleted: none, skipped }; +} + +function countOf(candidates: readonly Candidate[]): { objects: number; bytes: number } { + return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) }; +} + +function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean { + const firstSeen = parseDbTimestamp(mark?.first_seen_at); + return firstSeen !== null && firstSeen.getTime() <= cutoff; +} + +/** The source answers for some part of the category. */ +function claimsAny(source: StorageReferenceSource, category: ServedCategory): boolean { + return source.claims.some((claim) => claim.category === category); +} + +/** The source answers for the whole category: what the ledger's required sources must do. */ +function answersFor(source: StorageReferenceSource, category: ServedCategory): boolean { + return source.claims.some((claim) => claim.category === category && claim.prefix === ''); +} + +function claimsKey(source: StorageReferenceSource, category: ServedCategory, key: string): boolean { + return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix)); +} + +function chunk(items: readonly T[], size: number): T[][] { + return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size)); +} + +function messageOf(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/server/src/nest/storage/storage-reclaim.settings.ts b/server/src/nest/storage/storage-reclaim.settings.ts new file mode 100644 index 000000000..af3d1c29a --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.settings.ts @@ -0,0 +1,32 @@ +import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; +import { logWarn } from '../audit/audit-log.logger'; +import { storageReclaimReportSchema, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared'; + +/** The admin's switch: 'true' lets a pass delete. Absent, or anything else, means report only. */ +export const RECLAIM_DELETE_KEY = 'storage.reclaim_delete'; +/** The last pass's report, one JSON row (the `storage.usage` pattern). */ +export const RECLAIM_REPORT_KEY = 'storage.reclaim_report'; + +/** Fail closed: only the exact stored 'true' switches deletion on. */ +export async function readReclaimDeleteEnabled(appSettings: AppSettingsRepository): Promise { + return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === 'true'; +} + +/** The stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */ +async function readReclaimReport(appSettings: AppSettingsRepository): Promise { + const value = await appSettings.getValue(RECLAIM_REPORT_KEY); + if (value == null) return null; + try { + return storageReclaimReportSchema.parse(JSON.parse(value)); + } catch { + logWarn('[storage] the stored reclaim report is unparseable, treating it as never run'); + return null; + } +} + +export async function readReclaimState(appSettings: AppSettingsRepository): Promise { + return { + deleteEnabled: await readReclaimDeleteEnabled(appSettings), + lastRun: await readReclaimReport(appSettings), + }; +} diff --git a/server/src/nest/storage/storage-reclaim.types.ts b/server/src/nest/storage/storage-reclaim.types.ts new file mode 100644 index 000000000..ee2fc3806 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.types.ts @@ -0,0 +1,174 @@ +import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; + +/** + * The contract between the storage reclaim pass and the domains whose rows + * name stored objects (#2924). Storage cannot read those rows itself: it may + * import no domain that imports it, and it may not inject another domain's + * repositories. So each owner registers a source with + * `StorageReclaimRegistryService` and answers, for a page of keys, which of + * them its rows still name. + * + * The ids are a closed list: the ledger in storage names, per category, which + * of them must have answered before a single object of that category may be + * called unreferenced. + */ +export const STORAGE_REFERENCE_SOURCE_IDS = [ + 'trips', + 'collections', + 'places', + 'journey', + 'photos', + 'files', + 'auth', +] as const; +export type StorageReferenceSourceId = (typeof STORAGE_REFERENCE_SOURCE_IDS)[number]; + +/** + * The categories a pass may delete from, at their root level only (the + * `files/assistant/` and `journey/thumbs/` subtrees have sweeps of their own). + * A source whose columns hold free-form upload paths (a cover, an image url, + * a photo path) claims all of them, since such a value can point anywhere. + */ +export const STORAGE_RECLAIMABLE_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places'] as const; +export type StorageReclaimableCategory = (typeof STORAGE_RECLAIMABLE_CATEGORIES)[number]; + +/** A part of a category a source answers for: the keys starting with `prefix` ('' is the whole category). */ +export interface StorageReferenceClaim { + readonly category: ServedCategory; + readonly prefix: string; +} + +export interface StorageReferenceSource { + readonly id: StorageReferenceSourceId; + /** + * Every category the source's columns can point into, not only the one they + * usually do: a trip cover may name `/uploads/places/x`, and that object + * must stay. A source asked about a category it does not claim is not asked. + */ + readonly claims: ReadonlyArray; + /** + * Of these category-relative keys (as `StorageService.list` yields them, at + * most a page of 500), the ones a row still names. A row that is trashed, + * pending or otherwise not gone counts. Throwing stops the pass for the + * category; it never deletes on a failed answer. + */ + referenced(category: ServedCategory, keys: readonly string[]): Promise>; +} + +/** One object a stored value names. */ +export interface StorageKeyRef { + category: string; + key: string; +} + +/** What a column's value can mean besides the `/uploads//` and `/` spellings. */ +export interface StorageKeyOptions { + /** The column holds a bare name of this category (`users.avatar`, `trip_files.filename`). */ + bare?: ServedCategory; + /** Legacy rows may carry any path before the name; the last segment names an object of `bare` too. */ + basename?: boolean; +} + +const UPLOADS_PATH = /(?:^|\/)uploads\/([a-z0-9][a-z0-9-]*)\//gi; +const KEY_END = /[?#"'<>\r\n]/; +const SERVED = new Set(SERVED_CATEGORIES); + +/** + * Every `/uploads//` in a value, wherever it stands: a path, an + * absolute URL of this instance (`https://host/uploads/covers/x.jpg`), a leading + * `uploads/` without the slash, several in one text. A query or a fragment is + * dropped, the key is percent-decoded and trimmed. A key with a blank in it is + * reported both whole and cut at the blank, since a value may be a sentence: + * reporting one key too many only keeps an object, one too few would lose it. + */ +export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] { + if (typeof value !== 'string' || value === '') return []; + const matches = [...value.matchAll(UPLOADS_PATH)]; + return matches.flatMap((match, i) => { + const start = match.index + match[0].length; + const next = matches[i + 1]; + const tail = value.slice(start, next ? next.index : value.length); + const end = tail.search(KEY_END); + const category = match[1]!.toLowerCase(); + return keyVariants(end === -1 ? tail : tail.slice(0, end)).map((key) => ({ category, key })); + }); +} + +/** + * Every object a stored value can name: the upload paths above, the relative + * `/` spelling (`journeys.cover_image` holds `covers/x`), and + * with `options` the bare or legacy-prefixed name of one category. A value + * that names nothing this way (an https URL elsewhere, a data: URL, garbage) + * answers an empty list: it references nothing a pass can find. + */ +export function storageKeysIn(value: string | null | undefined, options: StorageKeyOptions = {}): StorageKeyRef[] { + if (typeof value !== 'string') return []; + const trimmed = stripSuffix(value).trim(); + if (trimmed === '') return []; + const refs = uploadKeysIn(value); + const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/i.exec(trimmed); + if (relative && SERVED.has(relative[1]!.toLowerCase())) { + const category = relative[1]!.toLowerCase(); + for (const key of keyVariants(relative[2]!)) refs.push({ category, key }); + } + if (options.bare) { + const bare = options.bare; + if (!/[/\\:]/.test(trimmed)) { + for (const key of keyVariants(trimmed)) refs.push({ category: bare, key }); + } else if (options.basename) { + const last = trimmed.slice(Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')) + 1); + for (const key of keyVariants(last)) refs.push({ category: bare, key }); + } + } + return refs; +} + +/** + * The candidates of `category` that any of `values` names, compared without + * regard to case: a local volume on Windows or macOS serves `A.jpg` for `a.jpg`, + * so an object is kept when a value names it in either spelling. The helper a + * source builds `referenced()` from, so no source parses with a regex of its own. + */ +export function referencedAmong( + category: ServedCategory, + candidates: readonly string[], + values: Iterable, + options: StorageKeyOptions = {}, +): Set { + const named = new Set(); + for (const value of values) { + for (const ref of storageKeysIn(value, options)) { + if (ref.category === category) named.add(ref.key.toLowerCase()); + } + } + return new Set(candidates.filter((key) => named.has(key.toLowerCase()))); +} + +function stripSuffix(value: string): string { + const cut = value.search(/[?#]/); + return cut === -1 ? value : value.slice(0, cut); +} + +/** The raw key, decoded, and cut at a blank; never an empty key or one that climbs out of its category. */ +function keyVariants(raw: string): string[] { + const forms = new Set(); + const add = (key: string) => { + const clean = key.trim().replace(/^\/+/, ''); + if (clean === '' || clean.split('/').some((segment) => segment === '..' || segment === '.')) return; + forms.add(clean); + }; + const whole = stripSuffix(raw); + for (const form of [whole, decoded(whole)]) { + add(form); + add(form.trim().split(/\s/)[0]!); + } + return [...forms]; +} + +function decoded(value: string): string { + try { + return decodeURIComponent(value); + } catch { + return value; + } +} diff --git a/server/src/nest/storage/storage.module.ts b/server/src/nest/storage/storage.module.ts index 710f9ee9d..7462fc90e 100644 --- a/server/src/nest/storage/storage.module.ts +++ b/server/src/nest/storage/storage.module.ts @@ -1,11 +1,17 @@ import { AppSettings } from '../../db/entities/AppSettings.entity'; +import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity'; import { AppConfigModule } from '../app-config/app-config.module'; import { AuditModule } from '../audit/audit.module'; +import { DatabaseLifecycleModule } from '../database/database-lifecycle.module'; import { SchedulingModule } from '../scheduling/scheduling.module'; import { StorageAdminController } from './storage-admin.controller'; import { StorageAdminService } from './storage-admin.service'; import { StorageEventsService } from './storage-events.service'; import { StorageJobsService } from './storage-jobs.service'; +import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; +import { StorageReclaimController } from './storage-reclaim.controller'; +import { StorageReclaimJob } from './storage-reclaim.job'; +import { StorageReclaimService } from './storage-reclaim.service'; import { StorageRegistryService } from './storage-registry.service'; import { StorageStatsService } from './storage-stats.service'; import { StorageUsageScanJob } from './storage-usage-scan.job'; @@ -27,11 +33,20 @@ import { Module } from '@nestjs/common'; * may cache drivers or trigger reloads. MikroOrmModule.forFeature registers * AppSettingsRepository for StorageRegistryService/StorageAdminService/ * StorageStatsService's @InjectRepository (Plan 3i, storage task — SR/SA/SS's - * 14 call sites, all on app_settings). + * 14 call sites, all on app_settings). The reclaim pass adds its marks table + * and follows database reopens (DatabaseLifecycleModule); the domains whose + * rows name stored objects register their reference sources with the exported + * StorageReclaimRegistryService, so storage never imports them. */ @Module({ - imports: [AppConfigModule, AuditModule, SchedulingModule, MikroOrmModule.forFeature([AppSettings])], - controllers: [StorageAdminController], + imports: [ + AppConfigModule, + AuditModule, + DatabaseLifecycleModule, + SchedulingModule, + MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks]), + ], + controllers: [StorageAdminController, StorageReclaimController], providers: [ StorageRegistryService, StorageService, @@ -40,7 +55,10 @@ import { Module } from '@nestjs/common'; StorageJobsService, StorageStatsService, StorageUsageScanJob, + StorageReclaimRegistryService, + StorageReclaimService, + StorageReclaimJob, ], - exports: [StorageService, StorageEventsService], + exports: [StorageService, StorageEventsService, StorageReclaimRegistryService], }) export class StorageModule {} diff --git a/server/tests/e2e/storage-reclaim.e2e.test.ts b/server/tests/e2e/storage-reclaim.e2e.test.ts new file mode 100644 index 000000000..0d273775e --- /dev/null +++ b/server/tests/e2e/storage-reclaim.e2e.test.ts @@ -0,0 +1,163 @@ +/** + * Storage reclaim e2e: /api/admin/storage/reclaim through the real + * JwtAuthGuard + AdminGuard + ManagedGuard against a migrated temp SQLite, the + * Zod pipe on both bodies and the response contracts the handlers declare. + * This graph holds StorageModule alone, so no domain registers a reference + * source: every reclaimable category reports `sources_missing` and a pass + * cannot delete anything, which is the point of the ledger. RCLE-001 to RCLE-007. + */ +import { db } from '../../src/db/database'; +import { AppSettings } from '../../src/db/entities/AppSettings.entity'; +import { AuditLog } from '../../src/db/entities/AuditLog.entity'; +import { ManagedGuard } from '../../src/nest/common/managed.guard'; +import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; +import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { StorageJobsService } from '../../src/nest/storage/storage-jobs.service'; +import { STORAGE_RECLAIMABLE_CATEGORIES } from '../../src/nest/storage/storage-reclaim.types'; +import { StorageModule } from '../../src/nest/storage/storage.module'; +import { countRows, deleteRows, findRow } from '../helpers/factories/rows'; +import { makeAdmin, makeUser } from '../helpers/factories/users'; +import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm'; +import { TestUnitOfWorkModule } from '../helpers/test-uow'; +import { sessionCookie } from './harness'; +import type { INestApplication } from '@nestjs/common'; +import { APP_GUARD } from '@nestjs/core'; +import { Test } from '@nestjs/testing'; + +import cookieParser from 'cookie-parser'; +import type { Server } from 'http'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/db/database', async () => { + const { createSnapshotTestDb, buildDbMock } = await import('../helpers/db-mock'); + return buildDbMock(createSnapshotTestDb()); +}); +vi.mock('../../src/nest/audit/audit-log.logger', () => ({ + LOG_LEVEL: 'error', + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +describe('Storage reclaim e2e (real auth + admin guard + managed guard + migrated temp SQLite)', () => { + let orm: TestOrm; + let app: INestApplication; + let server: Server; + let adminCookie: string; + let userCookie: string; + + beforeAll(async () => { + orm = await createTestOrm(db); + await makeUser(orm, { id: 1, email: 'e2e-reclaim@example.test' }); + await makeAdmin(orm, { id: 2, email: 'e2e-reclaim-admin@example.test' }); + userCookie = sessionCookie(1); + adminCookie = sessionCookie(2); + const moduleRef = await Test.createTestingModule({ + imports: [await TestUnitOfWorkModule.forRoot(db), await createTestMikroOrmModule(db), StorageModule], + providers: [{ provide: APP_GUARD, useClass: ManagedGuard }], + }).compile(); + app = moduleRef.createNestApplication(); + app.use(cookieParser()); + app.useGlobalPipes(new ZodValidationPipe()); + app.useGlobalFilters(new TrekExceptionFilter()); + await app.init(); + server = app.getHttpServer(); + }); + + beforeEach(async () => { + await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } }); + await deleteRows(orm, AuditLog); + }); + + afterAll(async () => { + await app.close(); + await orm.close(); + }); + + it('RCLE-001: 401 without a session on both routes', async () => { + expect((await request(server).put('/api/admin/storage/reclaim').send({ deleteEnabled: true })).status).toBe(401); + expect((await request(server).post('/api/admin/storage/reclaim/run').send({})).status).toBe(401); + }); + + it('RCLE-002: 403 for a non-admin, and nothing is stored', async () => { + const res = await request(server) + .put('/api/admin/storage/reclaim') + .set('Cookie', userCookie) + .send({ deleteEnabled: true }); + expect(res.status).toBe(403); + expect(res.body).toEqual({ error: 'Admin access required' }); + expect(await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' })).toBeFalsy(); + }); + + it('RCLE-003: PUT stores the switch, answers the state, audits, and GET /api/admin/storage carries it', async () => { + const res = await request(server) + .put('/api/admin/storage/reclaim') + .set('Cookie', adminCookie) + .send({ deleteEnabled: true }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ deleteEnabled: true, lastRun: null }); + expect(await countRows(orm, AuditLog, { action: 'admin.storage_reclaim_update' })).toBe(1); + const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(state.status).toBe(200); + expect(state.body.reclaim).toEqual({ deleteEnabled: true, lastRun: null }); + }); + + it('RCLE-004: PUT refuses a body outside the contract', async () => { + for (const body of [{}, { deleteEnabled: 'yes' }, { deleteEnabled: true, extra: 1 }]) { + const res = await request(server).put('/api/admin/storage/reclaim').set('Cookie', adminCookie).send(body); + expect(res.status).toBe(400); + } + const run = await request(server) + .post('/api/admin/storage/reclaim/run') + .set('Cookie', adminCookie) + .send({ reportOnly: 'no' }); + expect(run.status).toBe(400); + expect(await countRows(orm, AuditLog)).toBe(0); + }); + + it('RCLE-005: a pass without registered sources skips every reclaimable category, persists and audits', async () => { + const res = await request(server) + .post('/api/admin/storage/reclaim/run') + .set('Cookie', adminCookie) + .send({ reportOnly: true }); + expect(res.status).toBe(200); + expect(res.body.mode).toBe('report'); + for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { + expect(res.body.categories[category].skipped).toBe('sources_missing'); + } + expect(res.body.categories.backups.skipped).toBe('not_reclaimed'); + const audit = await findRow(orm, AuditLog, { action: 'admin.storage_reclaim_run' }); + expect(JSON.parse(String(audit?.details))).toEqual({ mode: 'report', deleted: 0, bytes: 0 }); + const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(state.body.reclaim.lastRun).toEqual(res.body); + }); + + it('RCLE-006: 409 while another storage job holds the slot', async () => { + const jobs = app.get(StorageJobsService); + expect(jobs.claimExclusive('a test job')).toBe(true); + try { + const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({}); + expect(res.status).toBe(409); + expect(res.body).toEqual({ error: 'another storage job is running, one storage job at a time' }); + expect(await countRows(orm, AuditLog)).toBe(0); + } finally { + jobs.releaseExclusive('a test job'); + } + }); + + it('RCLE-007: managed mode refuses the surface with the standard body', async () => { + process.env.TREK_MANAGED = 'true'; + try { + const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({}); + expect(res.status).toBe(403); + expect(res.body).toEqual({ + error: 'This is configured by the operator of this instance.', + code: 'MANAGED_FORBIDDEN', + }); + } finally { + delete process.env.TREK_MANAGED; + } + }); +}); diff --git a/server/tests/integration/storage-reclaim-sources.test.ts b/server/tests/integration/storage-reclaim-sources.test.ts new file mode 100644 index 000000000..0393aa0e7 --- /dev/null +++ b/server/tests/integration/storage-reclaim-sources.test.ts @@ -0,0 +1,58 @@ +/** + * RECLAIM-SRC: the full app registers exactly the reference sources the + * reclaim ledger waits for, and each answers for every category that waits on + * it. Registration cannot fail a boot (the partial e2e graphs lack most + * owners), so this is the one place a domain that forgot to register, or a + * source that stopped claiming a category its columns can point into, shows: + * on a real install that category would be skipped as `sources_missing` + * forever, or swept while a domain's rows were never asked. + */ +import { buildApp } from '../../src/bootstrap'; +import { db as testDb } from '../../src/db/database'; +import { StorageReclaimRegistryService } from '../../src/nest/storage/storage-reclaim-registry.service'; +import { + RECLAIM_LEDGER, + reclaimableCategories, + requiredReclaimSources, +} from '../../src/nest/storage/storage-reclaim.ledger'; +import type { INestApplication } from '@nestjs/common'; + +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/db/database', async () => { + const { createSnapshotTestDb, buildDbMock } = await import('../helpers/db-mock'); + return buildDbMock(createSnapshotTestDb()); +}); + +describe('RECLAIM-SRC: the app registers what the reclaim ledger waits for', () => { + let app: INestApplication; + + beforeAll(async () => { + app = await buildApp(); + }); + + afterAll(async () => { + await app.close(); + testDb.close(); + }); + + it('RECLAIM-SRC-001: the registered source ids are exactly the ones the ledger requires', () => { + const registry = app.get(StorageReclaimRegistryService); + expect([...registry.ids()].sort()).toEqual([...requiredReclaimSources()].sort()); + }); + + it('RECLAIM-SRC-002: every required source answers for the whole of each category waiting on it', () => { + const sources = app.get(StorageReclaimRegistryService).sources(); + for (const category of reclaimableCategories()) { + const rule = RECLAIM_LEDGER[category]; + if (!rule.reclaim) continue; + for (const id of rule.requiredSources) { + const answering = sources.filter( + (source) => + source.id === id && source.claims.some((claim) => claim.category === category && claim.prefix === ''), + ); + expect(answering.length, `${id} answers for ${category}`).toBeGreaterThan(0); + } + } + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts new file mode 100644 index 000000000..1fe8e076f --- /dev/null +++ b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts @@ -0,0 +1,85 @@ +/** + * StorageReclaimController over a stub service: what it forwards, what it + * audits, and that a refusal of the service reaches the client unchanged. + * RCLC-001 to RCLC-004. + */ +import type { AuditService } from '../../../../src/nest/audit/audit.service'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import type { StorageReclaimRunDto, StorageReclaimSettingsDto } from '../../../../src/nest/storage/storage-admin.dto'; +import { StorageReclaimController } from '../../../../src/nest/storage/storage-reclaim.controller'; +import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; +import { StorageModule } from '../../../../src/nest/storage/storage.module'; +import type { User } from '../../../../src/types'; +import { expectRegisteredController } from '../../../helpers/module-providers'; +import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared'; + +import type { Request } from 'express'; +import { describe, expect, it, vi } from 'vitest'; + +const user = { id: 7 } as User; +const req = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } as unknown as Request; + +function report(): StorageReclaimReport { + const none = { objects: 0, bytes: 0 }; + const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null }; + return { + startedAt: 1, + finishedAt: 2, + mode: 'delete', + stopped: null, + categories: { + ...(Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])) as StorageReclaimReport['categories']), + covers: { ...row, objects: 9, deleted: { objects: 2, bytes: 40 } }, + places: { ...row, objects: 3, deleted: { objects: 1, bytes: 2 } }, + }, + }; +} + +function makeController(over: Partial> = {}) { + const writeAudit = vi.fn(); + const service = { + setDeleteEnabled: vi.fn(async (enabled: boolean) => ({ deleteEnabled: enabled, lastRun: null })), + run: vi.fn(async () => report()), + ...over, + } as unknown as StorageReclaimService; + const controller = new StorageReclaimController(service, { writeAudit } as unknown as AuditService); + return { controller, service, writeAudit }; +} + +describe('StorageReclaimController', () => { + it('RCLC-001: is registered in StorageModule', () => { + expectRegisteredController(StorageModule, StorageReclaimController); + }); + + it('RCLC-002: PUT stores the switch, answers the fresh state and audits the value', async () => { + const { controller, service, writeAudit } = makeController(); + const body = { deleteEnabled: true } as StorageReclaimSettingsDto; + expect(await controller.update(user, body, req)).toEqual({ deleteEnabled: true, lastRun: null }); + expect(service.setDeleteEnabled).toHaveBeenCalledWith(true); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 7, action: 'admin.storage_reclaim_update', details: { deleteEnabled: true } }), + ); + }); + + it('RCLC-003: POST run forwards reportOnly, answers the report and audits the totals', async () => { + const { controller, service, writeAudit } = makeController(); + const answer = await controller.run(user, { reportOnly: true } as StorageReclaimRunDto, req); + expect(answer.categories.covers.deleted.objects).toBe(2); + expect(service.run).toHaveBeenCalledWith({ reportOnly: true }); + await controller.run(user, {} as StorageReclaimRunDto, req); + expect(service.run).toHaveBeenLastCalledWith({ reportOnly: false }); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'admin.storage_reclaim_run', + details: { mode: 'delete', deleted: 3, bytes: 42 }, + }), + ); + }); + + it('RCLC-004: a refused pass propagates as the DomainError and writes no audit row', async () => { + const refusal = new DomainError(409, 'another storage job is running, one storage job at a time'); + const { controller, writeAudit } = makeController({ run: vi.fn().mockRejectedValue(refusal) }); + await expect(controller.run(user, {} as StorageReclaimRunDto, req)).rejects.toBe(refusal); + expect(writeAudit).not.toHaveBeenCalled(); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.job.test.ts b/server/tests/unit/nest/storage/storage-reclaim.job.test.ts new file mode 100644 index 000000000..5cddc3157 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-reclaim.job.test.ts @@ -0,0 +1,135 @@ +/** + * StorageReclaimJob: the nightly 04:05 pass. It hands the registrar the pass's + * own promise, so the lease covers the whole run, skips a demo instance and a + * busy slot quietly, and logs one line only when the pass had something to say. + * RCLJ-001 to RCLJ-006. + */ +import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { logDebug, logError, logInfo } from '../../../../src/nest/audit/audit-log.logger'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service'; +import { StorageReclaimJob } from '../../../../src/nest/storage/storage-reclaim.job'; +import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; +import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +function registrarStub(enabled = true) { + return { + isEnabled: vi.fn(() => enabled), + register: vi.fn((_name: string, _expression: string, _onTick: () => void | Promise) => enabled), + unregister: vi.fn(), + }; +} + +function report( + over: Partial = {}, + stopped: StorageReclaimReport['stopped'] = null, +) { + const none = { objects: 0, bytes: 0 }; + const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null }; + return { + startedAt: 1, + finishedAt: 2, + mode: 'delete', + stopped, + categories: { ...Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])), ...over }, + } as StorageReclaimReport; +} + +function makeJob(run: () => Promise, opts: { enabled?: boolean; demo?: boolean } = {}) { + const registrar = registrarStub(opts.enabled ?? true); + const reclaim = { run: vi.fn(run) }; + const env = { isDemoMode: () => opts.demo === true }; + const job = new StorageReclaimJob( + registrar as unknown as CronRegistrarService, + reclaim as unknown as StorageReclaimService, + env as unknown as RuntimeEnvService, + ); + return { job, registrar, reclaim }; +} + +beforeEach(() => vi.clearAllMocks()); + +describe('StorageReclaimJob', () => { + it('RCLJ-001: registers at 04:05 and hands the registrar the pass promise itself', async () => { + let finish: (value: StorageReclaimReport) => void = () => undefined; + const { job, registrar, reclaim } = makeJob(() => new Promise((resolve) => (finish = resolve))); + job.onApplicationBootstrap(); + expect(registrar.register).toHaveBeenCalledWith('storage-reclaim', '5 4 * * *', expect.any(Function)); + const onTick = registrar.register.mock.calls[0]![2]; + const ticking = onTick(); + expect(ticking).toBeInstanceOf(Promise); + let settled = false; + void (ticking as Promise).then(() => (settled = true)); + await Promise.resolve(); + expect(settled).toBe(false); + finish(report()); + await ticking; + expect(reclaim.run).toHaveBeenCalledTimes(1); + }); + + it('RCLJ-002: does not register under the test gate, and never sweeps at boot', () => { + const { job, registrar, reclaim } = makeJob(async () => report(), { enabled: false }); + job.onApplicationBootstrap(); + expect(registrar.register).not.toHaveBeenCalled(); + expect(reclaim.run).not.toHaveBeenCalled(); + }); + + it('RCLJ-003: a demo instance is skipped without a pass', async () => { + const { job, reclaim } = makeJob(async () => report(), { demo: true }); + await job.tick(); + expect(reclaim.run).not.toHaveBeenCalled(); + }); + + it('RCLJ-004: logs one line only when something was found, deleted, skipped or stopped', async () => { + const quiet = makeJob(async () => report({ photos: { ...report().categories.photos, skipped: 'not_reclaimed' } })); + await quiet.job.tick(new Date(5)); + expect(quiet.reclaim.run).toHaveBeenCalledWith({ now: new Date(5) }); + expect(logInfo).not.toHaveBeenCalled(); + + const busy = report( + { + covers: { + objects: 10, + unreferenced: { objects: 2, bytes: 30 }, + eligible: { objects: 1, bytes: 10 }, + deleted: { objects: 1, bytes: 10 }, + skipped: null, + }, + avatars: { ...report().categories.avatars, skipped: 'sources_missing' }, + }, + 'database_reopened', + ); + await makeJob(async () => busy).job.tick(); + expect(logInfo).toHaveBeenCalledWith( + 'Storage reclaim: 2 unreferenced object(s); deleted 1 (10 bytes); skipped avatars (sources_missing); stopped: database_reopened', + ); + }); + + it('RCLJ-005: a refusal (busy, demo) is a debug line, not an error', async () => { + const { job } = makeJob(async () => { + throw new DomainError(409, 'another storage job is running', { code: 'reclaim_busy' }); + }); + await expect(job.tick()).resolves.toBeUndefined(); + expect(logDebug).toHaveBeenCalledWith('Storage reclaim skipped: another storage job is running'); + expect(logError).not.toHaveBeenCalled(); + }); + + it('RCLJ-006: any other failure is contained in one error line', async () => { + await makeJob(async () => { + throw new Error('registry not initialized'); + }).job.tick(); + expect(logError).toHaveBeenCalledWith('Storage reclaim: registry not initialized'); + await makeJob(vi.fn<() => Promise>().mockRejectedValue('plain')).job.tick(); + expect(logError).toHaveBeenCalledWith('Storage reclaim: plain'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts new file mode 100644 index 000000000..b40d4ee9a --- /dev/null +++ b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts @@ -0,0 +1,194 @@ +/** + * The reclaim ledger and the location check (#2924), plus the registry the + * reference sources land in. RCLL-001 to RCLL-011. + */ +import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service'; +import { + inReclaimScope, + RECLAIM_LEDGER, + reclaimableCategories, + requiredReclaimSources, + sharedLocationCategories, + type CategoryAssignment, + type ReclaimRule, +} from '../../../../src/nest/storage/storage-reclaim.ledger'; +import { + STORAGE_RECLAIMABLE_CATEGORIES, + STORAGE_REFERENCE_SOURCE_IDS, + type StorageReferenceSource, +} from '../../../../src/nest/storage/storage-reclaim.types'; +import type { BackendSnapshot } from '../../../../src/nest/storage/storage-registry.service'; +import { CATEGORY_PREFIXES } from '../../../../src/nest/storage/storage-registry.service'; +import { SERVED_CATEGORIES, type ServedCategory } from '../../../../src/nest/storage/storage.types'; + +import path from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +const UPLOADS = path.resolve('/srv/trek/uploads'); +const BACKUPS = path.resolve('/srv/trek/backups'); + +function local(name: string, root: string): BackendSnapshot { + return { name, type: 'local', source: 'settings', options: { root } }; +} + +/** The default layout: every served category on uploads-local, backups on its own backend. */ +function defaults(overrides: Partial> = {}) { + const map = new Map(); + for (const category of SERVED_CATEGORIES) { + map.set(category, { backendName: 'uploads-local', keyPrefix: CATEGORY_PREFIXES[category] }); + } + map.set('backups', { backendName: 'backups-local', keyPrefix: '' }); + for (const [category, assignment] of Object.entries(overrides)) map.set(category as ServedCategory, assignment); + return map; +} + +const BASE_BACKENDS = [local('uploads-local', UPLOADS), local('backups-local', BACKUPS)]; + +describe('RECLAIM_LEDGER', () => { + it('RCLL-001: reclaims exactly the five content categories and never a cache, the backups or legacy photos', () => { + expect(reclaimableCategories().sort()).toEqual([...STORAGE_RECLAIMABLE_CATEGORIES].sort()); + for (const category of ['photos', 'photos-google', 'photos-trek', 'backups'] as const) { + const rule: ReclaimRule = RECLAIM_LEDGER[category]; + expect(rule.reclaim).toBe(false); + if (!rule.reclaim) expect(rule.reason.length).toBeGreaterThan(10); + } + }); + + it('RCLL-002: every reclaimable category waits for all the free-form owners, files and avatars for their own too', () => { + const freeForm = ['trips', 'collections', 'places', 'journey', 'photos']; + for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { + const rule = RECLAIM_LEDGER[category]; + expect(rule.reclaim).toBe(true); + if (rule.reclaim) expect(rule.requiredSources).toEqual(expect.arrayContaining(freeForm)); + } + const files = RECLAIM_LEDGER.files; + const avatars = RECLAIM_LEDGER.avatars; + expect(files.reclaim && files.requiredSources).toContain('files'); + expect(avatars.reclaim && avatars.requiredSources).toContain('auth'); + expect([...requiredReclaimSources()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort()); + }); + + it('RCLL-003: only root-level keys are in scope', () => { + expect(inReclaimScope('a.jpg')).toBe(true); + expect(inReclaimScope('assistant/a.pdf')).toBe(false); + expect(inReclaimScope('thumbs/x.jpg')).toBe(false); + expect(inReclaimScope('')).toBe(false); + }); +}); + +describe('sharedLocationCategories', () => { + it('RCLL-004: the default layout shares only the legacy photos folder, which holds both caches', () => { + expect(sharedLocationCategories(defaults(), BASE_BACKENDS)).toEqual(new Set(['photos'])); + }); + + it('RCLL-005: TREK_PLACE_PHOTO_DIR inside the uploads folder overlaps the category it lands in', () => { + const backends = [...BASE_BACKENDS, local('place-photos-local', path.join(UPLOADS, 'places'))]; + const shared = sharedLocationCategories( + defaults({ 'photos-google': { backendName: 'place-photos-local', keyPrefix: '' } }), + backends, + ); + expect(shared.has('places')).toBe(true); + expect(shared.has('photos-google')).toBe(true); + expect(shared.has('covers')).toBe(false); + }); + + it('RCLL-006: backups on the uploads backend sit at its root and contain everything, yet only backups is excluded', () => { + const shared = sharedLocationCategories(defaults({ backups: { backendName: 'uploads-local', keyPrefix: '' } }), [ + local('uploads-local', UPLOADS), + ]); + expect(shared.has('backups')).toBe(true); + expect(shared.has('covers')).toBe(false); + }); + + it('RCLL-007: two backends of other names on one folder overlap, compared without regard to case', () => { + const backends = [...BASE_BACKENDS, local('again', UPLOADS.toUpperCase() + path.sep)]; + const shared = sharedLocationCategories( + defaults({ avatars: { backendName: 'again', keyPrefix: 'covers/' } }), + backends, + ); + expect(shared.has('covers')).toBe(true); + expect(shared.has('avatars')).toBe(true); + expect(shared.has('files')).toBe(false); + }); + + it('RCLL-008: two s3 backends on one bucket and key prefix overlap, a different key prefix does not', () => { + const s3 = (name: string, keyPrefix: string): BackendSnapshot => ({ + name, + type: 's3', + source: 'settings', + options: { endpoint: 'https://s3.example.com/', bucket: 'trek', keyPrefix }, + }); + const backends = [...BASE_BACKENDS, s3('a', '/prod/'), s3('b', 'prod'), s3('c', 'dev'), s3('d', '')]; + const shared = sharedLocationCategories( + defaults({ + covers: { backendName: 'a', keyPrefix: 'covers/' }, + avatars: { backendName: 'b', keyPrefix: 'covers/' }, + places: { backendName: 'c', keyPrefix: 'places/' }, + files: { backendName: 'd', keyPrefix: 'files/' }, + }), + backends, + ); + expect(shared.has('covers')).toBe(true); + expect(shared.has('avatars')).toBe(true); + expect(shared.has('places')).toBe(false); + expect(shared.has('files')).toBe(false); + }); + + it('RCLL-009: a mirror counts its primary and every replica; a replica inside another category excludes it', () => { + const backends: BackendSnapshot[] = [ + ...BASE_BACKENDS, + local('primary', path.resolve('/mnt/primary')), + local('replica', path.join(UPLOADS, 'journey')), + { name: 'm', type: 'mirror', source: 'settings', options: { primary: 'primary', replicas: ['replica'] } }, + ]; + const shared = sharedLocationCategories(defaults({ covers: { backendName: 'm', keyPrefix: 'covers/' } }), backends); + // The replica's copy of covers lands at uploads/journey/covers/, inside journey's location. + expect(shared.has('journey')).toBe(true); + expect(shared.has('covers')).toBe(false); + }); + + it('RCLL-010: a location that cannot be resolved fails closed, for itself and for every other category', () => { + const backends: BackendSnapshot[] = [ + ...BASE_BACKENDS, + { name: 'odd', type: 'local', source: 'settings', options: { root: '' } }, + { name: 'odd-s3', type: 's3', source: 'settings', options: { bucket: 'x' } }, + { name: 'odd-mirror', type: 'mirror', source: 'settings', options: { primary: 'gone', replicas: 'x' } }, + ]; + expect( + sharedLocationCategories(defaults({ covers: { backendName: 'odd', keyPrefix: 'covers/' } }), backends).has( + 'files', + ), + ).toBe(true); + for (const backendName of ['odd-s3', 'odd-mirror', 'missing']) { + const shared = sharedLocationCategories(defaults({ covers: { backendName, keyPrefix: 'covers/' } }), backends); + expect(shared.has('covers')).toBe(true); + } + }); +}); + +describe('StorageReclaimRegistryService', () => { + const source = (id: StorageReferenceSource['id']): StorageReferenceSource => ({ + id, + claims: [{ category: 'covers', prefix: '' }], + referenced: async () => new Set(), + }); + + it('RCLL-011: registers once per instance, keeps two sources of one id, and answers the ids', () => { + const registry = new StorageReclaimRegistryService(); + const trips = source('trips'); + registry.register(trips); + registry.register(trips); + registry.register(source('trips')); + registry.register(source('auth')); + expect(registry.sources()).toHaveLength(3); + expect(registry.ids()).toEqual(new Set(['trips', 'auth'])); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts new file mode 100644 index 000000000..0c663ea43 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts @@ -0,0 +1,465 @@ +/** + * The storage reclaim pass (#2924) over a real StorageModule: a real registry + * and LocalDriver on temp folders, the marks table in the snapshot db, and + * fake reference sources standing in for the owner domains. The pass marks + * first, deletes a week later only with the switch on and only after asking + * every source about the single key again; every brake skips a category and + * starts its marks over. RCLS-001 to RCLS-021. + */ +import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; +import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity'; +import { dbNow } from '../../../../src/db/types'; +import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { logError } from '../../../../src/nest/audit/audit-log.logger'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service'; +import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; +import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; +import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service'; +import { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; +import { + STORAGE_RECLAIMABLE_CATEGORIES, + STORAGE_REFERENCE_SOURCE_IDS, + type StorageReferenceClaim, + type StorageReferenceSource, + type StorageReferenceSourceId, +} from '../../../../src/nest/storage/storage-reclaim.types'; +import { StorageModule } from '../../../../src/nest/storage/storage.module'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import type { ServedCategory } from '../../../../src/nest/storage/storage.types'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { deleteRows } from '../../../helpers/factories/rows'; +import { readAppSetting, setAppSetting } from '../../../helpers/factories/settings'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Readable } from 'node:stream'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +const DAY = 24 * 60 * 60 * 1000; +const db = createSnapshotTestDb(); +let t: TestModule; +let orm: TestOrm; +let uploadsRoot: string; +let backupsRoot: string; +let reclaim: StorageReclaimService; +let storage: StorageService; +let references: StorageReclaimRegistryService; +let reopen: () => Promise; +/** What the fake sources still name, per category. */ +let named: Map>; +/** Called on every question a source is asked, before it answers. */ +let onAsk: ((category: ServedCategory, keys: readonly string[]) => void | Promise) | null; + +function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource { + const categories: readonly ServedCategory[] = + id === 'auth' ? ['avatars'] : id === 'files' ? ['files'] : STORAGE_RECLAIMABLE_CATEGORIES; + return { + id, + claims: categories.map((category): StorageReferenceClaim => ({ category, prefix: '' })), + async referenced(category, keys) { + await onAsk?.(category, keys); + const set = named.get(category) ?? new Set(); + return new Set(keys.filter((key) => set.has(key))); + }, + }; +} + +function backends(extra: unknown[] = []): string { + return JSON.stringify([ + { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, + { name: 'backups-local', type: 'local', options: { root: backupsRoot } }, + ...extra, + ]); +} + +async function put(category: ServedCategory, key: string, body = '12345'): Promise { + await storage.put(category, key, Readable.from(body)); +} + +function objectPath(category: string, key: string): string { + return path.join(uploadsRoot, category, key); +} + +/** The orphans plus `kept` named covers, so the orphans stay under the brake's fifth. */ +async function seedCovers(orphans: string[], kept = 4): Promise { + const keptKeys = Array.from({ length: kept }, (_unused, i) => `kept-${i}.jpg`); + await Promise.all([...orphans, ...keptKeys].map((key) => put('covers', key))); + named.set('covers', new Set(keptKeys)); +} + +async function marksOf(category: string): Promise> { + const rows = await orm.repo(StorageReclaimMarks).marksOf(category); + return rows + .map((row) => ({ key: row.key, first_seen_at: row.first_seen_at })) + .sort((a, b) => a.key.localeCompare(b.key)); +} + +beforeAll(async () => { + uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-uploads-')); + backupsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-backups-')); + orm = await createTestOrm(db); + await setAppSetting(orm, 'storage.backends', backends()); + t = await createTestModule({ + db, + imports: [StorageModule], + init: true, + overrides: [ + { + provide: DatabaseLifecycle, + useValue: { + onReopened: (listener: () => Promise) => { + reopen = listener; + }, + }, + }, + ], + }); + reclaim = t.get(StorageReclaimService); + storage = t.get(StorageService); + references = t.get(StorageReclaimRegistryService); + for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id)); +}); + +beforeEach(async () => { + vi.restoreAllMocks(); + vi.mocked(logError).mockClear(); + for (const entry of fs.readdirSync(uploadsRoot)) { + if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true }); + } + await deleteRows(orm, StorageReclaimMarks); + await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } }); + named = new Map(); + onAsk = null; +}); + +afterAll(async () => { + await t.close(); + await orm.close(); + db.close(); + fs.rmSync(uploadsRoot, { recursive: true, force: true }); + fs.rmSync(backupsRoot, { recursive: true, force: true }); +}); + +const T0 = new Date(Date.now() + 2 * DAY); +const WEEK_LATER = new Date(T0.getTime() + 8 * DAY); + +describe('StorageReclaimService: mark, then sweep', () => { + it('RCLS-001: a first pass only reports and marks; the report is persisted and the state carries it', async () => { + await seedCovers(['orphan.jpg']); + const report = await reclaim.run({ now: T0 }); + expect(report.mode).toBe('report'); + expect(report.stopped).toBeNull(); + expect(report.categories.covers).toEqual({ + objects: 5, + unreferenced: { objects: 1, bytes: 5 }, + eligible: { objects: 0, bytes: 0 }, + deleted: { objects: 0, bytes: 0 }, + skipped: null, + }); + expect(report.categories.files).toMatchObject({ objects: 0, skipped: null }); + for (const category of ['photos', 'photos-google', 'photos-trek', 'backups'] as const) { + expect(report.categories[category].skipped).toBe('not_reclaimed'); + } + expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]); + expect(await reclaim.state()).toEqual({ deleteEnabled: false, lastRun: report }); + expect((await t.get(StorageAdminService).state()).reclaim).toEqual({ deleteEnabled: false, lastRun: report }); + }); + + it('RCLS-002: with deletion on, a mark a week old goes through the facade and takes its mark along', async () => { + await seedCovers(['orphan.jpg']); + await reclaim.setDeleteEnabled(true); + const first = await reclaim.run({ now: T0 }); + expect(first.mode).toBe('delete'); + expect(first.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + + const second = await reclaim.run({ now: WEEK_LATER }); + expect(second.categories.covers.eligible).toEqual({ objects: 1, bytes: 5 }); + expect(second.categories.covers.deleted).toEqual({ objects: 1, bytes: 5 }); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(false); + expect(fs.existsSync(objectPath('covers', 'kept-0.jpg'))).toBe(true); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-003: a report-only pass deletes nothing even with deletion on', async () => { + await seedCovers(['orphan.jpg']); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: T0 }); + const report = await reclaim.run({ now: WEEK_LATER, reportOnly: true }); + expect(report.mode).toBe('report'); + expect(report.categories.covers.eligible.objects).toBe(1); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + }); + + it('RCLS-004: an object named again loses its mark, and its week starts over once it is unnamed', async () => { + await seedCovers(['orphan.jpg']); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: T0 }); + named.get('covers')!.add('orphan.jpg'); + const renamed = await reclaim.run({ now: new Date(T0.getTime() + DAY) }); + expect(renamed.categories.covers.unreferenced.objects).toBe(0); + expect(await marksOf('covers')).toEqual([]); + + named.get('covers')!.delete('orphan.jpg'); + const again = await reclaim.run({ now: WEEK_LATER }); + expect(again.categories.covers.eligible.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(WEEK_LATER) }]); + }); + + it('RCLS-005: the single-key re-check right before the delete keeps an object named in between', async () => { + await seedCovers(['orphan.jpg']); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: T0 }); + onAsk = (category, keys) => { + if (category === 'covers' && keys.length === 1) named.get('covers')!.add(keys[0]!); + }; + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.eligible.objects).toBe(1); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-006: young objects, a missing modification time and nested subtrees are never judged', async () => { + await put('covers', 'young.jpg'); + await put('covers', 'zero.jpg'); + await put('files', 'assistant/attached.pdf'); + await put('journey', 'thumbs/abcdef.jpg'); + const now = T0; + fs.utimesSync( + objectPath('covers', 'young.jpg'), + new Date(now.getTime() - 60_000), + new Date(now.getTime() - 60_000), + ); + fs.utimesSync(objectPath('covers', 'zero.jpg'), 0, 0); + const report = await reclaim.run({ now }); + expect(report.categories.covers).toMatchObject({ objects: 2, unreferenced: { objects: 0, bytes: 0 } }); + expect(report.categories.files.objects).toBe(0); + expect(report.categories.journey.objects).toBe(0); + expect(await marksOf('covers')).toEqual([]); + }); +}); + +describe('StorageReclaimService: brakes', () => { + it('RCLS-007: more than a fifth unreferenced skips the category, deletes nothing and clears its marks', async () => { + await seedCovers(['o1.jpg', 'o2.jpg'], 1); + await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'o1.jpg', size: 5 }], dbNow(new Date(0))); + await reclaim.setDeleteEnabled(true); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers).toEqual({ + objects: 3, + unreferenced: { objects: 2, bytes: 10 }, + eligible: { objects: 0, bytes: 0 }, + deleted: { objects: 0, bytes: 0 }, + skipped: 'too_many_unreferenced', + }); + expect(fs.existsSync(objectPath('covers', 'o1.jpg'))).toBe(true); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-008: a category whose required source is missing, or answers for a part only, is skipped', async () => { + await seedCovers(['orphan.jpg']); + const all = [...references.sources()]; + const withoutAuth = all.filter((source) => source.id !== 'auth'); + vi.spyOn(references, 'sources').mockReturnValue(withoutAuth); + let report = await reclaim.run({ now: T0 }); + expect(report.categories.avatars.skipped).toBe('sources_missing'); + expect(report.categories.covers.skipped).toBeNull(); + + const partialTrips: StorageReferenceSource = { + id: 'trips', + claims: [{ category: 'covers', prefix: 'sub/' }], + referenced: async () => new Set(), + }; + vi.spyOn(references, 'sources').mockReturnValue([...all.filter((source) => source.id !== 'trips'), partialTrips]); + report = await reclaim.run({ now: T0 }); + expect(report.categories.covers.skipped).toBe('sources_missing'); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-009: a source answering for a subtree only is not asked about root keys', async () => { + await seedCovers(['orphan.jpg']); + const subtree = vi.fn(async () => new Set()); + vi.spyOn(references, 'sources').mockReturnValue([ + ...references.sources(), + { id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], referenced: subtree }, + ]); + const report = await reclaim.run({ now: T0 }); + expect(report.categories.covers.unreferenced.objects).toBe(1); + expect(subtree).not.toHaveBeenCalled(); + }); + + it('RCLS-010: a category whose location holds another category is skipped', async () => { + await seedCovers(['orphan.jpg']); + const nestedRoot = path.join(uploadsRoot, 'covers'); + await setAppSetting( + orm, + 'storage.backends', + backends([{ name: 'nested', type: 'local', options: { root: nestedRoot } }]), + ); + await setAppSetting(orm, 'storage.categories', JSON.stringify({ 'photos-trek': 'nested' })); + await storage.reloadConfig(); + try { + const report = await reclaim.run({ now: T0 }); + expect(report.categories.covers.skipped).toBe('shared_location'); + expect(report.categories.files.skipped).toBeNull(); + } finally { + await setAppSetting(orm, 'storage.backends', backends()); + await deleteRows(orm, AppSettings, { key: 'storage.categories' }); + await storage.reloadConfig(); + } + }); + + it('RCLS-011: a listing that fails skips the category and clears its marks', async () => { + await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'x.jpg', size: 1 }], dbNow(new Date(0))); + vi.spyOn(storage, 'list').mockImplementation(() => ({ + [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(new Error('backend down')) }), + })); + const report = await reclaim.run({ now: T0 }); + expect(report.categories.covers.skipped).toBe('list_failed'); + expect(report.categories.files.skipped).toBe('list_failed'); + expect(logError).toHaveBeenCalledWith(expect.stringContaining('backend down')); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-012: a source that fails skips the category, never reads it as unreferenced', async () => { + await seedCovers(['orphan.jpg']); + onAsk = (category) => { + if (category === 'covers') throw new Error('trips table locked'); + }; + const report = await reclaim.run({ now: T0 }); + expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'source_failed' }); + expect(report.categories.places.skipped).toBeNull(); + expect(logError).toHaveBeenCalledWith(expect.stringContaining('trips table locked')); + }); + + it('RCLS-013: a pass refuses on a demo instance and while another storage job holds the slot', async () => { + const demo = vi.spyOn(t.get(RuntimeEnvService), 'isDemoMode').mockReturnValue(true); + await expect(reclaim.run()).rejects.toMatchObject({ code: 'reclaim_demo' }); + demo.mockRestore(); + + const jobs = t.get(StorageJobsService); + expect(jobs.claimExclusive('a test')).toBe(true); + try { + const refusal = await reclaim.run().catch((err: unknown) => err); + expect(refusal).toBeInstanceOf(DomainError); + expect(refusal).toMatchObject({ code: 'reclaim_busy' }); + expect((refusal as DomainError).getStatus()).toBe(409); + } finally { + jobs.releaseExclusive('a test'); + } + }); + + it('RCLS-014: while a pass holds the slot, a migration is refused with its name, and the slot is free after', async () => { + const jobs = t.get(StorageJobsService); + let refusal: unknown = null; + onAsk = async (category) => { + if (category !== 'covers' || refusal) return; + refusal = await jobs.startMigration('covers', 'backups-local').catch((err: unknown) => err); + }; + await seedCovers(['orphan.jpg']); + await reclaim.run({ now: T0 }); + expect(String(refusal)).toContain('a storage reclaim is running, one storage job at a time'); + expect(jobs.claimExclusive('after')).toBe(true); + jobs.releaseExclusive('after'); + }); + + it('RCLS-015: a database reopened mid-pass stops it before anything else is judged', async () => { + await seedCovers(['orphan.jpg']); + await put('places', 'p.jpg'); + let reopened = false; + onAsk = async (category) => { + if (category === 'covers' && !reopened) { + reopened = true; + await reopen(); + } + }; + const report = await reclaim.run({ now: T0 }); + expect(report.stopped).toBe('database_reopened'); + expect(report.categories.covers.unreferenced.objects).toBe(0); + expect(report.categories.places.objects).toBe(0); + expect(await marksOf('covers')).toEqual([]); + expect((await reclaim.state()).lastRun?.stopped).toBe('database_reopened'); + }); +}); + +describe('StorageReclaimService: the delete itself', () => { + async function markedWeekOld(orphans: string[], kept = 4): Promise { + await seedCovers(orphans, kept); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: T0 }); + } + + it('RCLS-016: a re-check that fails keeps the object', async () => { + await markedWeekOld(['orphan.jpg']); + onAsk = (_category, keys) => { + if (keys.length === 1) throw new Error('lookup timed out'); + }; + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + expect(logError).toHaveBeenCalledWith(expect.stringContaining('lookup timed out')); + }); + + it('RCLS-017: an object gone or written again since it was judged is left and its mark dropped', async () => { + await markedWeekOld(['gone.jpg', 'fresh.jpg'], 8); + onAsk = (_category, keys) => { + if (keys.length !== 1) return; + if (keys[0] === 'gone.jpg') fs.rmSync(objectPath('covers', 'gone.jpg'), { force: true }); + if (keys[0] === 'fresh.jpg') fs.utimesSync(objectPath('covers', 'fresh.jpg'), WEEK_LATER, WEEK_LATER); + }; + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.eligible.objects).toBe(2); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'fresh.jpg'))).toBe(true); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-018: a delete that fails keeps the mark for the next pass', async () => { + await markedWeekOld(['orphan.jpg']); + vi.spyOn(storage, 'delete').mockRejectedValue(new Error('permission denied')); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]); + expect(logError).toHaveBeenCalledWith(expect.stringContaining('permission denied')); + }); + + it('RCLS-019: an object the facade keeps on delete is not counted as reclaimed', async () => { + await markedWeekOld(['orphan.jpg']); + vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + }); +}); + +describe('StorageReclaimService: the switch and the stored report', () => { + it('RCLS-020: the switch is stored as true or false and only the exact true deletes', async () => { + expect((await reclaim.setDeleteEnabled(true)).deleteEnabled).toBe(true); + expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBe('true'); + expect((await reclaim.setDeleteEnabled(false)).deleteEnabled).toBe(false); + await setAppSetting(orm, 'storage.reclaim_delete', 'yes'); + expect((await reclaim.state()).deleteEnabled).toBe(false); + }); + + it('RCLS-021: a stored report that no longer parses reads as never run', async () => { + await setAppSetting(orm, 'storage.reclaim_report', '{not json'); + expect((await reclaim.state()).lastRun).toBeNull(); + await setAppSetting(orm, 'storage.reclaim_report', JSON.stringify({ startedAt: 'yesterday' })); + expect((await reclaim.state()).lastRun).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.types.test.ts b/server/tests/unit/nest/storage/storage-reclaim.types.test.ts new file mode 100644 index 000000000..e8573fcaf --- /dev/null +++ b/server/tests/unit/nest/storage/storage-reclaim.types.test.ts @@ -0,0 +1,125 @@ +/** + * The reference parser every reclaim source builds on (#2924): which objects a + * stored value names, in every spelling the columns hold. A key reported too + * many only keeps an object; one missed would delete it, so the cases lean on + * the spellings the validators let through (absolute self-URLs, queries, + * relative journey covers, legacy file names). RCLP-001 to RCLP-012. + */ +import { + referencedAmong, + storageKeysIn, + uploadKeysIn, + type StorageKeyOptions, + type StorageKeyRef, +} from '../../../../src/nest/storage/storage-reclaim.types'; + +import { describe, expect, it } from 'vitest'; + +const ref = (category: string, key: string): StorageKeyRef => ({ category, key }); + +describe('uploadKeysIn', () => { + it('RCLP-001: finds the path in its plain, slash-less and absolute self-URL spellings', () => { + expect(uploadKeysIn('/uploads/covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]); + expect(uploadKeysIn('uploads/files/b.pdf')).toEqual([ref('files', 'b.pdf')]); + expect(uploadKeysIn('https://trip.example.com/uploads/places/c.png')).toEqual([ref('places', 'c.png')]); + expect(uploadKeysIn('HTTPS://HOST/Uploads/Covers/D.JPG')).toEqual([ref('covers', 'D.JPG')]); + }); + + it('RCLP-002: drops a query or a fragment, decodes and trims the key', () => { + expect(uploadKeysIn('/uploads/places/x.jpg?v=2')).toEqual([ref('places', 'x.jpg')]); + expect(uploadKeysIn('/uploads/places/x.jpg#top')).toEqual([ref('places', 'x.jpg')]); + expect(uploadKeysIn(' /uploads/covers/y.jpg ')).toEqual([ref('covers', 'y.jpg')]); + expect(uploadKeysIn('/uploads/photos/my%20photo.jpg')).toEqual([ + ref('photos', 'my%20photo.jpg'), + ref('photos', 'my photo.jpg'), + ref('photos', 'my'), + ]); + }); + + it('RCLP-003: finds every path in a text, each cut where the next begins or the text ends', () => { + expect(uploadKeysIn('see /uploads/covers/a.jpg and "/uploads/places/b.jpg" too')).toEqual([ + ref('covers', 'a.jpg and'), + ref('covers', 'a.jpg'), + ref('places', 'b.jpg'), + ]); + expect(uploadKeysIn('/uploads/covers/a.jpg/uploads/places/b.jpg')).toEqual([ + ref('covers', 'a.jpg'), + ref('places', 'b.jpg'), + ]); + }); + + it('RCLP-004: keeps a nested key whole and refuses empty or climbing keys', () => { + expect(uploadKeysIn('/uploads/journey/thumbs/abc.jpg')).toEqual([ref('journey', 'thumbs/abc.jpg')]); + expect(uploadKeysIn('/uploads/covers/')).toEqual([]); + expect(uploadKeysIn('/uploads/covers/../avatars/a.jpg')).toEqual([]); + expect(uploadKeysIn('/uploads/covers/%2E%2E/x')).toEqual([ref('covers', '%2E%2E/x')]); + }); + + it('RCLP-005: names nothing for empty values, other hosts and other paths', () => { + expect(uploadKeysIn(null)).toEqual([]); + expect(uploadKeysIn(undefined)).toEqual([]); + expect(uploadKeysIn('')).toEqual([]); + expect(uploadKeysIn('https://images.unsplash.com/photo-123?w=800')).toEqual([]); + expect(uploadKeysIn('/api/maps/place-photo/abc/bytes')).toEqual([]); + expect(uploadKeysIn('/my-uploads/covers/a.jpg')).toEqual([]); + }); + + it('RCLP-006: a broken percent sequence keeps the raw key', () => { + expect(uploadKeysIn('/uploads/covers/100%.jpg')).toEqual([ref('covers', '100%.jpg')]); + }); +}); + +describe('storageKeysIn', () => { + it('RCLP-007: reads the relative / spelling of journey covers', () => { + expect(storageKeysIn('covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]); + expect(storageKeysIn('journey/b.jpg?x=1')).toEqual([ref('journey', 'b.jpg')]); + expect(storageKeysIn('./journey/thumbs/c.jpg')).toEqual([ref('journey', 'thumbs/c.jpg')]); + expect(storageKeysIn('notacategory/a.jpg')).toEqual([]); + }); + + it('RCLP-008: a bare name belongs to the column category, a URL or a path does not', () => { + expect(storageKeysIn('a1b2.jpg', { bare: 'avatars' })).toEqual([ref('avatars', 'a1b2.jpg')]); + expect(storageKeysIn('https://idp.example.com/pic.jpg', { bare: 'avatars' })).toEqual([]); + expect(storageKeysIn('some/dir/x.pdf', { bare: 'files' })).toEqual([]); + expect(storageKeysIn('a1b2.jpg')).toEqual([]); + }); + + it('RCLP-009: with basename, a legacy path still names its last segment', () => { + expect(storageKeysIn('files/x.pdf', { bare: 'files', basename: true })).toEqual([ + ref('files', 'x.pdf'), + ref('files', 'x.pdf'), + ]); + expect(storageKeysIn('/uploads/files/y.pdf', { bare: 'files', basename: true })).toEqual([ + ref('files', 'y.pdf'), + ref('files', 'y.pdf'), + ]); + expect(storageKeysIn('C:\\old\\uploads\\z.pdf', { bare: 'files', basename: true })).toEqual([ + ref('files', 'z.pdf'), + ]); + }); + + it('RCLP-010: names nothing for a blank or a non-string', () => { + expect(storageKeysIn(' ', { bare: 'files' })).toEqual([]); + expect(storageKeysIn('?only=query', { bare: 'files' })).toEqual([]); + expect(storageKeysIn(null)).toEqual([]); + expect(storageKeysIn(42 as unknown as string)).toEqual([]); + }); +}); + +describe('referencedAmong', () => { + it('RCLP-011: answers the candidates of one category the values name, without regard to case', () => { + const values = ['/uploads/covers/A.JPG', 'covers/b.jpg', '/uploads/places/c.jpg', null, 'https://elsewhere/x']; + expect(referencedAmong('covers', ['a.jpg', 'b.jpg', 'c.jpg', 'd.jpg'], values)).toEqual( + new Set(['a.jpg', 'b.jpg']), + ); + expect(referencedAmong('places', ['c.jpg'], values)).toEqual(new Set(['c.jpg'])); + }); + + it('RCLP-012: passes the column options on, so a bare avatar name keeps its object', () => { + const avatarColumn: StorageKeyOptions = { bare: 'avatars' }; + expect(referencedAmong('avatars', ['u1.png'], ['u1.png'], avatarColumn)).toEqual(new Set(['u1.png'])); + expect( + referencedAmong('avatars', ['u1.png', 'u2.png'], ['u1.png', 'https://idp/u2.png'], { bare: 'avatars' }), + ).toEqual(new Set(['u1.png'])); + }); +});