mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-13 07:40:59 +00:00
feat(storage): find and reclaim upload objects no row names any more
Domains whose rows name stored objects register a reference source with StorageReclaimRegistryService; storage never imports them. A ledger in storage decides per category whether it is reclaimed at all and which sources must have answered first, and a category whose resolved location overlaps another one is left alone. A nightly pass at 04:05 lists the root level of files, journey, covers, avatars and places, skips objects younger than a day or without a modification time, asks every source in pages of 500 and marks the rest. Only with deletion switched on, and only for marks a week old, it asks every source about the single key again and deletes through StorageService. A category is skipped and its marks start over when a source is missing or fails, the listing fails, or more than a fifth of its objects look unreferenced. A pass shares the one-job slot with backfills and migrations, never runs on a demo instance and stops when the database is reopened. Admins switch deletion and run a pass on demand under /api/admin/storage/reclaim; both are audited, and GET /api/admin/storage carries the switch and the last report. Deletion is off after install and upgrade.
This commit is contained in:
Maurice
@@ -31,6 +31,8 @@ const ACTION_LABELS: Record<string, string> = {
|
||||
'admin.storage_backfill': 'started a storage sync',
|
||||
'admin.storage_backfill_cancel': 'cancelled a storage sync',
|
||||
'admin.storage_stats_refresh': 'refreshed storage usage stats',
|
||||
'admin.storage_reclaim_update': 'switched storage reclaim deletion',
|
||||
'admin.storage_reclaim_run': 'ran a storage reclaim pass',
|
||||
'admin.places_contribute_decision': 'decided on sharing place details with TREK Places',
|
||||
'admin.places_contribute_withdraw': 'withdrew the place details this installation shared',
|
||||
'immich.private_ip_configured': 'configured Immich with private IP',
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
import { storageConfigPutSchema, storageMigrationRequestSchema, storageTestRequestSchema } from '@trek/shared';
|
||||
import {
|
||||
storageConfigPutSchema,
|
||||
storageMigrationRequestSchema,
|
||||
storageReclaimRunRequestSchema,
|
||||
storageReclaimSettingsRequestSchema,
|
||||
storageTestRequestSchema,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { createZodDto } from 'nestjs-zod';
|
||||
|
||||
@@ -12,3 +18,5 @@ import { createZodDto } from 'nestjs-zod';
|
||||
export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {}
|
||||
export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {}
|
||||
export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {}
|
||||
export class StorageReclaimSettingsDto extends createZodDto(storageReclaimSettingsRequestSchema) {}
|
||||
export class StorageReclaimRunDto extends createZodDto(storageReclaimRunRequestSchema) {}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { getSeedConfigPath } from './storage-paths';
|
||||
import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe';
|
||||
import { readReclaimState } from './storage-reclaim.settings';
|
||||
import { BACKENDS_KEY, CATEGORIES_KEY, VERSION_KEY, StorageRegistryService } from './storage-registry.service';
|
||||
import {
|
||||
assertNoMaskSentinels,
|
||||
@@ -63,6 +64,7 @@ export class StorageAdminService {
|
||||
health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) },
|
||||
seedFilePresent: fs.existsSync(getSeedConfigPath()),
|
||||
usage: await this.stats.readUsage(),
|
||||
reclaim: await readReclaimState(this.appSettings),
|
||||
backfills: this.jobs.statuses(),
|
||||
migrations: this.jobs.migrationStatuses(),
|
||||
version: await this.registry.currentConfigVersion(),
|
||||
|
||||
@@ -41,15 +41,38 @@ export class StorageJobsService {
|
||||
private readonly jobs = new Map<string, ActiveJob>();
|
||||
private readonly migrations = new Map<string, ActiveMigration>();
|
||||
private ttlMs = 10 * 60_000;
|
||||
/** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */
|
||||
private exclusive: string | null = null;
|
||||
|
||||
/** One storage job at a time, across both backfills and category migrations. */
|
||||
/** One storage job at a time, across backfills, category migrations and the reclaim pass. */
|
||||
private anyJobRunning(): boolean {
|
||||
return (
|
||||
this.exclusive !== null ||
|
||||
[...this.jobs.values()].some((j) => j.status.status === 'running') ||
|
||||
[...this.migrations.values()].some((m) => m.status.status === 'running')
|
||||
);
|
||||
}
|
||||
|
||||
/** The refusal a start meets while the slot is taken, naming what holds it. */
|
||||
private busyError(fallback: string): BackfillBusyError {
|
||||
return new BackfillBusyError(this.exclusive ? `${this.exclusive} is running, one storage job at a time` : fallback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the one-job slot for a pass that runs in the caller (the reclaim
|
||||
* pass): false while a backfill, a migration or another such pass runs.
|
||||
* The caller releases it in a finally.
|
||||
*/
|
||||
claimExclusive(name: string): boolean {
|
||||
if (this.anyJobRunning()) return false;
|
||||
this.exclusive = name;
|
||||
return true;
|
||||
}
|
||||
|
||||
releaseExclusive(name: string): void {
|
||||
if (this.exclusive === name) this.exclusive = null;
|
||||
}
|
||||
|
||||
constructor(private readonly registry: StorageRegistryService) {}
|
||||
|
||||
/** Test-only factory for a short TTL, since a second constructor param would confuse Nest DI. */
|
||||
@@ -74,7 +97,7 @@ export class StorageJobsService {
|
||||
throw new BackfillTargetError(`'${mirrorName}' is not a mirror backend`);
|
||||
}
|
||||
if (this.anyJobRunning()) {
|
||||
throw new BackfillBusyError('a sync is already running — one backfill at a time');
|
||||
throw this.busyError('a sync is already running — one backfill at a time');
|
||||
}
|
||||
const prefixes = resolved.map((r) => r.keyPrefix);
|
||||
|
||||
@@ -156,7 +179,7 @@ export class StorageJobsService {
|
||||
const target = this.registry.driverByName(to);
|
||||
if (!target) throw new MigrationTargetError(`no backend named '${to}'`);
|
||||
if (this.anyJobRunning()) {
|
||||
throw new BackfillBusyError('a sync is already running — one storage job at a time');
|
||||
throw this.busyError('a sync is already running — one storage job at a time');
|
||||
}
|
||||
// The DESTINATION prefix, computed for `to` before the flip — may differ
|
||||
// from `current.keyPrefix` (audit #8: a mode-A '' source migrating onto a
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import type { StorageReferenceSource, StorageReferenceSourceId } from './storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Where the domains whose rows name stored objects register their reference
|
||||
* source (`storage-reclaim.types.ts`), the StorageEventsService shape: an owner
|
||||
* imports StorageModule, injects this and calls `register` from its own
|
||||
* `onApplicationBootstrap`, so storage never imports the owner back.
|
||||
*
|
||||
* Registering never throws: the e2e suites boot partial module graphs, and a
|
||||
* graph without a source is no error here. The reclaim pass checks the
|
||||
* registered ids against its ledger and skips every category that lacks one,
|
||||
* and a full-app test pins that the app registers exactly what the ledger asks.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimRegistryService {
|
||||
private readonly registered: StorageReferenceSource[] = [];
|
||||
|
||||
register(source: StorageReferenceSource): void {
|
||||
// A provider listed in two modules bootstraps twice: one entry is enough.
|
||||
if (this.registered.includes(source)) return;
|
||||
if (this.registered.some((existing) => existing.id === source.id)) {
|
||||
// Two sources under one id are both asked; neither may hide the other.
|
||||
logWarn(`[storage] a second reference source registered as '${source.id}', both are asked`);
|
||||
}
|
||||
this.registered.push(source);
|
||||
}
|
||||
|
||||
sources(): readonly StorageReferenceSource[] {
|
||||
return this.registered;
|
||||
}
|
||||
|
||||
ids(): Set<StorageReferenceSourceId> {
|
||||
return new Set(this.registered.map((source) => source.id));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { User } from '../../types';
|
||||
import { AuditService } from '../audit/audit.service';
|
||||
import { getClientIp } from '../audit/client-ip';
|
||||
import { AdminGuard } from '../auth-core/admin.guard';
|
||||
import { CurrentUser } from '../auth-core/current-user.decorator';
|
||||
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
|
||||
import { ManagedForbidden } from '../common/managed';
|
||||
import { ResponseContract } from '../common/response-contract';
|
||||
import { StorageReclaimRunDto, StorageReclaimSettingsDto } from './storage-admin.dto';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { Body, Controller, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common';
|
||||
import {
|
||||
storageReclaimReportSchema,
|
||||
storageReclaimStateSchema,
|
||||
type StorageReclaimReport,
|
||||
type StorageReclaimState,
|
||||
} from '@trek/shared';
|
||||
|
||||
import type { Request } from 'express';
|
||||
|
||||
/**
|
||||
* /api/admin/storage/reclaim: the admin's switch for the reclaim pass and a
|
||||
* pass on demand. The state itself rides on GET /api/admin/storage. Deleting
|
||||
* stays off until an admin turns it on here; a pass then still deletes only
|
||||
* what has been unreferenced for a week (StorageReclaimService).
|
||||
*/
|
||||
@Controller('api/admin/storage/reclaim')
|
||||
@UseGuards(JwtAuthGuard, AdminGuard)
|
||||
@ManagedForbidden('storage backends and their contents are hoster-level configuration')
|
||||
export class StorageReclaimController {
|
||||
constructor(
|
||||
private readonly reclaim: StorageReclaimService,
|
||||
private readonly audit: AuditService,
|
||||
) {}
|
||||
|
||||
@Put()
|
||||
@ResponseContract(storageReclaimStateSchema)
|
||||
async update(
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: StorageReclaimSettingsDto,
|
||||
@Req() req: Request,
|
||||
): Promise<StorageReclaimState> {
|
||||
const state = await this.reclaim.setDeleteEnabled(body.deleteEnabled);
|
||||
await this.audit.writeAudit({
|
||||
userId: user.id,
|
||||
action: 'admin.storage_reclaim_update',
|
||||
ip: getClientIp(req),
|
||||
details: { deleteEnabled: body.deleteEnabled },
|
||||
});
|
||||
return state;
|
||||
}
|
||||
|
||||
/** One pass now, honouring the switch unless `reportOnly`. 409 on a demo or while another storage job runs. */
|
||||
@Post('run')
|
||||
@HttpCode(200)
|
||||
@ResponseContract(storageReclaimReportSchema)
|
||||
async run(
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: StorageReclaimRunDto,
|
||||
@Req() req: Request,
|
||||
): Promise<StorageReclaimReport> {
|
||||
const report = await this.reclaim.run({ reportOnly: body.reportOnly === true });
|
||||
const rows = Object.values(report.categories);
|
||||
await this.audit.writeAudit({
|
||||
userId: user.id,
|
||||
action: 'admin.storage_reclaim_run',
|
||||
ip: getClientIp(req),
|
||||
details: {
|
||||
mode: report.mode,
|
||||
deleted: rows.reduce((sum, row) => sum + row.deleted.objects, 0),
|
||||
bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0),
|
||||
},
|
||||
});
|
||||
return report;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { logDebug, logError, logInfo } from '../audit/audit-log.logger';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
import type { StorageReclaimReport } from '@trek/shared';
|
||||
|
||||
/**
|
||||
* The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00 and
|
||||
* before the usage scan at 04:15, so the next usage figures already show what
|
||||
* it freed. No pass at boot: a pass that may delete waits for its hour.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
private readonly registrar: CronRegistrarService,
|
||||
private readonly reclaim: StorageReclaimService,
|
||||
private readonly runtimeEnv: RuntimeEnvService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('storage-reclaim', '5 4 * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: each category's marks and each delete
|
||||
* stand on their own (StorageReclaimService.run).
|
||||
*/
|
||||
async tick(now: Date = new Date()): Promise<void> {
|
||||
if (this.runtimeEnv.isDemoMode()) return;
|
||||
try {
|
||||
const report = await this.reclaim.run({ now });
|
||||
const line = summary(report);
|
||||
if (line) logInfo(`Storage reclaim: ${line}`);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof DomainError && err.getStatus() === 409) {
|
||||
logDebug(`Storage reclaim skipped: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
logError(`Storage reclaim: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One line when the pass found, deleted or refused something; null when there was nothing to say. */
|
||||
function summary(report: StorageReclaimReport): string | null {
|
||||
const rows = Object.values(report.categories);
|
||||
const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0);
|
||||
const deleted = rows.reduce((sum, row) => sum + row.deleted.objects, 0);
|
||||
const bytes = rows.reduce((sum, row) => sum + row.deleted.bytes, 0);
|
||||
const braked = Object.entries(report.categories)
|
||||
.filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed')
|
||||
.map(([category, row]) => `${category} (${row.skipped})`);
|
||||
const parts: string[] = [];
|
||||
if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`);
|
||||
if (deleted > 0) parts.push(`deleted ${deleted} (${bytes} bytes)`);
|
||||
if (braked.length > 0) parts.push(`skipped ${braked.join(', ')}`);
|
||||
if (report.stopped) parts.push(`stopped: ${report.stopped}`);
|
||||
return parts.length > 0 ? parts.join('; ') : null;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import type { StorageReclaimableCategory, StorageReferenceSourceId } from './storage-reclaim.types';
|
||||
import type { BackendSnapshot } from './storage-registry.service';
|
||||
import type { ServedCategory } from './storage.types';
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
export type ReclaimRule =
|
||||
| { readonly reclaim: false; readonly reason: string }
|
||||
| { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] };
|
||||
|
||||
/** The owners of columns that hold a free-form upload path, which can point into any category. */
|
||||
const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos'];
|
||||
|
||||
/**
|
||||
* What the reclaim pass may do per category, and which reference sources must
|
||||
* have registered before it may call one object of it unreferenced. A Record,
|
||||
* so a new category does not compile until it is decided here (the
|
||||
* KEPT_ON_DEMO precedent). A category missing one of its sources is skipped,
|
||||
* never swept on a partial answer.
|
||||
*/
|
||||
export const RECLAIM_LEDGER: Readonly<Record<ServedCategory, ReclaimRule>> = {
|
||||
// trip_files.filename (trashed rows included) holds the bare name; collab
|
||||
// note files, chat images and doc-sync copies are trip_files rows too.
|
||||
files: { reclaim: true, requiredSources: ['files', ...FREE_FORM_SOURCES] },
|
||||
// trek_photos.file_path and thumbnail_path (posters), journeys.cover_image.
|
||||
journey: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
// Trip, collection and journey covers; a trip copy and a journey share one object.
|
||||
covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] },
|
||||
// places.image_url, collection_places.image_url and the contribution outbox.
|
||||
places: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
photos: {
|
||||
reclaim: false,
|
||||
reason:
|
||||
'the legacy photo folder is served to every signed-in user by name, holds the place-photo caches in its ' +
|
||||
'google/ and trek/ subtrees, and nothing writes to it any more',
|
||||
},
|
||||
'photos-google': { reclaim: false, reason: 'a cache the place-photo sweep keeps against its meta rows' },
|
||||
'photos-trek': { reclaim: false, reason: 'a cache with a one-hour lifetime and a sweep of its own' },
|
||||
backups: { reclaim: false, reason: 'no row names a backup; auto-backup rotates them by name' },
|
||||
};
|
||||
|
||||
/** Every source id some category of the ledger waits for: what a full app must register. */
|
||||
export function requiredReclaimSources(): Set<StorageReferenceSourceId> {
|
||||
const ids = new Set<StorageReferenceSourceId>();
|
||||
for (const rule of Object.values(RECLAIM_LEDGER)) {
|
||||
if (rule.reclaim) for (const id of rule.requiredSources) ids.add(id);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/** The categories the ledger lets a pass delete from. */
|
||||
export function reclaimableCategories(): StorageReclaimableCategory[] {
|
||||
return (Object.keys(RECLAIM_LEDGER) as ServedCategory[]).filter(
|
||||
(category): category is StorageReclaimableCategory => RECLAIM_LEDGER[category].reclaim,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Only the root level of a category is swept: `files/assistant/` belongs to the
|
||||
* assistant's retention, `journey/thumbs/` to the thumbnail sweep, and a nested
|
||||
* key elsewhere was not written by TREK.
|
||||
*/
|
||||
export function inReclaimScope(key: string): boolean {
|
||||
return key !== '' && !key.includes('/');
|
||||
}
|
||||
|
||||
export interface CategoryAssignment {
|
||||
backendName: string;
|
||||
keyPrefix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The categories whose objects may sit where another category's do, decided
|
||||
* from the RESOLVED backends rather than their names: two backends can share a
|
||||
* root or a bucket, `TREK_PLACE_PHOTO_DIR` can point inside the uploads folder,
|
||||
* the legacy `photos/` folder holds the two photo caches, and `backups` on a
|
||||
* shared backend is that backend's whole root. A category is excluded when its
|
||||
* location, or a mirror member's copy of it, contains another category's (equal
|
||||
* counts), or when a location cannot be resolved at all.
|
||||
*/
|
||||
export function sharedLocationCategories(
|
||||
assignments: ReadonlyMap<ServedCategory, CategoryAssignment>,
|
||||
backends: readonly BackendSnapshot[],
|
||||
): Set<ServedCategory> {
|
||||
const byName = new Map(backends.map((backend) => [backend.name, backend]));
|
||||
const locations = new Map<ServedCategory, string[] | null>();
|
||||
for (const [category, assignment] of assignments) {
|
||||
locations.set(category, locationsOf(assignment, byName));
|
||||
}
|
||||
const shared = new Set<ServedCategory>();
|
||||
for (const [category, own] of locations) {
|
||||
if (own === null) {
|
||||
shared.add(category);
|
||||
continue;
|
||||
}
|
||||
for (const [other, theirs] of locations) {
|
||||
if (other === category) continue;
|
||||
if (theirs === null || theirs.some((location) => own.some((mine) => location.startsWith(mine)))) {
|
||||
shared.add(category);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return shared;
|
||||
}
|
||||
|
||||
function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap<string, BackendSnapshot>): string[] | null {
|
||||
const backend = backends.get(assignment.backendName);
|
||||
if (!backend) return null;
|
||||
const members =
|
||||
backend.type === 'mirror'
|
||||
? [backend.options.primary, ...asList(backend.options.replicas)].map((name) => backends.get(String(name)))
|
||||
: [backend];
|
||||
const bases: string[] = [];
|
||||
for (const member of members) {
|
||||
const base = member ? baseOf(member) : null;
|
||||
if (base === null) return null;
|
||||
bases.push(base);
|
||||
}
|
||||
return bases.map((base) => base + assignment.keyPrefix.toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a backend keeps its objects, lowercased: comparing more paths as equal
|
||||
* than the volume does only excludes more, never less.
|
||||
*/
|
||||
function baseOf(backend: BackendSnapshot): string | null {
|
||||
const { options } = backend;
|
||||
if (backend.type === 'local') {
|
||||
if (typeof options.root !== 'string' || options.root === '') return null;
|
||||
return `local:${trimSlashes(path.resolve(options.root).replace(/\\/g, '/'))}/`.toLowerCase();
|
||||
}
|
||||
if (backend.type === 's3') {
|
||||
if (typeof options.endpoint !== 'string' || typeof options.bucket !== 'string') return null;
|
||||
const keyPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : '';
|
||||
const base = `s3:${trimSlashes(options.endpoint)}/${options.bucket}/`;
|
||||
return (keyPrefix === '' ? base : `${base}${keyPrefix}/`).toLowerCase();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function asList(value: string | number | string[] | undefined): string[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function trimSlashes(value: string): string {
|
||||
let start = 0;
|
||||
let end = value.length;
|
||||
while (start < end && value[start] === '/') start += 1;
|
||||
while (end > start && value[end - 1] === '/') end -= 1;
|
||||
return value.slice(start, end);
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import { AppSettings } from '../../db/entities/AppSettings.entity';
|
||||
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
|
||||
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
|
||||
import type {
|
||||
ReclaimMarkRow,
|
||||
StorageReclaimMarksRepository,
|
||||
} from '../../db/repositories/StorageReclaimMarks.repository';
|
||||
import { dbNow, parseDbTimestamp } from '../../db/types';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
|
||||
import { inReclaimScope, RECLAIM_LEDGER, sharedLocationCategories } from './storage-reclaim.ledger';
|
||||
import {
|
||||
RECLAIM_DELETE_KEY,
|
||||
RECLAIM_REPORT_KEY,
|
||||
readReclaimDeleteEnabled,
|
||||
readReclaimState,
|
||||
} from './storage-reclaim.settings';
|
||||
import type { StorageReferenceSource } from './storage-reclaim.types';
|
||||
import { StorageRegistryService } from './storage-registry.service';
|
||||
import { StorageService } from './storage.service';
|
||||
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnModuleInit } from '@nestjs/common';
|
||||
import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared';
|
||||
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
/** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */
|
||||
const RECLAIM_GRACE_MS = 24 * HOUR_MS;
|
||||
/** How long an object must stay unreferenced, on every pass, before a pass may delete it. */
|
||||
const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS;
|
||||
/** More unreferenced than this share of a category points at a bucket another install shares: hands off. */
|
||||
const RECLAIM_BRAKE_SHARE = 0.2;
|
||||
/** Keys per reference question, the page the sources are promised. */
|
||||
const SOURCE_PAGE = 500;
|
||||
/** Marks per statement: four bound values each, well under SQLite's limit. */
|
||||
const MARK_BATCH = 100;
|
||||
/** Deletes in flight at once. */
|
||||
const DELETE_BATCH = 20;
|
||||
const JOB_NAME = 'a storage reclaim';
|
||||
|
||||
interface Candidate {
|
||||
key: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
interface PassContext {
|
||||
now: number;
|
||||
deleting: boolean;
|
||||
sources: readonly StorageReferenceSource[];
|
||||
shared: ReadonlySet<ServedCategory>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
interface ReclaimRunOptions {
|
||||
/** Delete nothing, whatever the switch says. */
|
||||
reportOnly?: boolean;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */
|
||||
class ReclaimStopped extends Error {}
|
||||
|
||||
/**
|
||||
* The reclaim pass (#2924): objects in upload storage that no row names any
|
||||
* more, found in two phases and, only when the admin switched deletion on,
|
||||
* deleted. A pass lists each category the ledger allows, skips objects younger
|
||||
* than a day (the bytes of an upload land before its row), asks every
|
||||
* registered reference source which keys are still named, and marks the rest
|
||||
* with the time it first saw them. An object goes only once its mark is a week
|
||||
* old, after one more question to every source about that single key, through
|
||||
* `StorageService.delete` (so replicas follow and a demo keeps its bytes).
|
||||
*
|
||||
* Brakes, each failing closed: a category whose sources are not all
|
||||
* registered, whose objects share a location with another category's, whose
|
||||
* listing or lookups fail, or where more than a fifth of the objects look
|
||||
* unreferenced is skipped and its marks start over. A pass never runs beside
|
||||
* a backfill or a category migration (the one-job slot of StorageJobsService),
|
||||
* never on a demo instance, and stops when the database is reopened under it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimService implements OnModuleInit {
|
||||
private generation = 0;
|
||||
|
||||
constructor(
|
||||
private readonly storage: StorageService,
|
||||
private readonly registry: StorageRegistryService,
|
||||
private readonly references: StorageReclaimRegistryService,
|
||||
private readonly jobs: StorageJobsService,
|
||||
private readonly lifecycle: DatabaseLifecycle,
|
||||
private readonly runtimeEnv: RuntimeEnvService,
|
||||
@InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository,
|
||||
@InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
this.lifecycle.onReopened(async () => {
|
||||
this.generation += 1;
|
||||
});
|
||||
}
|
||||
|
||||
async state(): Promise<StorageReclaimState> {
|
||||
return await readReclaimState(this.appSettings);
|
||||
}
|
||||
|
||||
async setDeleteEnabled(enabled: boolean): Promise<StorageReclaimState> {
|
||||
await this.appSettings.setValue(RECLAIM_DELETE_KEY, enabled ? 'true' : 'false');
|
||||
return await this.state();
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass over every category, its report persisted and returned. Refuses
|
||||
* with a 409 on a demo instance and while another storage job holds the slot.
|
||||
*/
|
||||
async run(options: ReclaimRunOptions = {}): Promise<StorageReclaimReport> {
|
||||
if (this.runtimeEnv.isDemoMode()) {
|
||||
throw new DomainError(409, 'storage reclaim does not run on a demo instance', { code: 'reclaim_demo' });
|
||||
}
|
||||
if (!this.jobs.claimExclusive(JOB_NAME)) {
|
||||
throw new DomainError(409, 'another storage job is running, one storage job at a time', {
|
||||
code: 'reclaim_busy',
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await this.pass(options.reportOnly === true, options.now ?? new Date());
|
||||
} finally {
|
||||
this.jobs.releaseExclusive(JOB_NAME);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: each category's marks and each delete
|
||||
* stand on their own, and a pass cut short leaves the work done so far valid.
|
||||
*/
|
||||
private async pass(reportOnly: boolean, now: Date): Promise<StorageReclaimReport> {
|
||||
const wallStart = Date.now();
|
||||
const ctx: PassContext = {
|
||||
now: now.getTime(),
|
||||
deleting: !reportOnly && (await readReclaimDeleteEnabled(this.appSettings)),
|
||||
sources: [...this.references.sources()],
|
||||
shared: this.sharedLocations(),
|
||||
generation: this.generation,
|
||||
};
|
||||
const categories = {} as Record<ServedCategory, StorageReclaimCategoryReport>;
|
||||
let stopped: StorageReclaimReport['stopped'] = null;
|
||||
await SERVED_CATEGORIES.reduce<Promise<void>>(async (previous, category) => {
|
||||
await previous;
|
||||
if (stopped === null) {
|
||||
try {
|
||||
categories[category] = await this.reclaimCategory(category, ctx);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (!(err instanceof ReclaimStopped)) throw err;
|
||||
stopped = 'database_reopened';
|
||||
}
|
||||
}
|
||||
categories[category] = emptyReport(null, 0);
|
||||
}, Promise.resolve());
|
||||
const report: StorageReclaimReport = {
|
||||
startedAt: ctx.now,
|
||||
finishedAt: ctx.now + (Date.now() - wallStart),
|
||||
mode: ctx.deleting ? 'delete' : 'report',
|
||||
stopped,
|
||||
categories,
|
||||
};
|
||||
await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, JSON.stringify(report));
|
||||
return report;
|
||||
}
|
||||
|
||||
/** The categories whose resolved location overlaps another's, from the live registry. */
|
||||
private sharedLocations(): Set<ServedCategory> {
|
||||
const assignments = new Map(
|
||||
SERVED_CATEGORIES.map((category) => {
|
||||
const { backendName, keyPrefix } = this.registry.resolve(category);
|
||||
return [category, { backendName, keyPrefix }] as const;
|
||||
}),
|
||||
);
|
||||
return sharedLocationCategories(assignments, this.registry.snapshot().backends);
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: marks are written per batch and every
|
||||
* delete stands alone, so a failure part way keeps what came before.
|
||||
*/
|
||||
private async reclaimCategory(category: ServedCategory, ctx: PassContext): Promise<StorageReclaimCategoryReport> {
|
||||
const rule = RECLAIM_LEDGER[category];
|
||||
if (!rule.reclaim) return emptyReport('not_reclaimed', 0);
|
||||
const sources = ctx.sources.filter((source) => claimsAny(source, category));
|
||||
const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id));
|
||||
if (rule.requiredSources.some((id) => !answering.has(id))) return this.skip(category, 'sources_missing', 0);
|
||||
if (ctx.shared.has(category)) return this.skip(category, 'shared_location', 0);
|
||||
|
||||
let listed: { objects: number; old: Candidate[] };
|
||||
try {
|
||||
listed = await this.listOld(category, ctx.now);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`);
|
||||
return this.skip(category, 'list_failed', 0);
|
||||
}
|
||||
let unreferenced: Candidate[];
|
||||
try {
|
||||
unreferenced = await this.unreferencedOf(category, listed.old, sources);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`);
|
||||
return this.skip(category, 'source_failed', listed.objects);
|
||||
}
|
||||
this.assertSameDatabase(ctx);
|
||||
const unreferencedCount = countOf(unreferenced);
|
||||
if (unreferenced.length > listed.objects * RECLAIM_BRAKE_SHARE) {
|
||||
return {
|
||||
...(await this.skip(category, 'too_many_unreferenced', listed.objects)),
|
||||
unreferenced: unreferencedCount,
|
||||
};
|
||||
}
|
||||
|
||||
const marks = new Map((await this.marks.marksOf(category)).map((mark) => [mark.key, mark]));
|
||||
await this.writeMarks(category, unreferenced, marks, ctx.now);
|
||||
const cutoff = ctx.now - RECLAIM_DELAY_MS;
|
||||
const eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff));
|
||||
const deleted = ctx.deleting ? await this.deleteEligible(category, eligible, sources, ctx) : [];
|
||||
return {
|
||||
objects: listed.objects,
|
||||
unreferenced: unreferencedCount,
|
||||
eligible: countOf(eligible),
|
||||
deleted: countOf(deleted),
|
||||
skipped: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** A category this pass cannot judge: nothing is deleted and its marks start over. */
|
||||
private async skip(
|
||||
category: ServedCategory,
|
||||
reason: NonNullable<StorageReclaimCategoryReport['skipped']>,
|
||||
objects: number,
|
||||
): Promise<StorageReclaimCategoryReport> {
|
||||
await this.marks.deleteCategory(category);
|
||||
return emptyReport(reason, objects);
|
||||
}
|
||||
|
||||
/** The root-level objects of a category, and of them the ones past the grace period. */
|
||||
private async listOld(category: ServedCategory, now: number): Promise<{ objects: number; old: Candidate[] }> {
|
||||
let objects = 0;
|
||||
const old: Candidate[] = [];
|
||||
for await (const stat of this.storage.list(category)) {
|
||||
if (!inReclaimScope(stat.key)) continue;
|
||||
objects += 1;
|
||||
// A missing or zero modification time reads as infinitely old: such an object is kept.
|
||||
if (!(stat.mtimeMs > 0) || now - stat.mtimeMs < RECLAIM_GRACE_MS) continue;
|
||||
old.push({ key: stat.key, size: stat.size });
|
||||
}
|
||||
return { objects, old };
|
||||
}
|
||||
|
||||
/** The candidates no source names, asked a page at a time. */
|
||||
private async unreferencedOf(
|
||||
category: ServedCategory,
|
||||
candidates: readonly Candidate[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
): Promise<Candidate[]> {
|
||||
return chunk(candidates, SOURCE_PAGE).reduce<Promise<Candidate[]>>(async (previous, page) => {
|
||||
const done = await previous;
|
||||
const referenced = await this.referencedIn(
|
||||
category,
|
||||
page.map((candidate) => candidate.key),
|
||||
sources,
|
||||
);
|
||||
done.push(...page.filter((candidate) => !referenced.has(candidate.key.toLowerCase())));
|
||||
return done;
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* The keys some source still names, lowercased. Every key is claimed: the
|
||||
* ledger's required sources answer for the whole category, and a category
|
||||
* missing one never gets this far.
|
||||
*/
|
||||
private async referencedIn(
|
||||
category: ServedCategory,
|
||||
keys: readonly string[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
): Promise<Set<string>> {
|
||||
const answers = await Promise.all(
|
||||
sources.map(async (source) => {
|
||||
const asked = keys.filter((key) => claimsKey(source, category, key));
|
||||
return asked.length === 0 ? new Set<string>() : await source.referenced(category, asked);
|
||||
}),
|
||||
);
|
||||
const referenced = new Set<string>();
|
||||
for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase());
|
||||
return referenced;
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a mark batch stands on its own: a pass that stops between
|
||||
* two leaves marks that are each true.
|
||||
*/
|
||||
private async writeMarks(
|
||||
category: ServedCategory,
|
||||
unreferenced: readonly Candidate[],
|
||||
marks: ReadonlyMap<string, ReclaimMarkRow>,
|
||||
now: number,
|
||||
): Promise<void> {
|
||||
const seenAt = dbNow(new Date(now));
|
||||
const fresh = unreferenced.filter((candidate) => !marks.has(candidate.key));
|
||||
const still = new Set(unreferenced.map((candidate) => candidate.key));
|
||||
const stale = [...marks.keys()].filter((key) => !still.has(key));
|
||||
await chunk(fresh, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
|
||||
await previous;
|
||||
await this.marks.insertMissing(category, page, seenAt);
|
||||
}, Promise.resolve());
|
||||
await chunk(stale, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
|
||||
await previous;
|
||||
await this.marks.deleteKeys(category, page);
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent one delete per object, each with its own re-check, so a
|
||||
* failure leaves the others' work standing.
|
||||
*/
|
||||
private async deleteEligible(
|
||||
category: ServedCategory,
|
||||
eligible: readonly Candidate[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
ctx: PassContext,
|
||||
): Promise<Candidate[]> {
|
||||
return chunk(eligible, DELETE_BATCH).reduce<Promise<Candidate[]>>(async (previous, batch) => {
|
||||
const done = await previous;
|
||||
const results = await Promise.all(batch.map((candidate) => this.deleteOne(category, candidate, sources, ctx)));
|
||||
done.push(...batch.filter((_candidate, i) => results[i]));
|
||||
return done;
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks every source about this one key again, checks the object is still the
|
||||
* old one the pass judged, then deletes it and its mark. False keeps it.
|
||||
*
|
||||
* @txIndependent the object goes first and the mark after, so a mark never
|
||||
* outlives a delete that failed.
|
||||
*/
|
||||
private async deleteOne(
|
||||
category: ServedCategory,
|
||||
candidate: Candidate,
|
||||
sources: readonly StorageReferenceSource[],
|
||||
ctx: PassContext,
|
||||
): Promise<boolean> {
|
||||
let referenced: Set<string>;
|
||||
try {
|
||||
referenced = await this.referencedIn(category, [candidate.key], sources);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim kept a '${category}' object, its re-check failed: ${messageOf(err)}`);
|
||||
return false;
|
||||
}
|
||||
if (referenced.size > 0) {
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return false;
|
||||
}
|
||||
const stat = await this.storage.stat(category, candidate.key);
|
||||
if (stat === null || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS) {
|
||||
// Gone, or written again since it was judged: the next pass looks at it afresh.
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return false;
|
||||
}
|
||||
if (this.storage.keepsOnDelete(category, candidate.key)) return false;
|
||||
this.assertSameDatabase(ctx);
|
||||
try {
|
||||
await this.storage.delete(category, candidate.key);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not delete a '${category}' object: ${messageOf(err)}`);
|
||||
return false;
|
||||
}
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return true;
|
||||
}
|
||||
|
||||
private assertSameDatabase(ctx: PassContext): void {
|
||||
if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass');
|
||||
}
|
||||
}
|
||||
|
||||
function emptyReport(skipped: StorageReclaimCategoryReport['skipped'], objects: number): StorageReclaimCategoryReport {
|
||||
const none = { objects: 0, bytes: 0 };
|
||||
return { objects, unreferenced: none, eligible: none, deleted: none, skipped };
|
||||
}
|
||||
|
||||
function countOf(candidates: readonly Candidate[]): { objects: number; bytes: number } {
|
||||
return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) };
|
||||
}
|
||||
|
||||
function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean {
|
||||
const firstSeen = parseDbTimestamp(mark?.first_seen_at);
|
||||
return firstSeen !== null && firstSeen.getTime() <= cutoff;
|
||||
}
|
||||
|
||||
/** The source answers for some part of the category. */
|
||||
function claimsAny(source: StorageReferenceSource, category: ServedCategory): boolean {
|
||||
return source.claims.some((claim) => claim.category === category);
|
||||
}
|
||||
|
||||
/** The source answers for the whole category: what the ledger's required sources must do. */
|
||||
function answersFor(source: StorageReferenceSource, category: ServedCategory): boolean {
|
||||
return source.claims.some((claim) => claim.category === category && claim.prefix === '');
|
||||
}
|
||||
|
||||
function claimsKey(source: StorageReferenceSource, category: ServedCategory, key: string): boolean {
|
||||
return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix));
|
||||
}
|
||||
|
||||
function chunk<T>(items: readonly T[], size: number): T[][] {
|
||||
return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size));
|
||||
}
|
||||
|
||||
function messageOf(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import { storageReclaimReportSchema, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared';
|
||||
|
||||
/** The admin's switch: 'true' lets a pass delete. Absent, or anything else, means report only. */
|
||||
export const RECLAIM_DELETE_KEY = 'storage.reclaim_delete';
|
||||
/** The last pass's report, one JSON row (the `storage.usage` pattern). */
|
||||
export const RECLAIM_REPORT_KEY = 'storage.reclaim_report';
|
||||
|
||||
/** Fail closed: only the exact stored 'true' switches deletion on. */
|
||||
export async function readReclaimDeleteEnabled(appSettings: AppSettingsRepository): Promise<boolean> {
|
||||
return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === 'true';
|
||||
}
|
||||
|
||||
/** The stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */
|
||||
async function readReclaimReport(appSettings: AppSettingsRepository): Promise<StorageReclaimReport | null> {
|
||||
const value = await appSettings.getValue(RECLAIM_REPORT_KEY);
|
||||
if (value == null) return null;
|
||||
try {
|
||||
return storageReclaimReportSchema.parse(JSON.parse(value));
|
||||
} catch {
|
||||
logWarn('[storage] the stored reclaim report is unparseable, treating it as never run');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readReclaimState(appSettings: AppSettingsRepository): Promise<StorageReclaimState> {
|
||||
return {
|
||||
deleteEnabled: await readReclaimDeleteEnabled(appSettings),
|
||||
lastRun: await readReclaimReport(appSettings),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
|
||||
|
||||
/**
|
||||
* The contract between the storage reclaim pass and the domains whose rows
|
||||
* name stored objects (#2924). Storage cannot read those rows itself: it may
|
||||
* import no domain that imports it, and it may not inject another domain's
|
||||
* repositories. So each owner registers a source with
|
||||
* `StorageReclaimRegistryService` and answers, for a page of keys, which of
|
||||
* them its rows still name.
|
||||
*
|
||||
* The ids are a closed list: the ledger in storage names, per category, which
|
||||
* of them must have answered before a single object of that category may be
|
||||
* called unreferenced.
|
||||
*/
|
||||
export const STORAGE_REFERENCE_SOURCE_IDS = [
|
||||
'trips',
|
||||
'collections',
|
||||
'places',
|
||||
'journey',
|
||||
'photos',
|
||||
'files',
|
||||
'auth',
|
||||
] as const;
|
||||
export type StorageReferenceSourceId = (typeof STORAGE_REFERENCE_SOURCE_IDS)[number];
|
||||
|
||||
/**
|
||||
* The categories a pass may delete from, at their root level only (the
|
||||
* `files/assistant/` and `journey/thumbs/` subtrees have sweeps of their own).
|
||||
* A source whose columns hold free-form upload paths (a cover, an image url,
|
||||
* a photo path) claims all of them, since such a value can point anywhere.
|
||||
*/
|
||||
export const STORAGE_RECLAIMABLE_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places'] as const;
|
||||
export type StorageReclaimableCategory = (typeof STORAGE_RECLAIMABLE_CATEGORIES)[number];
|
||||
|
||||
/** A part of a category a source answers for: the keys starting with `prefix` ('' is the whole category). */
|
||||
export interface StorageReferenceClaim {
|
||||
readonly category: ServedCategory;
|
||||
readonly prefix: string;
|
||||
}
|
||||
|
||||
export interface StorageReferenceSource {
|
||||
readonly id: StorageReferenceSourceId;
|
||||
/**
|
||||
* Every category the source's columns can point into, not only the one they
|
||||
* usually do: a trip cover may name `/uploads/places/x`, and that object
|
||||
* must stay. A source asked about a category it does not claim is not asked.
|
||||
*/
|
||||
readonly claims: ReadonlyArray<StorageReferenceClaim>;
|
||||
/**
|
||||
* Of these category-relative keys (as `StorageService.list` yields them, at
|
||||
* most a page of 500), the ones a row still names. A row that is trashed,
|
||||
* pending or otherwise not gone counts. Throwing stops the pass for the
|
||||
* category; it never deletes on a failed answer.
|
||||
*/
|
||||
referenced(category: ServedCategory, keys: readonly string[]): Promise<Set<string>>;
|
||||
}
|
||||
|
||||
/** One object a stored value names. */
|
||||
export interface StorageKeyRef {
|
||||
category: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
/** What a column's value can mean besides the `/uploads/<category>/<key>` and `<category>/<key>` spellings. */
|
||||
export interface StorageKeyOptions {
|
||||
/** The column holds a bare name of this category (`users.avatar`, `trip_files.filename`). */
|
||||
bare?: ServedCategory;
|
||||
/** Legacy rows may carry any path before the name; the last segment names an object of `bare` too. */
|
||||
basename?: boolean;
|
||||
}
|
||||
|
||||
const UPLOADS_PATH = /(?:^|\/)uploads\/([a-z0-9][a-z0-9-]*)\//gi;
|
||||
const KEY_END = /[?#"'<>\r\n]/;
|
||||
const SERVED = new Set<string>(SERVED_CATEGORIES);
|
||||
|
||||
/**
|
||||
* Every `/uploads/<category>/<key>` in a value, wherever it stands: a path, an
|
||||
* absolute URL of this instance (`https://host/uploads/covers/x.jpg`), a leading
|
||||
* `uploads/` without the slash, several in one text. A query or a fragment is
|
||||
* dropped, the key is percent-decoded and trimmed. A key with a blank in it is
|
||||
* reported both whole and cut at the blank, since a value may be a sentence:
|
||||
* reporting one key too many only keeps an object, one too few would lose it.
|
||||
*/
|
||||
export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] {
|
||||
if (typeof value !== 'string' || value === '') return [];
|
||||
const matches = [...value.matchAll(UPLOADS_PATH)];
|
||||
return matches.flatMap((match, i) => {
|
||||
const start = match.index + match[0].length;
|
||||
const next = matches[i + 1];
|
||||
const tail = value.slice(start, next ? next.index : value.length);
|
||||
const end = tail.search(KEY_END);
|
||||
const category = match[1]!.toLowerCase();
|
||||
return keyVariants(end === -1 ? tail : tail.slice(0, end)).map((key) => ({ category, key }));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Every object a stored value can name: the upload paths above, the relative
|
||||
* `<category>/<key>` spelling (`journeys.cover_image` holds `covers/x`), and
|
||||
* with `options` the bare or legacy-prefixed name of one category. A value
|
||||
* that names nothing this way (an https URL elsewhere, a data: URL, garbage)
|
||||
* answers an empty list: it references nothing a pass can find.
|
||||
*/
|
||||
export function storageKeysIn(value: string | null | undefined, options: StorageKeyOptions = {}): StorageKeyRef[] {
|
||||
if (typeof value !== 'string') return [];
|
||||
const trimmed = stripSuffix(value).trim();
|
||||
if (trimmed === '') return [];
|
||||
const refs = uploadKeysIn(value);
|
||||
const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/i.exec(trimmed);
|
||||
if (relative && SERVED.has(relative[1]!.toLowerCase())) {
|
||||
const category = relative[1]!.toLowerCase();
|
||||
for (const key of keyVariants(relative[2]!)) refs.push({ category, key });
|
||||
}
|
||||
if (options.bare) {
|
||||
const bare = options.bare;
|
||||
if (!/[/\\:]/.test(trimmed)) {
|
||||
for (const key of keyVariants(trimmed)) refs.push({ category: bare, key });
|
||||
} else if (options.basename) {
|
||||
const last = trimmed.slice(Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')) + 1);
|
||||
for (const key of keyVariants(last)) refs.push({ category: bare, key });
|
||||
}
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* The candidates of `category` that any of `values` names, compared without
|
||||
* regard to case: a local volume on Windows or macOS serves `A.jpg` for `a.jpg`,
|
||||
* so an object is kept when a value names it in either spelling. The helper a
|
||||
* source builds `referenced()` from, so no source parses with a regex of its own.
|
||||
*/
|
||||
export function referencedAmong(
|
||||
category: ServedCategory,
|
||||
candidates: readonly string[],
|
||||
values: Iterable<string | null | undefined>,
|
||||
options: StorageKeyOptions = {},
|
||||
): Set<string> {
|
||||
const named = new Set<string>();
|
||||
for (const value of values) {
|
||||
for (const ref of storageKeysIn(value, options)) {
|
||||
if (ref.category === category) named.add(ref.key.toLowerCase());
|
||||
}
|
||||
}
|
||||
return new Set(candidates.filter((key) => named.has(key.toLowerCase())));
|
||||
}
|
||||
|
||||
function stripSuffix(value: string): string {
|
||||
const cut = value.search(/[?#]/);
|
||||
return cut === -1 ? value : value.slice(0, cut);
|
||||
}
|
||||
|
||||
/** The raw key, decoded, and cut at a blank; never an empty key or one that climbs out of its category. */
|
||||
function keyVariants(raw: string): string[] {
|
||||
const forms = new Set<string>();
|
||||
const add = (key: string) => {
|
||||
const clean = key.trim().replace(/^\/+/, '');
|
||||
if (clean === '' || clean.split('/').some((segment) => segment === '..' || segment === '.')) return;
|
||||
forms.add(clean);
|
||||
};
|
||||
const whole = stripSuffix(raw);
|
||||
for (const form of [whole, decoded(whole)]) {
|
||||
add(form);
|
||||
add(form.trim().split(/\s/)[0]!);
|
||||
}
|
||||
return [...forms];
|
||||
}
|
||||
|
||||
function decoded(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,17 @@
|
||||
import { AppSettings } from '../../db/entities/AppSettings.entity';
|
||||
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
|
||||
import { AppConfigModule } from '../app-config/app-config.module';
|
||||
import { AuditModule } from '../audit/audit.module';
|
||||
import { DatabaseLifecycleModule } from '../database/database-lifecycle.module';
|
||||
import { SchedulingModule } from '../scheduling/scheduling.module';
|
||||
import { StorageAdminController } from './storage-admin.controller';
|
||||
import { StorageAdminService } from './storage-admin.service';
|
||||
import { StorageEventsService } from './storage-events.service';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
|
||||
import { StorageReclaimController } from './storage-reclaim.controller';
|
||||
import { StorageReclaimJob } from './storage-reclaim.job';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { StorageRegistryService } from './storage-registry.service';
|
||||
import { StorageStatsService } from './storage-stats.service';
|
||||
import { StorageUsageScanJob } from './storage-usage-scan.job';
|
||||
@@ -27,11 +33,20 @@ import { Module } from '@nestjs/common';
|
||||
* may cache drivers or trigger reloads. MikroOrmModule.forFeature registers
|
||||
* AppSettingsRepository for StorageRegistryService/StorageAdminService/
|
||||
* StorageStatsService's @InjectRepository (Plan 3i, storage task — SR/SA/SS's
|
||||
* 14 call sites, all on app_settings).
|
||||
* 14 call sites, all on app_settings). The reclaim pass adds its marks table
|
||||
* and follows database reopens (DatabaseLifecycleModule); the domains whose
|
||||
* rows name stored objects register their reference sources with the exported
|
||||
* StorageReclaimRegistryService, so storage never imports them.
|
||||
*/
|
||||
@Module({
|
||||
imports: [AppConfigModule, AuditModule, SchedulingModule, MikroOrmModule.forFeature([AppSettings])],
|
||||
controllers: [StorageAdminController],
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
AuditModule,
|
||||
DatabaseLifecycleModule,
|
||||
SchedulingModule,
|
||||
MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks]),
|
||||
],
|
||||
controllers: [StorageAdminController, StorageReclaimController],
|
||||
providers: [
|
||||
StorageRegistryService,
|
||||
StorageService,
|
||||
@@ -40,7 +55,10 @@ import { Module } from '@nestjs/common';
|
||||
StorageJobsService,
|
||||
StorageStatsService,
|
||||
StorageUsageScanJob,
|
||||
StorageReclaimRegistryService,
|
||||
StorageReclaimService,
|
||||
StorageReclaimJob,
|
||||
],
|
||||
exports: [StorageService, StorageEventsService],
|
||||
exports: [StorageService, StorageEventsService, StorageReclaimRegistryService],
|
||||
})
|
||||
export class StorageModule {}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Storage reclaim e2e: /api/admin/storage/reclaim through the real
|
||||
* JwtAuthGuard + AdminGuard + ManagedGuard against a migrated temp SQLite, the
|
||||
* Zod pipe on both bodies and the response contracts the handlers declare.
|
||||
* This graph holds StorageModule alone, so no domain registers a reference
|
||||
* source: every reclaimable category reports `sources_missing` and a pass
|
||||
* cannot delete anything, which is the point of the ledger. RCLE-001 to RCLE-007.
|
||||
*/
|
||||
import { db } from '../../src/db/database';
|
||||
import { AppSettings } from '../../src/db/entities/AppSettings.entity';
|
||||
import { AuditLog } from '../../src/db/entities/AuditLog.entity';
|
||||
import { ManagedGuard } from '../../src/nest/common/managed.guard';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
|
||||
import { StorageJobsService } from '../../src/nest/storage/storage-jobs.service';
|
||||
import { STORAGE_RECLAIMABLE_CATEGORIES } from '../../src/nest/storage/storage-reclaim.types';
|
||||
import { StorageModule } from '../../src/nest/storage/storage.module';
|
||||
import { countRows, deleteRows, findRow } from '../helpers/factories/rows';
|
||||
import { makeAdmin, makeUser } from '../helpers/factories/users';
|
||||
import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm';
|
||||
import { TestUnitOfWorkModule } from '../helpers/test-uow';
|
||||
import { sessionCookie } from './harness';
|
||||
import type { INestApplication } from '@nestjs/common';
|
||||
import { APP_GUARD } from '@nestjs/core';
|
||||
import { Test } from '@nestjs/testing';
|
||||
|
||||
import cookieParser from 'cookie-parser';
|
||||
import type { Server } from 'http';
|
||||
import request from 'supertest';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../src/db/database', async () => {
|
||||
const { createSnapshotTestDb, buildDbMock } = await import('../helpers/db-mock');
|
||||
return buildDbMock(createSnapshotTestDb());
|
||||
});
|
||||
vi.mock('../../src/nest/audit/audit-log.logger', () => ({
|
||||
LOG_LEVEL: 'error',
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
describe('Storage reclaim e2e (real auth + admin guard + managed guard + migrated temp SQLite)', () => {
|
||||
let orm: TestOrm;
|
||||
let app: INestApplication;
|
||||
let server: Server;
|
||||
let adminCookie: string;
|
||||
let userCookie: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
orm = await createTestOrm(db);
|
||||
await makeUser(orm, { id: 1, email: 'e2e-reclaim@example.test' });
|
||||
await makeAdmin(orm, { id: 2, email: 'e2e-reclaim-admin@example.test' });
|
||||
userCookie = sessionCookie(1);
|
||||
adminCookie = sessionCookie(2);
|
||||
const moduleRef = await Test.createTestingModule({
|
||||
imports: [await TestUnitOfWorkModule.forRoot(db), await createTestMikroOrmModule(db), StorageModule],
|
||||
providers: [{ provide: APP_GUARD, useClass: ManagedGuard }],
|
||||
}).compile();
|
||||
app = moduleRef.createNestApplication();
|
||||
app.use(cookieParser());
|
||||
app.useGlobalPipes(new ZodValidationPipe());
|
||||
app.useGlobalFilters(new TrekExceptionFilter());
|
||||
await app.init();
|
||||
server = app.getHttpServer();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } });
|
||||
await deleteRows(orm, AuditLog);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
await orm.close();
|
||||
});
|
||||
|
||||
it('RCLE-001: 401 without a session on both routes', async () => {
|
||||
expect((await request(server).put('/api/admin/storage/reclaim').send({ deleteEnabled: true })).status).toBe(401);
|
||||
expect((await request(server).post('/api/admin/storage/reclaim/run').send({})).status).toBe(401);
|
||||
});
|
||||
|
||||
it('RCLE-002: 403 for a non-admin, and nothing is stored', async () => {
|
||||
const res = await request(server)
|
||||
.put('/api/admin/storage/reclaim')
|
||||
.set('Cookie', userCookie)
|
||||
.send({ deleteEnabled: true });
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body).toEqual({ error: 'Admin access required' });
|
||||
expect(await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' })).toBeFalsy();
|
||||
});
|
||||
|
||||
it('RCLE-003: PUT stores the switch, answers the state, audits, and GET /api/admin/storage carries it', async () => {
|
||||
const res = await request(server)
|
||||
.put('/api/admin/storage/reclaim')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ deleteEnabled: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ deleteEnabled: true, lastRun: null });
|
||||
expect(await countRows(orm, AuditLog, { action: 'admin.storage_reclaim_update' })).toBe(1);
|
||||
const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie);
|
||||
expect(state.status).toBe(200);
|
||||
expect(state.body.reclaim).toEqual({ deleteEnabled: true, lastRun: null });
|
||||
});
|
||||
|
||||
it('RCLE-004: PUT refuses a body outside the contract', async () => {
|
||||
for (const body of [{}, { deleteEnabled: 'yes' }, { deleteEnabled: true, extra: 1 }]) {
|
||||
const res = await request(server).put('/api/admin/storage/reclaim').set('Cookie', adminCookie).send(body);
|
||||
expect(res.status).toBe(400);
|
||||
}
|
||||
const run = await request(server)
|
||||
.post('/api/admin/storage/reclaim/run')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ reportOnly: 'no' });
|
||||
expect(run.status).toBe(400);
|
||||
expect(await countRows(orm, AuditLog)).toBe(0);
|
||||
});
|
||||
|
||||
it('RCLE-005: a pass without registered sources skips every reclaimable category, persists and audits', async () => {
|
||||
const res = await request(server)
|
||||
.post('/api/admin/storage/reclaim/run')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ reportOnly: true });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.mode).toBe('report');
|
||||
for (const category of STORAGE_RECLAIMABLE_CATEGORIES) {
|
||||
expect(res.body.categories[category].skipped).toBe('sources_missing');
|
||||
}
|
||||
expect(res.body.categories.backups.skipped).toBe('not_reclaimed');
|
||||
const audit = await findRow(orm, AuditLog, { action: 'admin.storage_reclaim_run' });
|
||||
expect(JSON.parse(String(audit?.details))).toEqual({ mode: 'report', deleted: 0, bytes: 0 });
|
||||
const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie);
|
||||
expect(state.body.reclaim.lastRun).toEqual(res.body);
|
||||
});
|
||||
|
||||
it('RCLE-006: 409 while another storage job holds the slot', async () => {
|
||||
const jobs = app.get(StorageJobsService);
|
||||
expect(jobs.claimExclusive('a test job')).toBe(true);
|
||||
try {
|
||||
const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({});
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body).toEqual({ error: 'another storage job is running, one storage job at a time' });
|
||||
expect(await countRows(orm, AuditLog)).toBe(0);
|
||||
} finally {
|
||||
jobs.releaseExclusive('a test job');
|
||||
}
|
||||
});
|
||||
|
||||
it('RCLE-007: managed mode refuses the surface with the standard body', async () => {
|
||||
process.env.TREK_MANAGED = 'true';
|
||||
try {
|
||||
const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({});
|
||||
expect(res.status).toBe(403);
|
||||
expect(res.body).toEqual({
|
||||
error: 'This is configured by the operator of this instance.',
|
||||
code: 'MANAGED_FORBIDDEN',
|
||||
});
|
||||
} finally {
|
||||
delete process.env.TREK_MANAGED;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* RECLAIM-SRC: the full app registers exactly the reference sources the
|
||||
* reclaim ledger waits for, and each answers for every category that waits on
|
||||
* it. Registration cannot fail a boot (the partial e2e graphs lack most
|
||||
* owners), so this is the one place a domain that forgot to register, or a
|
||||
* source that stopped claiming a category its columns can point into, shows:
|
||||
* on a real install that category would be skipped as `sources_missing`
|
||||
* forever, or swept while a domain's rows were never asked.
|
||||
*/
|
||||
import { buildApp } from '../../src/bootstrap';
|
||||
import { db as testDb } from '../../src/db/database';
|
||||
import { StorageReclaimRegistryService } from '../../src/nest/storage/storage-reclaim-registry.service';
|
||||
import {
|
||||
RECLAIM_LEDGER,
|
||||
reclaimableCategories,
|
||||
requiredReclaimSources,
|
||||
} from '../../src/nest/storage/storage-reclaim.ledger';
|
||||
import type { INestApplication } from '@nestjs/common';
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../src/db/database', async () => {
|
||||
const { createSnapshotTestDb, buildDbMock } = await import('../helpers/db-mock');
|
||||
return buildDbMock(createSnapshotTestDb());
|
||||
});
|
||||
|
||||
describe('RECLAIM-SRC: the app registers what the reclaim ledger waits for', () => {
|
||||
let app: INestApplication;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
testDb.close();
|
||||
});
|
||||
|
||||
it('RECLAIM-SRC-001: the registered source ids are exactly the ones the ledger requires', () => {
|
||||
const registry = app.get(StorageReclaimRegistryService);
|
||||
expect([...registry.ids()].sort()).toEqual([...requiredReclaimSources()].sort());
|
||||
});
|
||||
|
||||
it('RECLAIM-SRC-002: every required source answers for the whole of each category waiting on it', () => {
|
||||
const sources = app.get(StorageReclaimRegistryService).sources();
|
||||
for (const category of reclaimableCategories()) {
|
||||
const rule = RECLAIM_LEDGER[category];
|
||||
if (!rule.reclaim) continue;
|
||||
for (const id of rule.requiredSources) {
|
||||
const answering = sources.filter(
|
||||
(source) =>
|
||||
source.id === id && source.claims.some((claim) => claim.category === category && claim.prefix === ''),
|
||||
);
|
||||
expect(answering.length, `${id} answers for ${category}`).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* StorageReclaimController over a stub service: what it forwards, what it
|
||||
* audits, and that a refusal of the service reaches the client unchanged.
|
||||
* RCLC-001 to RCLC-004.
|
||||
*/
|
||||
import type { AuditService } from '../../../../src/nest/audit/audit.service';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import type { StorageReclaimRunDto, StorageReclaimSettingsDto } from '../../../../src/nest/storage/storage-admin.dto';
|
||||
import { StorageReclaimController } from '../../../../src/nest/storage/storage-reclaim.controller';
|
||||
import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service';
|
||||
import { StorageModule } from '../../../../src/nest/storage/storage.module';
|
||||
import type { User } from '../../../../src/types';
|
||||
import { expectRegisteredController } from '../../../helpers/module-providers';
|
||||
import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared';
|
||||
|
||||
import type { Request } from 'express';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const user = { id: 7 } as User;
|
||||
const req = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } as unknown as Request;
|
||||
|
||||
function report(): StorageReclaimReport {
|
||||
const none = { objects: 0, bytes: 0 };
|
||||
const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null };
|
||||
return {
|
||||
startedAt: 1,
|
||||
finishedAt: 2,
|
||||
mode: 'delete',
|
||||
stopped: null,
|
||||
categories: {
|
||||
...(Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])) as StorageReclaimReport['categories']),
|
||||
covers: { ...row, objects: 9, deleted: { objects: 2, bytes: 40 } },
|
||||
places: { ...row, objects: 3, deleted: { objects: 1, bytes: 2 } },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeController(over: Partial<Record<keyof StorageReclaimService, unknown>> = {}) {
|
||||
const writeAudit = vi.fn();
|
||||
const service = {
|
||||
setDeleteEnabled: vi.fn(async (enabled: boolean) => ({ deleteEnabled: enabled, lastRun: null })),
|
||||
run: vi.fn(async () => report()),
|
||||
...over,
|
||||
} as unknown as StorageReclaimService;
|
||||
const controller = new StorageReclaimController(service, { writeAudit } as unknown as AuditService);
|
||||
return { controller, service, writeAudit };
|
||||
}
|
||||
|
||||
describe('StorageReclaimController', () => {
|
||||
it('RCLC-001: is registered in StorageModule', () => {
|
||||
expectRegisteredController(StorageModule, StorageReclaimController);
|
||||
});
|
||||
|
||||
it('RCLC-002: PUT stores the switch, answers the fresh state and audits the value', async () => {
|
||||
const { controller, service, writeAudit } = makeController();
|
||||
const body = { deleteEnabled: true } as StorageReclaimSettingsDto;
|
||||
expect(await controller.update(user, body, req)).toEqual({ deleteEnabled: true, lastRun: null });
|
||||
expect(service.setDeleteEnabled).toHaveBeenCalledWith(true);
|
||||
expect(writeAudit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ userId: 7, action: 'admin.storage_reclaim_update', details: { deleteEnabled: true } }),
|
||||
);
|
||||
});
|
||||
|
||||
it('RCLC-003: POST run forwards reportOnly, answers the report and audits the totals', async () => {
|
||||
const { controller, service, writeAudit } = makeController();
|
||||
const answer = await controller.run(user, { reportOnly: true } as StorageReclaimRunDto, req);
|
||||
expect(answer.categories.covers.deleted.objects).toBe(2);
|
||||
expect(service.run).toHaveBeenCalledWith({ reportOnly: true });
|
||||
await controller.run(user, {} as StorageReclaimRunDto, req);
|
||||
expect(service.run).toHaveBeenLastCalledWith({ reportOnly: false });
|
||||
expect(writeAudit).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: 'admin.storage_reclaim_run',
|
||||
details: { mode: 'delete', deleted: 3, bytes: 42 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('RCLC-004: a refused pass propagates as the DomainError and writes no audit row', async () => {
|
||||
const refusal = new DomainError(409, 'another storage job is running, one storage job at a time');
|
||||
const { controller, writeAudit } = makeController({ run: vi.fn().mockRejectedValue(refusal) });
|
||||
await expect(controller.run(user, {} as StorageReclaimRunDto, req)).rejects.toBe(refusal);
|
||||
expect(writeAudit).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* StorageReclaimJob: the nightly 04:05 pass. It hands the registrar the pass's
|
||||
* own promise, so the lease covers the whole run, skips a demo instance and a
|
||||
* busy slot quietly, and logs one line only when the pass had something to say.
|
||||
* RCLJ-001 to RCLJ-006.
|
||||
*/
|
||||
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import { logDebug, logError, logInfo } from '../../../../src/nest/audit/audit-log.logger';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service';
|
||||
import { StorageReclaimJob } from '../../../../src/nest/storage/storage-reclaim.job';
|
||||
import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service';
|
||||
import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared';
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('../../../../src/nest/audit/audit-log.logger')>()),
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
function registrarStub(enabled = true) {
|
||||
return {
|
||||
isEnabled: vi.fn(() => enabled),
|
||||
register: vi.fn((_name: string, _expression: string, _onTick: () => void | Promise<void>) => enabled),
|
||||
unregister: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function report(
|
||||
over: Partial<StorageReclaimReport['categories']> = {},
|
||||
stopped: StorageReclaimReport['stopped'] = null,
|
||||
) {
|
||||
const none = { objects: 0, bytes: 0 };
|
||||
const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null };
|
||||
return {
|
||||
startedAt: 1,
|
||||
finishedAt: 2,
|
||||
mode: 'delete',
|
||||
stopped,
|
||||
categories: { ...Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])), ...over },
|
||||
} as StorageReclaimReport;
|
||||
}
|
||||
|
||||
function makeJob(run: () => Promise<StorageReclaimReport>, opts: { enabled?: boolean; demo?: boolean } = {}) {
|
||||
const registrar = registrarStub(opts.enabled ?? true);
|
||||
const reclaim = { run: vi.fn(run) };
|
||||
const env = { isDemoMode: () => opts.demo === true };
|
||||
const job = new StorageReclaimJob(
|
||||
registrar as unknown as CronRegistrarService,
|
||||
reclaim as unknown as StorageReclaimService,
|
||||
env as unknown as RuntimeEnvService,
|
||||
);
|
||||
return { job, registrar, reclaim };
|
||||
}
|
||||
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
describe('StorageReclaimJob', () => {
|
||||
it('RCLJ-001: registers at 04:05 and hands the registrar the pass promise itself', async () => {
|
||||
let finish: (value: StorageReclaimReport) => void = () => undefined;
|
||||
const { job, registrar, reclaim } = makeJob(() => new Promise((resolve) => (finish = resolve)));
|
||||
job.onApplicationBootstrap();
|
||||
expect(registrar.register).toHaveBeenCalledWith('storage-reclaim', '5 4 * * *', expect.any(Function));
|
||||
const onTick = registrar.register.mock.calls[0]![2];
|
||||
const ticking = onTick();
|
||||
expect(ticking).toBeInstanceOf(Promise);
|
||||
let settled = false;
|
||||
void (ticking as Promise<void>).then(() => (settled = true));
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
finish(report());
|
||||
await ticking;
|
||||
expect(reclaim.run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('RCLJ-002: does not register under the test gate, and never sweeps at boot', () => {
|
||||
const { job, registrar, reclaim } = makeJob(async () => report(), { enabled: false });
|
||||
job.onApplicationBootstrap();
|
||||
expect(registrar.register).not.toHaveBeenCalled();
|
||||
expect(reclaim.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('RCLJ-003: a demo instance is skipped without a pass', async () => {
|
||||
const { job, reclaim } = makeJob(async () => report(), { demo: true });
|
||||
await job.tick();
|
||||
expect(reclaim.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('RCLJ-004: logs one line only when something was found, deleted, skipped or stopped', async () => {
|
||||
const quiet = makeJob(async () => report({ photos: { ...report().categories.photos, skipped: 'not_reclaimed' } }));
|
||||
await quiet.job.tick(new Date(5));
|
||||
expect(quiet.reclaim.run).toHaveBeenCalledWith({ now: new Date(5) });
|
||||
expect(logInfo).not.toHaveBeenCalled();
|
||||
|
||||
const busy = report(
|
||||
{
|
||||
covers: {
|
||||
objects: 10,
|
||||
unreferenced: { objects: 2, bytes: 30 },
|
||||
eligible: { objects: 1, bytes: 10 },
|
||||
deleted: { objects: 1, bytes: 10 },
|
||||
skipped: null,
|
||||
},
|
||||
avatars: { ...report().categories.avatars, skipped: 'sources_missing' },
|
||||
},
|
||||
'database_reopened',
|
||||
);
|
||||
await makeJob(async () => busy).job.tick();
|
||||
expect(logInfo).toHaveBeenCalledWith(
|
||||
'Storage reclaim: 2 unreferenced object(s); deleted 1 (10 bytes); skipped avatars (sources_missing); stopped: database_reopened',
|
||||
);
|
||||
});
|
||||
|
||||
it('RCLJ-005: a refusal (busy, demo) is a debug line, not an error', async () => {
|
||||
const { job } = makeJob(async () => {
|
||||
throw new DomainError(409, 'another storage job is running', { code: 'reclaim_busy' });
|
||||
});
|
||||
await expect(job.tick()).resolves.toBeUndefined();
|
||||
expect(logDebug).toHaveBeenCalledWith('Storage reclaim skipped: another storage job is running');
|
||||
expect(logError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('RCLJ-006: any other failure is contained in one error line', async () => {
|
||||
await makeJob(async () => {
|
||||
throw new Error('registry not initialized');
|
||||
}).job.tick();
|
||||
expect(logError).toHaveBeenCalledWith('Storage reclaim: registry not initialized');
|
||||
await makeJob(vi.fn<() => Promise<StorageReclaimReport>>().mockRejectedValue('plain')).job.tick();
|
||||
expect(logError).toHaveBeenCalledWith('Storage reclaim: plain');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* The reclaim ledger and the location check (#2924), plus the registry the
|
||||
* reference sources land in. RCLL-001 to RCLL-011.
|
||||
*/
|
||||
import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service';
|
||||
import {
|
||||
inReclaimScope,
|
||||
RECLAIM_LEDGER,
|
||||
reclaimableCategories,
|
||||
requiredReclaimSources,
|
||||
sharedLocationCategories,
|
||||
type CategoryAssignment,
|
||||
type ReclaimRule,
|
||||
} from '../../../../src/nest/storage/storage-reclaim.ledger';
|
||||
import {
|
||||
STORAGE_RECLAIMABLE_CATEGORIES,
|
||||
STORAGE_REFERENCE_SOURCE_IDS,
|
||||
type StorageReferenceSource,
|
||||
} from '../../../../src/nest/storage/storage-reclaim.types';
|
||||
import type { BackendSnapshot } from '../../../../src/nest/storage/storage-registry.service';
|
||||
import { CATEGORY_PREFIXES } from '../../../../src/nest/storage/storage-registry.service';
|
||||
import { SERVED_CATEGORIES, type ServedCategory } from '../../../../src/nest/storage/storage.types';
|
||||
|
||||
import path from 'node:path';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('../../../../src/nest/audit/audit-log.logger')>()),
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
const UPLOADS = path.resolve('/srv/trek/uploads');
|
||||
const BACKUPS = path.resolve('/srv/trek/backups');
|
||||
|
||||
function local(name: string, root: string): BackendSnapshot {
|
||||
return { name, type: 'local', source: 'settings', options: { root } };
|
||||
}
|
||||
|
||||
/** The default layout: every served category on uploads-local, backups on its own backend. */
|
||||
function defaults(overrides: Partial<Record<ServedCategory, CategoryAssignment>> = {}) {
|
||||
const map = new Map<ServedCategory, CategoryAssignment>();
|
||||
for (const category of SERVED_CATEGORIES) {
|
||||
map.set(category, { backendName: 'uploads-local', keyPrefix: CATEGORY_PREFIXES[category] });
|
||||
}
|
||||
map.set('backups', { backendName: 'backups-local', keyPrefix: '' });
|
||||
for (const [category, assignment] of Object.entries(overrides)) map.set(category as ServedCategory, assignment);
|
||||
return map;
|
||||
}
|
||||
|
||||
const BASE_BACKENDS = [local('uploads-local', UPLOADS), local('backups-local', BACKUPS)];
|
||||
|
||||
describe('RECLAIM_LEDGER', () => {
|
||||
it('RCLL-001: reclaims exactly the five content categories and never a cache, the backups or legacy photos', () => {
|
||||
expect(reclaimableCategories().sort()).toEqual([...STORAGE_RECLAIMABLE_CATEGORIES].sort());
|
||||
for (const category of ['photos', 'photos-google', 'photos-trek', 'backups'] as const) {
|
||||
const rule: ReclaimRule = RECLAIM_LEDGER[category];
|
||||
expect(rule.reclaim).toBe(false);
|
||||
if (!rule.reclaim) expect(rule.reason.length).toBeGreaterThan(10);
|
||||
}
|
||||
});
|
||||
|
||||
it('RCLL-002: every reclaimable category waits for all the free-form owners, files and avatars for their own too', () => {
|
||||
const freeForm = ['trips', 'collections', 'places', 'journey', 'photos'];
|
||||
for (const category of STORAGE_RECLAIMABLE_CATEGORIES) {
|
||||
const rule = RECLAIM_LEDGER[category];
|
||||
expect(rule.reclaim).toBe(true);
|
||||
if (rule.reclaim) expect(rule.requiredSources).toEqual(expect.arrayContaining(freeForm));
|
||||
}
|
||||
const files = RECLAIM_LEDGER.files;
|
||||
const avatars = RECLAIM_LEDGER.avatars;
|
||||
expect(files.reclaim && files.requiredSources).toContain('files');
|
||||
expect(avatars.reclaim && avatars.requiredSources).toContain('auth');
|
||||
expect([...requiredReclaimSources()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort());
|
||||
});
|
||||
|
||||
it('RCLL-003: only root-level keys are in scope', () => {
|
||||
expect(inReclaimScope('a.jpg')).toBe(true);
|
||||
expect(inReclaimScope('assistant/a.pdf')).toBe(false);
|
||||
expect(inReclaimScope('thumbs/x.jpg')).toBe(false);
|
||||
expect(inReclaimScope('')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sharedLocationCategories', () => {
|
||||
it('RCLL-004: the default layout shares only the legacy photos folder, which holds both caches', () => {
|
||||
expect(sharedLocationCategories(defaults(), BASE_BACKENDS)).toEqual(new Set(['photos']));
|
||||
});
|
||||
|
||||
it('RCLL-005: TREK_PLACE_PHOTO_DIR inside the uploads folder overlaps the category it lands in', () => {
|
||||
const backends = [...BASE_BACKENDS, local('place-photos-local', path.join(UPLOADS, 'places'))];
|
||||
const shared = sharedLocationCategories(
|
||||
defaults({ 'photos-google': { backendName: 'place-photos-local', keyPrefix: '' } }),
|
||||
backends,
|
||||
);
|
||||
expect(shared.has('places')).toBe(true);
|
||||
expect(shared.has('photos-google')).toBe(true);
|
||||
expect(shared.has('covers')).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLL-006: backups on the uploads backend sit at its root and contain everything, yet only backups is excluded', () => {
|
||||
const shared = sharedLocationCategories(defaults({ backups: { backendName: 'uploads-local', keyPrefix: '' } }), [
|
||||
local('uploads-local', UPLOADS),
|
||||
]);
|
||||
expect(shared.has('backups')).toBe(true);
|
||||
expect(shared.has('covers')).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLL-007: two backends of other names on one folder overlap, compared without regard to case', () => {
|
||||
const backends = [...BASE_BACKENDS, local('again', UPLOADS.toUpperCase() + path.sep)];
|
||||
const shared = sharedLocationCategories(
|
||||
defaults({ avatars: { backendName: 'again', keyPrefix: 'covers/' } }),
|
||||
backends,
|
||||
);
|
||||
expect(shared.has('covers')).toBe(true);
|
||||
expect(shared.has('avatars')).toBe(true);
|
||||
expect(shared.has('files')).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLL-008: two s3 backends on one bucket and key prefix overlap, a different key prefix does not', () => {
|
||||
const s3 = (name: string, keyPrefix: string): BackendSnapshot => ({
|
||||
name,
|
||||
type: 's3',
|
||||
source: 'settings',
|
||||
options: { endpoint: 'https://s3.example.com/', bucket: 'trek', keyPrefix },
|
||||
});
|
||||
const backends = [...BASE_BACKENDS, s3('a', '/prod/'), s3('b', 'prod'), s3('c', 'dev'), s3('d', '')];
|
||||
const shared = sharedLocationCategories(
|
||||
defaults({
|
||||
covers: { backendName: 'a', keyPrefix: 'covers/' },
|
||||
avatars: { backendName: 'b', keyPrefix: 'covers/' },
|
||||
places: { backendName: 'c', keyPrefix: 'places/' },
|
||||
files: { backendName: 'd', keyPrefix: 'files/' },
|
||||
}),
|
||||
backends,
|
||||
);
|
||||
expect(shared.has('covers')).toBe(true);
|
||||
expect(shared.has('avatars')).toBe(true);
|
||||
expect(shared.has('places')).toBe(false);
|
||||
expect(shared.has('files')).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLL-009: a mirror counts its primary and every replica; a replica inside another category excludes it', () => {
|
||||
const backends: BackendSnapshot[] = [
|
||||
...BASE_BACKENDS,
|
||||
local('primary', path.resolve('/mnt/primary')),
|
||||
local('replica', path.join(UPLOADS, 'journey')),
|
||||
{ name: 'm', type: 'mirror', source: 'settings', options: { primary: 'primary', replicas: ['replica'] } },
|
||||
];
|
||||
const shared = sharedLocationCategories(defaults({ covers: { backendName: 'm', keyPrefix: 'covers/' } }), backends);
|
||||
// The replica's copy of covers lands at uploads/journey/covers/, inside journey's location.
|
||||
expect(shared.has('journey')).toBe(true);
|
||||
expect(shared.has('covers')).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLL-010: a location that cannot be resolved fails closed, for itself and for every other category', () => {
|
||||
const backends: BackendSnapshot[] = [
|
||||
...BASE_BACKENDS,
|
||||
{ name: 'odd', type: 'local', source: 'settings', options: { root: '' } },
|
||||
{ name: 'odd-s3', type: 's3', source: 'settings', options: { bucket: 'x' } },
|
||||
{ name: 'odd-mirror', type: 'mirror', source: 'settings', options: { primary: 'gone', replicas: 'x' } },
|
||||
];
|
||||
expect(
|
||||
sharedLocationCategories(defaults({ covers: { backendName: 'odd', keyPrefix: 'covers/' } }), backends).has(
|
||||
'files',
|
||||
),
|
||||
).toBe(true);
|
||||
for (const backendName of ['odd-s3', 'odd-mirror', 'missing']) {
|
||||
const shared = sharedLocationCategories(defaults({ covers: { backendName, keyPrefix: 'covers/' } }), backends);
|
||||
expect(shared.has('covers')).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('StorageReclaimRegistryService', () => {
|
||||
const source = (id: StorageReferenceSource['id']): StorageReferenceSource => ({
|
||||
id,
|
||||
claims: [{ category: 'covers', prefix: '' }],
|
||||
referenced: async () => new Set<string>(),
|
||||
});
|
||||
|
||||
it('RCLL-011: registers once per instance, keeps two sources of one id, and answers the ids', () => {
|
||||
const registry = new StorageReclaimRegistryService();
|
||||
const trips = source('trips');
|
||||
registry.register(trips);
|
||||
registry.register(trips);
|
||||
registry.register(source('trips'));
|
||||
registry.register(source('auth'));
|
||||
expect(registry.sources()).toHaveLength(3);
|
||||
expect(registry.ids()).toEqual(new Set(['trips', 'auth']));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,465 @@
|
||||
/**
|
||||
* The storage reclaim pass (#2924) over a real StorageModule: a real registry
|
||||
* and LocalDriver on temp folders, the marks table in the snapshot db, and
|
||||
* fake reference sources standing in for the owner domains. The pass marks
|
||||
* first, deletes a week later only with the switch on and only after asking
|
||||
* every source about the single key again; every brake skips a category and
|
||||
* starts its marks over. RCLS-001 to RCLS-021.
|
||||
*/
|
||||
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
|
||||
import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity';
|
||||
import { dbNow } from '../../../../src/db/types';
|
||||
import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import { logError } from '../../../../src/nest/audit/audit-log.logger';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service';
|
||||
import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service';
|
||||
import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service';
|
||||
import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service';
|
||||
import { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service';
|
||||
import {
|
||||
STORAGE_RECLAIMABLE_CATEGORIES,
|
||||
STORAGE_REFERENCE_SOURCE_IDS,
|
||||
type StorageReferenceClaim,
|
||||
type StorageReferenceSource,
|
||||
type StorageReferenceSourceId,
|
||||
} from '../../../../src/nest/storage/storage-reclaim.types';
|
||||
import { StorageModule } from '../../../../src/nest/storage/storage.module';
|
||||
import { StorageService } from '../../../../src/nest/storage/storage.service';
|
||||
import type { ServedCategory } from '../../../../src/nest/storage/storage.types';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { deleteRows } from '../../../helpers/factories/rows';
|
||||
import { readAppSetting, setAppSetting } from '../../../helpers/factories/settings';
|
||||
import { createTestModule, type TestModule } from '../../../helpers/test-module';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('../../../../src/nest/audit/audit-log.logger')>()),
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
const DAY = 24 * 60 * 60 * 1000;
|
||||
const db = createSnapshotTestDb();
|
||||
let t: TestModule;
|
||||
let orm: TestOrm;
|
||||
let uploadsRoot: string;
|
||||
let backupsRoot: string;
|
||||
let reclaim: StorageReclaimService;
|
||||
let storage: StorageService;
|
||||
let references: StorageReclaimRegistryService;
|
||||
let reopen: () => Promise<void>;
|
||||
/** What the fake sources still name, per category. */
|
||||
let named: Map<ServedCategory, Set<string>>;
|
||||
/** Called on every question a source is asked, before it answers. */
|
||||
let onAsk: ((category: ServedCategory, keys: readonly string[]) => void | Promise<void>) | null;
|
||||
|
||||
function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource {
|
||||
const categories: readonly ServedCategory[] =
|
||||
id === 'auth' ? ['avatars'] : id === 'files' ? ['files'] : STORAGE_RECLAIMABLE_CATEGORIES;
|
||||
return {
|
||||
id,
|
||||
claims: categories.map((category): StorageReferenceClaim => ({ category, prefix: '' })),
|
||||
async referenced(category, keys) {
|
||||
await onAsk?.(category, keys);
|
||||
const set = named.get(category) ?? new Set<string>();
|
||||
return new Set(keys.filter((key) => set.has(key)));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function backends(extra: unknown[] = []): string {
|
||||
return JSON.stringify([
|
||||
{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } },
|
||||
{ name: 'backups-local', type: 'local', options: { root: backupsRoot } },
|
||||
...extra,
|
||||
]);
|
||||
}
|
||||
|
||||
async function put(category: ServedCategory, key: string, body = '12345'): Promise<void> {
|
||||
await storage.put(category, key, Readable.from(body));
|
||||
}
|
||||
|
||||
function objectPath(category: string, key: string): string {
|
||||
return path.join(uploadsRoot, category, key);
|
||||
}
|
||||
|
||||
/** The orphans plus `kept` named covers, so the orphans stay under the brake's fifth. */
|
||||
async function seedCovers(orphans: string[], kept = 4): Promise<void> {
|
||||
const keptKeys = Array.from({ length: kept }, (_unused, i) => `kept-${i}.jpg`);
|
||||
await Promise.all([...orphans, ...keptKeys].map((key) => put('covers', key)));
|
||||
named.set('covers', new Set(keptKeys));
|
||||
}
|
||||
|
||||
async function marksOf(category: string): Promise<Array<{ key: string; first_seen_at: string }>> {
|
||||
const rows = await orm.repo(StorageReclaimMarks).marksOf(category);
|
||||
return rows
|
||||
.map((row) => ({ key: row.key, first_seen_at: row.first_seen_at }))
|
||||
.sort((a, b) => a.key.localeCompare(b.key));
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-uploads-'));
|
||||
backupsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-backups-'));
|
||||
orm = await createTestOrm(db);
|
||||
await setAppSetting(orm, 'storage.backends', backends());
|
||||
t = await createTestModule({
|
||||
db,
|
||||
imports: [StorageModule],
|
||||
init: true,
|
||||
overrides: [
|
||||
{
|
||||
provide: DatabaseLifecycle,
|
||||
useValue: {
|
||||
onReopened: (listener: () => Promise<void>) => {
|
||||
reopen = listener;
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
reclaim = t.get(StorageReclaimService);
|
||||
storage = t.get(StorageService);
|
||||
references = t.get(StorageReclaimRegistryService);
|
||||
for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.mocked(logError).mockClear();
|
||||
for (const entry of fs.readdirSync(uploadsRoot)) {
|
||||
if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true });
|
||||
}
|
||||
await deleteRows(orm, StorageReclaimMarks);
|
||||
await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } });
|
||||
named = new Map();
|
||||
onAsk = null;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
await orm.close();
|
||||
db.close();
|
||||
fs.rmSync(uploadsRoot, { recursive: true, force: true });
|
||||
fs.rmSync(backupsRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const T0 = new Date(Date.now() + 2 * DAY);
|
||||
const WEEK_LATER = new Date(T0.getTime() + 8 * DAY);
|
||||
|
||||
describe('StorageReclaimService: mark, then sweep', () => {
|
||||
it('RCLS-001: a first pass only reports and marks; the report is persisted and the state carries it', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.mode).toBe('report');
|
||||
expect(report.stopped).toBeNull();
|
||||
expect(report.categories.covers).toEqual({
|
||||
objects: 5,
|
||||
unreferenced: { objects: 1, bytes: 5 },
|
||||
eligible: { objects: 0, bytes: 0 },
|
||||
deleted: { objects: 0, bytes: 0 },
|
||||
skipped: null,
|
||||
});
|
||||
expect(report.categories.files).toMatchObject({ objects: 0, skipped: null });
|
||||
for (const category of ['photos', 'photos-google', 'photos-trek', 'backups'] as const) {
|
||||
expect(report.categories[category].skipped).toBe('not_reclaimed');
|
||||
}
|
||||
expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]);
|
||||
expect(await reclaim.state()).toEqual({ deleteEnabled: false, lastRun: report });
|
||||
expect((await t.get(StorageAdminService).state()).reclaim).toEqual({ deleteEnabled: false, lastRun: report });
|
||||
});
|
||||
|
||||
it('RCLS-002: with deletion on, a mark a week old goes through the facade and takes its mark along', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
const first = await reclaim.run({ now: T0 });
|
||||
expect(first.mode).toBe('delete');
|
||||
expect(first.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
|
||||
const second = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(second.categories.covers.eligible).toEqual({ objects: 1, bytes: 5 });
|
||||
expect(second.categories.covers.deleted).toEqual({ objects: 1, bytes: 5 });
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(false);
|
||||
expect(fs.existsSync(objectPath('covers', 'kept-0.jpg'))).toBe(true);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-003: a report-only pass deletes nothing even with deletion on', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
await reclaim.run({ now: T0 });
|
||||
const report = await reclaim.run({ now: WEEK_LATER, reportOnly: true });
|
||||
expect(report.mode).toBe('report');
|
||||
expect(report.categories.covers.eligible.objects).toBe(1);
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
});
|
||||
|
||||
it('RCLS-004: an object named again loses its mark, and its week starts over once it is unnamed', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
await reclaim.run({ now: T0 });
|
||||
named.get('covers')!.add('orphan.jpg');
|
||||
const renamed = await reclaim.run({ now: new Date(T0.getTime() + DAY) });
|
||||
expect(renamed.categories.covers.unreferenced.objects).toBe(0);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
|
||||
named.get('covers')!.delete('orphan.jpg');
|
||||
const again = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(again.categories.covers.eligible.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(WEEK_LATER) }]);
|
||||
});
|
||||
|
||||
it('RCLS-005: the single-key re-check right before the delete keeps an object named in between', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
await reclaim.run({ now: T0 });
|
||||
onAsk = (category, keys) => {
|
||||
if (category === 'covers' && keys.length === 1) named.get('covers')!.add(keys[0]!);
|
||||
};
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers.eligible.objects).toBe(1);
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-006: young objects, a missing modification time and nested subtrees are never judged', async () => {
|
||||
await put('covers', 'young.jpg');
|
||||
await put('covers', 'zero.jpg');
|
||||
await put('files', 'assistant/attached.pdf');
|
||||
await put('journey', 'thumbs/abcdef.jpg');
|
||||
const now = T0;
|
||||
fs.utimesSync(
|
||||
objectPath('covers', 'young.jpg'),
|
||||
new Date(now.getTime() - 60_000),
|
||||
new Date(now.getTime() - 60_000),
|
||||
);
|
||||
fs.utimesSync(objectPath('covers', 'zero.jpg'), 0, 0);
|
||||
const report = await reclaim.run({ now });
|
||||
expect(report.categories.covers).toMatchObject({ objects: 2, unreferenced: { objects: 0, bytes: 0 } });
|
||||
expect(report.categories.files.objects).toBe(0);
|
||||
expect(report.categories.journey.objects).toBe(0);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StorageReclaimService: brakes', () => {
|
||||
it('RCLS-007: more than a fifth unreferenced skips the category, deletes nothing and clears its marks', async () => {
|
||||
await seedCovers(['o1.jpg', 'o2.jpg'], 1);
|
||||
await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'o1.jpg', size: 5 }], dbNow(new Date(0)));
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers).toEqual({
|
||||
objects: 3,
|
||||
unreferenced: { objects: 2, bytes: 10 },
|
||||
eligible: { objects: 0, bytes: 0 },
|
||||
deleted: { objects: 0, bytes: 0 },
|
||||
skipped: 'too_many_unreferenced',
|
||||
});
|
||||
expect(fs.existsSync(objectPath('covers', 'o1.jpg'))).toBe(true);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-008: a category whose required source is missing, or answers for a part only, is skipped', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
const all = [...references.sources()];
|
||||
const withoutAuth = all.filter((source) => source.id !== 'auth');
|
||||
vi.spyOn(references, 'sources').mockReturnValue(withoutAuth);
|
||||
let report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.avatars.skipped).toBe('sources_missing');
|
||||
expect(report.categories.covers.skipped).toBeNull();
|
||||
|
||||
const partialTrips: StorageReferenceSource = {
|
||||
id: 'trips',
|
||||
claims: [{ category: 'covers', prefix: 'sub/' }],
|
||||
referenced: async () => new Set<string>(),
|
||||
};
|
||||
vi.spyOn(references, 'sources').mockReturnValue([...all.filter((source) => source.id !== 'trips'), partialTrips]);
|
||||
report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.covers.skipped).toBe('sources_missing');
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-009: a source answering for a subtree only is not asked about root keys', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
const subtree = vi.fn(async () => new Set<string>());
|
||||
vi.spyOn(references, 'sources').mockReturnValue([
|
||||
...references.sources(),
|
||||
{ id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], referenced: subtree },
|
||||
]);
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.covers.unreferenced.objects).toBe(1);
|
||||
expect(subtree).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('RCLS-010: a category whose location holds another category is skipped', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
const nestedRoot = path.join(uploadsRoot, 'covers');
|
||||
await setAppSetting(
|
||||
orm,
|
||||
'storage.backends',
|
||||
backends([{ name: 'nested', type: 'local', options: { root: nestedRoot } }]),
|
||||
);
|
||||
await setAppSetting(orm, 'storage.categories', JSON.stringify({ 'photos-trek': 'nested' }));
|
||||
await storage.reloadConfig();
|
||||
try {
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.covers.skipped).toBe('shared_location');
|
||||
expect(report.categories.files.skipped).toBeNull();
|
||||
} finally {
|
||||
await setAppSetting(orm, 'storage.backends', backends());
|
||||
await deleteRows(orm, AppSettings, { key: 'storage.categories' });
|
||||
await storage.reloadConfig();
|
||||
}
|
||||
});
|
||||
|
||||
it('RCLS-011: a listing that fails skips the category and clears its marks', async () => {
|
||||
await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'x.jpg', size: 1 }], dbNow(new Date(0)));
|
||||
vi.spyOn(storage, 'list').mockImplementation(() => ({
|
||||
[Symbol.asyncIterator]: () => ({ next: () => Promise.reject(new Error('backend down')) }),
|
||||
}));
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.covers.skipped).toBe('list_failed');
|
||||
expect(report.categories.files.skipped).toBe('list_failed');
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('backend down'));
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-012: a source that fails skips the category, never reads it as unreferenced', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
onAsk = (category) => {
|
||||
if (category === 'covers') throw new Error('trips table locked');
|
||||
};
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'source_failed' });
|
||||
expect(report.categories.places.skipped).toBeNull();
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('trips table locked'));
|
||||
});
|
||||
|
||||
it('RCLS-013: a pass refuses on a demo instance and while another storage job holds the slot', async () => {
|
||||
const demo = vi.spyOn(t.get(RuntimeEnvService), 'isDemoMode').mockReturnValue(true);
|
||||
await expect(reclaim.run()).rejects.toMatchObject({ code: 'reclaim_demo' });
|
||||
demo.mockRestore();
|
||||
|
||||
const jobs = t.get(StorageJobsService);
|
||||
expect(jobs.claimExclusive('a test')).toBe(true);
|
||||
try {
|
||||
const refusal = await reclaim.run().catch((err: unknown) => err);
|
||||
expect(refusal).toBeInstanceOf(DomainError);
|
||||
expect(refusal).toMatchObject({ code: 'reclaim_busy' });
|
||||
expect((refusal as DomainError).getStatus()).toBe(409);
|
||||
} finally {
|
||||
jobs.releaseExclusive('a test');
|
||||
}
|
||||
});
|
||||
|
||||
it('RCLS-014: while a pass holds the slot, a migration is refused with its name, and the slot is free after', async () => {
|
||||
const jobs = t.get(StorageJobsService);
|
||||
let refusal: unknown = null;
|
||||
onAsk = async (category) => {
|
||||
if (category !== 'covers' || refusal) return;
|
||||
refusal = await jobs.startMigration('covers', 'backups-local').catch((err: unknown) => err);
|
||||
};
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await reclaim.run({ now: T0 });
|
||||
expect(String(refusal)).toContain('a storage reclaim is running, one storage job at a time');
|
||||
expect(jobs.claimExclusive('after')).toBe(true);
|
||||
jobs.releaseExclusive('after');
|
||||
});
|
||||
|
||||
it('RCLS-015: a database reopened mid-pass stops it before anything else is judged', async () => {
|
||||
await seedCovers(['orphan.jpg']);
|
||||
await put('places', 'p.jpg');
|
||||
let reopened = false;
|
||||
onAsk = async (category) => {
|
||||
if (category === 'covers' && !reopened) {
|
||||
reopened = true;
|
||||
await reopen();
|
||||
}
|
||||
};
|
||||
const report = await reclaim.run({ now: T0 });
|
||||
expect(report.stopped).toBe('database_reopened');
|
||||
expect(report.categories.covers.unreferenced.objects).toBe(0);
|
||||
expect(report.categories.places.objects).toBe(0);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
expect((await reclaim.state()).lastRun?.stopped).toBe('database_reopened');
|
||||
});
|
||||
});
|
||||
|
||||
describe('StorageReclaimService: the delete itself', () => {
|
||||
async function markedWeekOld(orphans: string[], kept = 4): Promise<void> {
|
||||
await seedCovers(orphans, kept);
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
await reclaim.run({ now: T0 });
|
||||
}
|
||||
|
||||
it('RCLS-016: a re-check that fails keeps the object', async () => {
|
||||
await markedWeekOld(['orphan.jpg']);
|
||||
onAsk = (_category, keys) => {
|
||||
if (keys.length === 1) throw new Error('lookup timed out');
|
||||
};
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('lookup timed out'));
|
||||
});
|
||||
|
||||
it('RCLS-017: an object gone or written again since it was judged is left and its mark dropped', async () => {
|
||||
await markedWeekOld(['gone.jpg', 'fresh.jpg'], 8);
|
||||
onAsk = (_category, keys) => {
|
||||
if (keys.length !== 1) return;
|
||||
if (keys[0] === 'gone.jpg') fs.rmSync(objectPath('covers', 'gone.jpg'), { force: true });
|
||||
if (keys[0] === 'fresh.jpg') fs.utimesSync(objectPath('covers', 'fresh.jpg'), WEEK_LATER, WEEK_LATER);
|
||||
};
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers.eligible.objects).toBe(2);
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'fresh.jpg'))).toBe(true);
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLS-018: a delete that fails keeps the mark for the next pass', async () => {
|
||||
await markedWeekOld(['orphan.jpg']);
|
||||
vi.spyOn(storage, 'delete').mockRejectedValue(new Error('permission denied'));
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]);
|
||||
expect(logError).toHaveBeenCalledWith(expect.stringContaining('permission denied'));
|
||||
});
|
||||
|
||||
it('RCLS-019: an object the facade keeps on delete is not counted as reclaimed', async () => {
|
||||
await markedWeekOld(['orphan.jpg']);
|
||||
vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true);
|
||||
const report = await reclaim.run({ now: WEEK_LATER });
|
||||
expect(report.categories.covers.deleted.objects).toBe(0);
|
||||
expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StorageReclaimService: the switch and the stored report', () => {
|
||||
it('RCLS-020: the switch is stored as true or false and only the exact true deletes', async () => {
|
||||
expect((await reclaim.setDeleteEnabled(true)).deleteEnabled).toBe(true);
|
||||
expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBe('true');
|
||||
expect((await reclaim.setDeleteEnabled(false)).deleteEnabled).toBe(false);
|
||||
await setAppSetting(orm, 'storage.reclaim_delete', 'yes');
|
||||
expect((await reclaim.state()).deleteEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLS-021: a stored report that no longer parses reads as never run', async () => {
|
||||
await setAppSetting(orm, 'storage.reclaim_report', '{not json');
|
||||
expect((await reclaim.state()).lastRun).toBeNull();
|
||||
await setAppSetting(orm, 'storage.reclaim_report', JSON.stringify({ startedAt: 'yesterday' }));
|
||||
expect((await reclaim.state()).lastRun).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* The reference parser every reclaim source builds on (#2924): which objects a
|
||||
* stored value names, in every spelling the columns hold. A key reported too
|
||||
* many only keeps an object; one missed would delete it, so the cases lean on
|
||||
* the spellings the validators let through (absolute self-URLs, queries,
|
||||
* relative journey covers, legacy file names). RCLP-001 to RCLP-012.
|
||||
*/
|
||||
import {
|
||||
referencedAmong,
|
||||
storageKeysIn,
|
||||
uploadKeysIn,
|
||||
type StorageKeyOptions,
|
||||
type StorageKeyRef,
|
||||
} from '../../../../src/nest/storage/storage-reclaim.types';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const ref = (category: string, key: string): StorageKeyRef => ({ category, key });
|
||||
|
||||
describe('uploadKeysIn', () => {
|
||||
it('RCLP-001: finds the path in its plain, slash-less and absolute self-URL spellings', () => {
|
||||
expect(uploadKeysIn('/uploads/covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]);
|
||||
expect(uploadKeysIn('uploads/files/b.pdf')).toEqual([ref('files', 'b.pdf')]);
|
||||
expect(uploadKeysIn('https://trip.example.com/uploads/places/c.png')).toEqual([ref('places', 'c.png')]);
|
||||
expect(uploadKeysIn('HTTPS://HOST/Uploads/Covers/D.JPG')).toEqual([ref('covers', 'D.JPG')]);
|
||||
});
|
||||
|
||||
it('RCLP-002: drops a query or a fragment, decodes and trims the key', () => {
|
||||
expect(uploadKeysIn('/uploads/places/x.jpg?v=2')).toEqual([ref('places', 'x.jpg')]);
|
||||
expect(uploadKeysIn('/uploads/places/x.jpg#top')).toEqual([ref('places', 'x.jpg')]);
|
||||
expect(uploadKeysIn(' /uploads/covers/y.jpg ')).toEqual([ref('covers', 'y.jpg')]);
|
||||
expect(uploadKeysIn('/uploads/photos/my%20photo.jpg')).toEqual([
|
||||
ref('photos', 'my%20photo.jpg'),
|
||||
ref('photos', 'my photo.jpg'),
|
||||
ref('photos', 'my'),
|
||||
]);
|
||||
});
|
||||
|
||||
it('RCLP-003: finds every path in a text, each cut where the next begins or the text ends', () => {
|
||||
expect(uploadKeysIn('see /uploads/covers/a.jpg and "/uploads/places/b.jpg" too')).toEqual([
|
||||
ref('covers', 'a.jpg and'),
|
||||
ref('covers', 'a.jpg'),
|
||||
ref('places', 'b.jpg'),
|
||||
]);
|
||||
expect(uploadKeysIn('/uploads/covers/a.jpg/uploads/places/b.jpg')).toEqual([
|
||||
ref('covers', 'a.jpg'),
|
||||
ref('places', 'b.jpg'),
|
||||
]);
|
||||
});
|
||||
|
||||
it('RCLP-004: keeps a nested key whole and refuses empty or climbing keys', () => {
|
||||
expect(uploadKeysIn('/uploads/journey/thumbs/abc.jpg')).toEqual([ref('journey', 'thumbs/abc.jpg')]);
|
||||
expect(uploadKeysIn('/uploads/covers/')).toEqual([]);
|
||||
expect(uploadKeysIn('/uploads/covers/../avatars/a.jpg')).toEqual([]);
|
||||
expect(uploadKeysIn('/uploads/covers/%2E%2E/x')).toEqual([ref('covers', '%2E%2E/x')]);
|
||||
});
|
||||
|
||||
it('RCLP-005: names nothing for empty values, other hosts and other paths', () => {
|
||||
expect(uploadKeysIn(null)).toEqual([]);
|
||||
expect(uploadKeysIn(undefined)).toEqual([]);
|
||||
expect(uploadKeysIn('')).toEqual([]);
|
||||
expect(uploadKeysIn('https://images.unsplash.com/photo-123?w=800')).toEqual([]);
|
||||
expect(uploadKeysIn('/api/maps/place-photo/abc/bytes')).toEqual([]);
|
||||
expect(uploadKeysIn('/my-uploads/covers/a.jpg')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLP-006: a broken percent sequence keeps the raw key', () => {
|
||||
expect(uploadKeysIn('/uploads/covers/100%.jpg')).toEqual([ref('covers', '100%.jpg')]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('storageKeysIn', () => {
|
||||
it('RCLP-007: reads the relative <category>/<key> spelling of journey covers', () => {
|
||||
expect(storageKeysIn('covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]);
|
||||
expect(storageKeysIn('journey/b.jpg?x=1')).toEqual([ref('journey', 'b.jpg')]);
|
||||
expect(storageKeysIn('./journey/thumbs/c.jpg')).toEqual([ref('journey', 'thumbs/c.jpg')]);
|
||||
expect(storageKeysIn('notacategory/a.jpg')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLP-008: a bare name belongs to the column category, a URL or a path does not', () => {
|
||||
expect(storageKeysIn('a1b2.jpg', { bare: 'avatars' })).toEqual([ref('avatars', 'a1b2.jpg')]);
|
||||
expect(storageKeysIn('https://idp.example.com/pic.jpg', { bare: 'avatars' })).toEqual([]);
|
||||
expect(storageKeysIn('some/dir/x.pdf', { bare: 'files' })).toEqual([]);
|
||||
expect(storageKeysIn('a1b2.jpg')).toEqual([]);
|
||||
});
|
||||
|
||||
it('RCLP-009: with basename, a legacy path still names its last segment', () => {
|
||||
expect(storageKeysIn('files/x.pdf', { bare: 'files', basename: true })).toEqual([
|
||||
ref('files', 'x.pdf'),
|
||||
ref('files', 'x.pdf'),
|
||||
]);
|
||||
expect(storageKeysIn('/uploads/files/y.pdf', { bare: 'files', basename: true })).toEqual([
|
||||
ref('files', 'y.pdf'),
|
||||
ref('files', 'y.pdf'),
|
||||
]);
|
||||
expect(storageKeysIn('C:\\old\\uploads\\z.pdf', { bare: 'files', basename: true })).toEqual([
|
||||
ref('files', 'z.pdf'),
|
||||
]);
|
||||
});
|
||||
|
||||
it('RCLP-010: names nothing for a blank or a non-string', () => {
|
||||
expect(storageKeysIn(' ', { bare: 'files' })).toEqual([]);
|
||||
expect(storageKeysIn('?only=query', { bare: 'files' })).toEqual([]);
|
||||
expect(storageKeysIn(null)).toEqual([]);
|
||||
expect(storageKeysIn(42 as unknown as string)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('referencedAmong', () => {
|
||||
it('RCLP-011: answers the candidates of one category the values name, without regard to case', () => {
|
||||
const values = ['/uploads/covers/A.JPG', 'covers/b.jpg', '/uploads/places/c.jpg', null, 'https://elsewhere/x'];
|
||||
expect(referencedAmong('covers', ['a.jpg', 'b.jpg', 'c.jpg', 'd.jpg'], values)).toEqual(
|
||||
new Set(['a.jpg', 'b.jpg']),
|
||||
);
|
||||
expect(referencedAmong('places', ['c.jpg'], values)).toEqual(new Set(['c.jpg']));
|
||||
});
|
||||
|
||||
it('RCLP-012: passes the column options on, so a bare avatar name keeps its object', () => {
|
||||
const avatarColumn: StorageKeyOptions = { bare: 'avatars' };
|
||||
expect(referencedAmong('avatars', ['u1.png'], ['u1.png'], avatarColumn)).toEqual(new Set(['u1.png']));
|
||||
expect(
|
||||
referencedAmong('avatars', ['u1.png', 'u2.png'], ['u1.png', 'https://idp/u2.png'], { bare: 'avatars' }),
|
||||
).toEqual(new Set(['u1.png']));
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user