feat(storage): find and reclaim upload objects no row names any more

Domains whose rows name stored objects register a reference source with
StorageReclaimRegistryService; storage never imports them. A ledger in
storage decides per category whether it is reclaimed at all and which
sources must have answered first, and a category whose resolved location
overlaps another one is left alone.

A nightly pass at 04:05 lists the root level of files, journey, covers,
avatars and places, skips objects younger than a day or without a
modification time, asks every source in pages of 500 and marks the rest.
Only with deletion switched on, and only for marks a week old, it asks every
source about the single key again and deletes through StorageService. A
category is skipped and its marks start over when a source is missing or
fails, the listing fails, or more than a fifth of its objects look
unreferenced. A pass shares the one-job slot with backfills and migrations,
never runs on a demo instance and stops when the database is reopened.

Admins switch deletion and run a pass on demand under
/api/admin/storage/reclaim; both are audited, and GET /api/admin/storage
carries the switch and the last report. Deletion is off after install and
upgrade.
This commit is contained in:
Maurice
2026-11-01 13:37:53 +01:00
parent 29298dcfe5
commit fb2429ebb2
19 changed files with 2239 additions and 8 deletions
+2
View File
@@ -31,6 +31,8 @@ const ACTION_LABELS: Record<string, string> = {
'admin.storage_backfill': 'started a storage sync',
'admin.storage_backfill_cancel': 'cancelled a storage sync',
'admin.storage_stats_refresh': 'refreshed storage usage stats',
'admin.storage_reclaim_update': 'switched storage reclaim deletion',
'admin.storage_reclaim_run': 'ran a storage reclaim pass',
'admin.places_contribute_decision': 'decided on sharing place details with TREK Places',
'admin.places_contribute_withdraw': 'withdrew the place details this installation shared',
'immich.private_ip_configured': 'configured Immich with private IP',
+9 -1
View File
@@ -1,4 +1,10 @@
import { storageConfigPutSchema, storageMigrationRequestSchema, storageTestRequestSchema } from '@trek/shared';
import {
storageConfigPutSchema,
storageMigrationRequestSchema,
storageReclaimRunRequestSchema,
storageReclaimSettingsRequestSchema,
storageTestRequestSchema,
} from '@trek/shared';
import { createZodDto } from 'nestjs-zod';
@@ -12,3 +18,5 @@ import { createZodDto } from 'nestjs-zod';
export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {}
export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {}
export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {}
export class StorageReclaimSettingsDto extends createZodDto(storageReclaimSettingsRequestSchema) {}
export class StorageReclaimRunDto extends createZodDto(storageReclaimRunRequestSchema) {}
@@ -4,6 +4,7 @@ import { UnitOfWork } from '../database/unit-of-work';
import { StorageJobsService } from './storage-jobs.service';
import { getSeedConfigPath } from './storage-paths';
import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe';
import { readReclaimState } from './storage-reclaim.settings';
import { BACKENDS_KEY, CATEGORIES_KEY, VERSION_KEY, StorageRegistryService } from './storage-registry.service';
import {
assertNoMaskSentinels,
@@ -63,6 +64,7 @@ export class StorageAdminService {
health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) },
seedFilePresent: fs.existsSync(getSeedConfigPath()),
usage: await this.stats.readUsage(),
reclaim: await readReclaimState(this.appSettings),
backfills: this.jobs.statuses(),
migrations: this.jobs.migrationStatuses(),
version: await this.registry.currentConfigVersion(),
@@ -41,15 +41,38 @@ export class StorageJobsService {
private readonly jobs = new Map<string, ActiveJob>();
private readonly migrations = new Map<string, ActiveMigration>();
private ttlMs = 10 * 60_000;
/** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */
private exclusive: string | null = null;
/** One storage job at a time, across both backfills and category migrations. */
/** One storage job at a time, across backfills, category migrations and the reclaim pass. */
private anyJobRunning(): boolean {
return (
this.exclusive !== null ||
[...this.jobs.values()].some((j) => j.status.status === 'running') ||
[...this.migrations.values()].some((m) => m.status.status === 'running')
);
}
/** The refusal a start meets while the slot is taken, naming what holds it. */
private busyError(fallback: string): BackfillBusyError {
return new BackfillBusyError(this.exclusive ? `${this.exclusive} is running, one storage job at a time` : fallback);
}
/**
* Takes the one-job slot for a pass that runs in the caller (the reclaim
* pass): false while a backfill, a migration or another such pass runs.
* The caller releases it in a finally.
*/
claimExclusive(name: string): boolean {
if (this.anyJobRunning()) return false;
this.exclusive = name;
return true;
}
releaseExclusive(name: string): void {
if (this.exclusive === name) this.exclusive = null;
}
constructor(private readonly registry: StorageRegistryService) {}
/** Test-only factory for a short TTL, since a second constructor param would confuse Nest DI. */
@@ -74,7 +97,7 @@ export class StorageJobsService {
throw new BackfillTargetError(`'${mirrorName}' is not a mirror backend`);
}
if (this.anyJobRunning()) {
throw new BackfillBusyError('a sync is already running — one backfill at a time');
throw this.busyError('a sync is already running — one backfill at a time');
}
const prefixes = resolved.map((r) => r.keyPrefix);
@@ -156,7 +179,7 @@ export class StorageJobsService {
const target = this.registry.driverByName(to);
if (!target) throw new MigrationTargetError(`no backend named '${to}'`);
if (this.anyJobRunning()) {
throw new BackfillBusyError('a sync is already running — one storage job at a time');
throw this.busyError('a sync is already running — one storage job at a time');
}
// The DESTINATION prefix, computed for `to` before the flip — may differ
// from `current.keyPrefix` (audit #8: a mode-A '' source migrating onto a
@@ -0,0 +1,37 @@
import { logWarn } from '../audit/audit-log.logger';
import type { StorageReferenceSource, StorageReferenceSourceId } from './storage-reclaim.types';
import { Injectable } from '@nestjs/common';
/**
* Where the domains whose rows name stored objects register their reference
* source (`storage-reclaim.types.ts`), the StorageEventsService shape: an owner
* imports StorageModule, injects this and calls `register` from its own
* `onApplicationBootstrap`, so storage never imports the owner back.
*
* Registering never throws: the e2e suites boot partial module graphs, and a
* graph without a source is no error here. The reclaim pass checks the
* registered ids against its ledger and skips every category that lacks one,
* and a full-app test pins that the app registers exactly what the ledger asks.
*/
@Injectable()
export class StorageReclaimRegistryService {
private readonly registered: StorageReferenceSource[] = [];
register(source: StorageReferenceSource): void {
// A provider listed in two modules bootstraps twice: one entry is enough.
if (this.registered.includes(source)) return;
if (this.registered.some((existing) => existing.id === source.id)) {
// Two sources under one id are both asked; neither may hide the other.
logWarn(`[storage] a second reference source registered as '${source.id}', both are asked`);
}
this.registered.push(source);
}
sources(): readonly StorageReferenceSource[] {
return this.registered;
}
ids(): Set<StorageReferenceSourceId> {
return new Set(this.registered.map((source) => source.id));
}
}
@@ -0,0 +1,76 @@
import type { User } from '../../types';
import { AuditService } from '../audit/audit.service';
import { getClientIp } from '../audit/client-ip';
import { AdminGuard } from '../auth-core/admin.guard';
import { CurrentUser } from '../auth-core/current-user.decorator';
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
import { ManagedForbidden } from '../common/managed';
import { ResponseContract } from '../common/response-contract';
import { StorageReclaimRunDto, StorageReclaimSettingsDto } from './storage-admin.dto';
import { StorageReclaimService } from './storage-reclaim.service';
import { Body, Controller, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common';
import {
storageReclaimReportSchema,
storageReclaimStateSchema,
type StorageReclaimReport,
type StorageReclaimState,
} from '@trek/shared';
import type { Request } from 'express';
/**
* /api/admin/storage/reclaim: the admin's switch for the reclaim pass and a
* pass on demand. The state itself rides on GET /api/admin/storage. Deleting
* stays off until an admin turns it on here; a pass then still deletes only
* what has been unreferenced for a week (StorageReclaimService).
*/
@Controller('api/admin/storage/reclaim')
@UseGuards(JwtAuthGuard, AdminGuard)
@ManagedForbidden('storage backends and their contents are hoster-level configuration')
export class StorageReclaimController {
constructor(
private readonly reclaim: StorageReclaimService,
private readonly audit: AuditService,
) {}
@Put()
@ResponseContract(storageReclaimStateSchema)
async update(
@CurrentUser() user: User,
@Body() body: StorageReclaimSettingsDto,
@Req() req: Request,
): Promise<StorageReclaimState> {
const state = await this.reclaim.setDeleteEnabled(body.deleteEnabled);
await this.audit.writeAudit({
userId: user.id,
action: 'admin.storage_reclaim_update',
ip: getClientIp(req),
details: { deleteEnabled: body.deleteEnabled },
});
return state;
}
/** One pass now, honouring the switch unless `reportOnly`. 409 on a demo or while another storage job runs. */
@Post('run')
@HttpCode(200)
@ResponseContract(storageReclaimReportSchema)
async run(
@CurrentUser() user: User,
@Body() body: StorageReclaimRunDto,
@Req() req: Request,
): Promise<StorageReclaimReport> {
const report = await this.reclaim.run({ reportOnly: body.reportOnly === true });
const rows = Object.values(report.categories);
await this.audit.writeAudit({
userId: user.id,
action: 'admin.storage_reclaim_run',
ip: getClientIp(req),
details: {
mode: report.mode,
deleted: rows.reduce((sum, row) => sum + row.deleted.objects, 0),
bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0),
},
});
return report;
}
}
@@ -0,0 +1,62 @@
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { logDebug, logError, logInfo } from '../audit/audit-log.logger';
import { DomainError } from '../common/domain-error';
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
import { StorageReclaimService } from './storage-reclaim.service';
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
import type { StorageReclaimReport } from '@trek/shared';
/**
* The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00 and
* before the usage scan at 04:15, so the next usage figures already show what
* it freed. No pass at boot: a pass that may delete waits for its hour.
*/
@Injectable()
export class StorageReclaimJob implements OnApplicationBootstrap {
constructor(
private readonly registrar: CronRegistrarService,
private readonly reclaim: StorageReclaimService,
private readonly runtimeEnv: RuntimeEnvService,
) {}
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
this.registrar.register('storage-reclaim', '5 4 * * *', () => this.tick());
}
/**
* @txIndependent a sweep in passes: each category's marks and each delete
* stand on their own (StorageReclaimService.run).
*/
async tick(now: Date = new Date()): Promise<void> {
if (this.runtimeEnv.isDemoMode()) return;
try {
const report = await this.reclaim.run({ now });
const line = summary(report);
if (line) logInfo(`Storage reclaim: ${line}`);
} catch (err: unknown) {
if (err instanceof DomainError && err.getStatus() === 409) {
logDebug(`Storage reclaim skipped: ${err.message}`);
return;
}
logError(`Storage reclaim: ${err instanceof Error ? err.message : String(err)}`);
}
}
}
/** One line when the pass found, deleted or refused something; null when there was nothing to say. */
function summary(report: StorageReclaimReport): string | null {
const rows = Object.values(report.categories);
const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0);
const deleted = rows.reduce((sum, row) => sum + row.deleted.objects, 0);
const bytes = rows.reduce((sum, row) => sum + row.deleted.bytes, 0);
const braked = Object.entries(report.categories)
.filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed')
.map(([category, row]) => `${category} (${row.skipped})`);
const parts: string[] = [];
if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`);
if (deleted > 0) parts.push(`deleted ${deleted} (${bytes} bytes)`);
if (braked.length > 0) parts.push(`skipped ${braked.join(', ')}`);
if (report.stopped) parts.push(`stopped: ${report.stopped}`);
return parts.length > 0 ? parts.join('; ') : null;
}
@@ -0,0 +1,153 @@
import type { StorageReclaimableCategory, StorageReferenceSourceId } from './storage-reclaim.types';
import type { BackendSnapshot } from './storage-registry.service';
import type { ServedCategory } from './storage.types';
import path from 'node:path';
export type ReclaimRule =
| { readonly reclaim: false; readonly reason: string }
| { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] };
/** The owners of columns that hold a free-form upload path, which can point into any category. */
const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos'];
/**
* What the reclaim pass may do per category, and which reference sources must
* have registered before it may call one object of it unreferenced. A Record,
* so a new category does not compile until it is decided here (the
* KEPT_ON_DEMO precedent). A category missing one of its sources is skipped,
* never swept on a partial answer.
*/
export const RECLAIM_LEDGER: Readonly<Record<ServedCategory, ReclaimRule>> = {
// trip_files.filename (trashed rows included) holds the bare name; collab
// note files, chat images and doc-sync copies are trip_files rows too.
files: { reclaim: true, requiredSources: ['files', ...FREE_FORM_SOURCES] },
// trek_photos.file_path and thumbnail_path (posters), journeys.cover_image.
journey: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
// Trip, collection and journey covers; a trip copy and a journey share one object.
covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] },
// places.image_url, collection_places.image_url and the contribution outbox.
places: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
photos: {
reclaim: false,
reason:
'the legacy photo folder is served to every signed-in user by name, holds the place-photo caches in its ' +
'google/ and trek/ subtrees, and nothing writes to it any more',
},
'photos-google': { reclaim: false, reason: 'a cache the place-photo sweep keeps against its meta rows' },
'photos-trek': { reclaim: false, reason: 'a cache with a one-hour lifetime and a sweep of its own' },
backups: { reclaim: false, reason: 'no row names a backup; auto-backup rotates them by name' },
};
/** Every source id some category of the ledger waits for: what a full app must register. */
export function requiredReclaimSources(): Set<StorageReferenceSourceId> {
const ids = new Set<StorageReferenceSourceId>();
for (const rule of Object.values(RECLAIM_LEDGER)) {
if (rule.reclaim) for (const id of rule.requiredSources) ids.add(id);
}
return ids;
}
/** The categories the ledger lets a pass delete from. */
export function reclaimableCategories(): StorageReclaimableCategory[] {
return (Object.keys(RECLAIM_LEDGER) as ServedCategory[]).filter(
(category): category is StorageReclaimableCategory => RECLAIM_LEDGER[category].reclaim,
);
}
/**
* Only the root level of a category is swept: `files/assistant/` belongs to the
* assistant's retention, `journey/thumbs/` to the thumbnail sweep, and a nested
* key elsewhere was not written by TREK.
*/
export function inReclaimScope(key: string): boolean {
return key !== '' && !key.includes('/');
}
export interface CategoryAssignment {
backendName: string;
keyPrefix: string;
}
/**
* The categories whose objects may sit where another category's do, decided
* from the RESOLVED backends rather than their names: two backends can share a
* root or a bucket, `TREK_PLACE_PHOTO_DIR` can point inside the uploads folder,
* the legacy `photos/` folder holds the two photo caches, and `backups` on a
* shared backend is that backend's whole root. A category is excluded when its
* location, or a mirror member's copy of it, contains another category's (equal
* counts), or when a location cannot be resolved at all.
*/
export function sharedLocationCategories(
assignments: ReadonlyMap<ServedCategory, CategoryAssignment>,
backends: readonly BackendSnapshot[],
): Set<ServedCategory> {
const byName = new Map(backends.map((backend) => [backend.name, backend]));
const locations = new Map<ServedCategory, string[] | null>();
for (const [category, assignment] of assignments) {
locations.set(category, locationsOf(assignment, byName));
}
const shared = new Set<ServedCategory>();
for (const [category, own] of locations) {
if (own === null) {
shared.add(category);
continue;
}
for (const [other, theirs] of locations) {
if (other === category) continue;
if (theirs === null || theirs.some((location) => own.some((mine) => location.startsWith(mine)))) {
shared.add(category);
break;
}
}
}
return shared;
}
function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap<string, BackendSnapshot>): string[] | null {
const backend = backends.get(assignment.backendName);
if (!backend) return null;
const members =
backend.type === 'mirror'
? [backend.options.primary, ...asList(backend.options.replicas)].map((name) => backends.get(String(name)))
: [backend];
const bases: string[] = [];
for (const member of members) {
const base = member ? baseOf(member) : null;
if (base === null) return null;
bases.push(base);
}
return bases.map((base) => base + assignment.keyPrefix.toLowerCase());
}
/**
* Where a backend keeps its objects, lowercased: comparing more paths as equal
* than the volume does only excludes more, never less.
*/
function baseOf(backend: BackendSnapshot): string | null {
const { options } = backend;
if (backend.type === 'local') {
if (typeof options.root !== 'string' || options.root === '') return null;
return `local:${trimSlashes(path.resolve(options.root).replace(/\\/g, '/'))}/`.toLowerCase();
}
if (backend.type === 's3') {
if (typeof options.endpoint !== 'string' || typeof options.bucket !== 'string') return null;
const keyPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : '';
const base = `s3:${trimSlashes(options.endpoint)}/${options.bucket}/`;
return (keyPrefix === '' ? base : `${base}${keyPrefix}/`).toLowerCase();
}
return null;
}
function asList(value: string | number | string[] | undefined): string[] {
return Array.isArray(value) ? value : [];
}
function trimSlashes(value: string): string {
let start = 0;
let end = value.length;
while (start < end && value[start] === '/') start += 1;
while (end > start && value[end - 1] === '/') end -= 1;
return value.slice(start, end);
}
@@ -0,0 +1,419 @@
import { AppSettings } from '../../db/entities/AppSettings.entity';
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
import type {
ReclaimMarkRow,
StorageReclaimMarksRepository,
} from '../../db/repositories/StorageReclaimMarks.repository';
import { dbNow, parseDbTimestamp } from '../../db/types';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { logError } from '../audit/audit-log.logger';
import { DomainError } from '../common/domain-error';
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
import { StorageJobsService } from './storage-jobs.service';
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
import { inReclaimScope, RECLAIM_LEDGER, sharedLocationCategories } from './storage-reclaim.ledger';
import {
RECLAIM_DELETE_KEY,
RECLAIM_REPORT_KEY,
readReclaimDeleteEnabled,
readReclaimState,
} from './storage-reclaim.settings';
import type { StorageReferenceSource } from './storage-reclaim.types';
import { StorageRegistryService } from './storage-registry.service';
import { StorageService } from './storage.service';
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable, type OnModuleInit } from '@nestjs/common';
import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared';
const HOUR_MS = 60 * 60 * 1000;
/** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */
const RECLAIM_GRACE_MS = 24 * HOUR_MS;
/** How long an object must stay unreferenced, on every pass, before a pass may delete it. */
const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS;
/** More unreferenced than this share of a category points at a bucket another install shares: hands off. */
const RECLAIM_BRAKE_SHARE = 0.2;
/** Keys per reference question, the page the sources are promised. */
const SOURCE_PAGE = 500;
/** Marks per statement: four bound values each, well under SQLite's limit. */
const MARK_BATCH = 100;
/** Deletes in flight at once. */
const DELETE_BATCH = 20;
const JOB_NAME = 'a storage reclaim';
interface Candidate {
key: string;
size: number;
}
interface PassContext {
now: number;
deleting: boolean;
sources: readonly StorageReferenceSource[];
shared: ReadonlySet<ServedCategory>;
generation: number;
}
interface ReclaimRunOptions {
/** Delete nothing, whatever the switch says. */
reportOnly?: boolean;
now?: Date;
}
/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */
class ReclaimStopped extends Error {}
/**
* The reclaim pass (#2924): objects in upload storage that no row names any
* more, found in two phases and, only when the admin switched deletion on,
* deleted. A pass lists each category the ledger allows, skips objects younger
* than a day (the bytes of an upload land before its row), asks every
* registered reference source which keys are still named, and marks the rest
* with the time it first saw them. An object goes only once its mark is a week
* old, after one more question to every source about that single key, through
* `StorageService.delete` (so replicas follow and a demo keeps its bytes).
*
* Brakes, each failing closed: a category whose sources are not all
* registered, whose objects share a location with another category's, whose
* listing or lookups fail, or where more than a fifth of the objects look
* unreferenced is skipped and its marks start over. A pass never runs beside
* a backfill or a category migration (the one-job slot of StorageJobsService),
* never on a demo instance, and stops when the database is reopened under it.
*/
@Injectable()
export class StorageReclaimService implements OnModuleInit {
private generation = 0;
constructor(
private readonly storage: StorageService,
private readonly registry: StorageRegistryService,
private readonly references: StorageReclaimRegistryService,
private readonly jobs: StorageJobsService,
private readonly lifecycle: DatabaseLifecycle,
private readonly runtimeEnv: RuntimeEnvService,
@InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository,
@InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository,
) {}
onModuleInit(): void {
this.lifecycle.onReopened(async () => {
this.generation += 1;
});
}
async state(): Promise<StorageReclaimState> {
return await readReclaimState(this.appSettings);
}
async setDeleteEnabled(enabled: boolean): Promise<StorageReclaimState> {
await this.appSettings.setValue(RECLAIM_DELETE_KEY, enabled ? 'true' : 'false');
return await this.state();
}
/**
* One pass over every category, its report persisted and returned. Refuses
* with a 409 on a demo instance and while another storage job holds the slot.
*/
async run(options: ReclaimRunOptions = {}): Promise<StorageReclaimReport> {
if (this.runtimeEnv.isDemoMode()) {
throw new DomainError(409, 'storage reclaim does not run on a demo instance', { code: 'reclaim_demo' });
}
if (!this.jobs.claimExclusive(JOB_NAME)) {
throw new DomainError(409, 'another storage job is running, one storage job at a time', {
code: 'reclaim_busy',
});
}
try {
return await this.pass(options.reportOnly === true, options.now ?? new Date());
} finally {
this.jobs.releaseExclusive(JOB_NAME);
}
}
/**
* @txIndependent a sweep in passes: each category's marks and each delete
* stand on their own, and a pass cut short leaves the work done so far valid.
*/
private async pass(reportOnly: boolean, now: Date): Promise<StorageReclaimReport> {
const wallStart = Date.now();
const ctx: PassContext = {
now: now.getTime(),
deleting: !reportOnly && (await readReclaimDeleteEnabled(this.appSettings)),
sources: [...this.references.sources()],
shared: this.sharedLocations(),
generation: this.generation,
};
const categories = {} as Record<ServedCategory, StorageReclaimCategoryReport>;
let stopped: StorageReclaimReport['stopped'] = null;
await SERVED_CATEGORIES.reduce<Promise<void>>(async (previous, category) => {
await previous;
if (stopped === null) {
try {
categories[category] = await this.reclaimCategory(category, ctx);
return;
} catch (err) {
if (!(err instanceof ReclaimStopped)) throw err;
stopped = 'database_reopened';
}
}
categories[category] = emptyReport(null, 0);
}, Promise.resolve());
const report: StorageReclaimReport = {
startedAt: ctx.now,
finishedAt: ctx.now + (Date.now() - wallStart),
mode: ctx.deleting ? 'delete' : 'report',
stopped,
categories,
};
await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, JSON.stringify(report));
return report;
}
/** The categories whose resolved location overlaps another's, from the live registry. */
private sharedLocations(): Set<ServedCategory> {
const assignments = new Map(
SERVED_CATEGORIES.map((category) => {
const { backendName, keyPrefix } = this.registry.resolve(category);
return [category, { backendName, keyPrefix }] as const;
}),
);
return sharedLocationCategories(assignments, this.registry.snapshot().backends);
}
/**
* @txIndependent a sweep in passes: marks are written per batch and every
* delete stands alone, so a failure part way keeps what came before.
*/
private async reclaimCategory(category: ServedCategory, ctx: PassContext): Promise<StorageReclaimCategoryReport> {
const rule = RECLAIM_LEDGER[category];
if (!rule.reclaim) return emptyReport('not_reclaimed', 0);
const sources = ctx.sources.filter((source) => claimsAny(source, category));
const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id));
if (rule.requiredSources.some((id) => !answering.has(id))) return this.skip(category, 'sources_missing', 0);
if (ctx.shared.has(category)) return this.skip(category, 'shared_location', 0);
let listed: { objects: number; old: Candidate[] };
try {
listed = await this.listOld(category, ctx.now);
} catch (err) {
logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`);
return this.skip(category, 'list_failed', 0);
}
let unreferenced: Candidate[];
try {
unreferenced = await this.unreferencedOf(category, listed.old, sources);
} catch (err) {
logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`);
return this.skip(category, 'source_failed', listed.objects);
}
this.assertSameDatabase(ctx);
const unreferencedCount = countOf(unreferenced);
if (unreferenced.length > listed.objects * RECLAIM_BRAKE_SHARE) {
return {
...(await this.skip(category, 'too_many_unreferenced', listed.objects)),
unreferenced: unreferencedCount,
};
}
const marks = new Map((await this.marks.marksOf(category)).map((mark) => [mark.key, mark]));
await this.writeMarks(category, unreferenced, marks, ctx.now);
const cutoff = ctx.now - RECLAIM_DELAY_MS;
const eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff));
const deleted = ctx.deleting ? await this.deleteEligible(category, eligible, sources, ctx) : [];
return {
objects: listed.objects,
unreferenced: unreferencedCount,
eligible: countOf(eligible),
deleted: countOf(deleted),
skipped: null,
};
}
/** A category this pass cannot judge: nothing is deleted and its marks start over. */
private async skip(
category: ServedCategory,
reason: NonNullable<StorageReclaimCategoryReport['skipped']>,
objects: number,
): Promise<StorageReclaimCategoryReport> {
await this.marks.deleteCategory(category);
return emptyReport(reason, objects);
}
/** The root-level objects of a category, and of them the ones past the grace period. */
private async listOld(category: ServedCategory, now: number): Promise<{ objects: number; old: Candidate[] }> {
let objects = 0;
const old: Candidate[] = [];
for await (const stat of this.storage.list(category)) {
if (!inReclaimScope(stat.key)) continue;
objects += 1;
// A missing or zero modification time reads as infinitely old: such an object is kept.
if (!(stat.mtimeMs > 0) || now - stat.mtimeMs < RECLAIM_GRACE_MS) continue;
old.push({ key: stat.key, size: stat.size });
}
return { objects, old };
}
/** The candidates no source names, asked a page at a time. */
private async unreferencedOf(
category: ServedCategory,
candidates: readonly Candidate[],
sources: readonly StorageReferenceSource[],
): Promise<Candidate[]> {
return chunk(candidates, SOURCE_PAGE).reduce<Promise<Candidate[]>>(async (previous, page) => {
const done = await previous;
const referenced = await this.referencedIn(
category,
page.map((candidate) => candidate.key),
sources,
);
done.push(...page.filter((candidate) => !referenced.has(candidate.key.toLowerCase())));
return done;
}, Promise.resolve([]));
}
/**
* The keys some source still names, lowercased. Every key is claimed: the
* ledger's required sources answer for the whole category, and a category
* missing one never gets this far.
*/
private async referencedIn(
category: ServedCategory,
keys: readonly string[],
sources: readonly StorageReferenceSource[],
): Promise<Set<string>> {
const answers = await Promise.all(
sources.map(async (source) => {
const asked = keys.filter((key) => claimsKey(source, category, key));
return asked.length === 0 ? new Set<string>() : await source.referenced(category, asked);
}),
);
const referenced = new Set<string>();
for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase());
return referenced;
}
/**
* @txIndependent a mark batch stands on its own: a pass that stops between
* two leaves marks that are each true.
*/
private async writeMarks(
category: ServedCategory,
unreferenced: readonly Candidate[],
marks: ReadonlyMap<string, ReclaimMarkRow>,
now: number,
): Promise<void> {
const seenAt = dbNow(new Date(now));
const fresh = unreferenced.filter((candidate) => !marks.has(candidate.key));
const still = new Set(unreferenced.map((candidate) => candidate.key));
const stale = [...marks.keys()].filter((key) => !still.has(key));
await chunk(fresh, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
await previous;
await this.marks.insertMissing(category, page, seenAt);
}, Promise.resolve());
await chunk(stale, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
await previous;
await this.marks.deleteKeys(category, page);
}, Promise.resolve());
}
/**
* @txIndependent one delete per object, each with its own re-check, so a
* failure leaves the others' work standing.
*/
private async deleteEligible(
category: ServedCategory,
eligible: readonly Candidate[],
sources: readonly StorageReferenceSource[],
ctx: PassContext,
): Promise<Candidate[]> {
return chunk(eligible, DELETE_BATCH).reduce<Promise<Candidate[]>>(async (previous, batch) => {
const done = await previous;
const results = await Promise.all(batch.map((candidate) => this.deleteOne(category, candidate, sources, ctx)));
done.push(...batch.filter((_candidate, i) => results[i]));
return done;
}, Promise.resolve([]));
}
/**
* Asks every source about this one key again, checks the object is still the
* old one the pass judged, then deletes it and its mark. False keeps it.
*
* @txIndependent the object goes first and the mark after, so a mark never
* outlives a delete that failed.
*/
private async deleteOne(
category: ServedCategory,
candidate: Candidate,
sources: readonly StorageReferenceSource[],
ctx: PassContext,
): Promise<boolean> {
let referenced: Set<string>;
try {
referenced = await this.referencedIn(category, [candidate.key], sources);
} catch (err) {
logError(`[storage] reclaim kept a '${category}' object, its re-check failed: ${messageOf(err)}`);
return false;
}
if (referenced.size > 0) {
await this.marks.deleteKeys(category, [candidate.key]);
return false;
}
const stat = await this.storage.stat(category, candidate.key);
if (stat === null || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS) {
// Gone, or written again since it was judged: the next pass looks at it afresh.
await this.marks.deleteKeys(category, [candidate.key]);
return false;
}
if (this.storage.keepsOnDelete(category, candidate.key)) return false;
this.assertSameDatabase(ctx);
try {
await this.storage.delete(category, candidate.key);
} catch (err) {
logError(`[storage] reclaim could not delete a '${category}' object: ${messageOf(err)}`);
return false;
}
await this.marks.deleteKeys(category, [candidate.key]);
return true;
}
private assertSameDatabase(ctx: PassContext): void {
if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass');
}
}
function emptyReport(skipped: StorageReclaimCategoryReport['skipped'], objects: number): StorageReclaimCategoryReport {
const none = { objects: 0, bytes: 0 };
return { objects, unreferenced: none, eligible: none, deleted: none, skipped };
}
function countOf(candidates: readonly Candidate[]): { objects: number; bytes: number } {
return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) };
}
function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean {
const firstSeen = parseDbTimestamp(mark?.first_seen_at);
return firstSeen !== null && firstSeen.getTime() <= cutoff;
}
/** The source answers for some part of the category. */
function claimsAny(source: StorageReferenceSource, category: ServedCategory): boolean {
return source.claims.some((claim) => claim.category === category);
}
/** The source answers for the whole category: what the ledger's required sources must do. */
function answersFor(source: StorageReferenceSource, category: ServedCategory): boolean {
return source.claims.some((claim) => claim.category === category && claim.prefix === '');
}
function claimsKey(source: StorageReferenceSource, category: ServedCategory, key: string): boolean {
return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix));
}
function chunk<T>(items: readonly T[], size: number): T[][] {
return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size));
}
function messageOf(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
@@ -0,0 +1,32 @@
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
import { logWarn } from '../audit/audit-log.logger';
import { storageReclaimReportSchema, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared';
/** The admin's switch: 'true' lets a pass delete. Absent, or anything else, means report only. */
export const RECLAIM_DELETE_KEY = 'storage.reclaim_delete';
/** The last pass's report, one JSON row (the `storage.usage` pattern). */
export const RECLAIM_REPORT_KEY = 'storage.reclaim_report';
/** Fail closed: only the exact stored 'true' switches deletion on. */
export async function readReclaimDeleteEnabled(appSettings: AppSettingsRepository): Promise<boolean> {
return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === 'true';
}
/** The stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */
async function readReclaimReport(appSettings: AppSettingsRepository): Promise<StorageReclaimReport | null> {
const value = await appSettings.getValue(RECLAIM_REPORT_KEY);
if (value == null) return null;
try {
return storageReclaimReportSchema.parse(JSON.parse(value));
} catch {
logWarn('[storage] the stored reclaim report is unparseable, treating it as never run');
return null;
}
}
export async function readReclaimState(appSettings: AppSettingsRepository): Promise<StorageReclaimState> {
return {
deleteEnabled: await readReclaimDeleteEnabled(appSettings),
lastRun: await readReclaimReport(appSettings),
};
}
@@ -0,0 +1,174 @@
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
/**
* The contract between the storage reclaim pass and the domains whose rows
* name stored objects (#2924). Storage cannot read those rows itself: it may
* import no domain that imports it, and it may not inject another domain's
* repositories. So each owner registers a source with
* `StorageReclaimRegistryService` and answers, for a page of keys, which of
* them its rows still name.
*
* The ids are a closed list: the ledger in storage names, per category, which
* of them must have answered before a single object of that category may be
* called unreferenced.
*/
export const STORAGE_REFERENCE_SOURCE_IDS = [
'trips',
'collections',
'places',
'journey',
'photos',
'files',
'auth',
] as const;
export type StorageReferenceSourceId = (typeof STORAGE_REFERENCE_SOURCE_IDS)[number];
/**
* The categories a pass may delete from, at their root level only (the
* `files/assistant/` and `journey/thumbs/` subtrees have sweeps of their own).
* A source whose columns hold free-form upload paths (a cover, an image url,
* a photo path) claims all of them, since such a value can point anywhere.
*/
export const STORAGE_RECLAIMABLE_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places'] as const;
export type StorageReclaimableCategory = (typeof STORAGE_RECLAIMABLE_CATEGORIES)[number];
/** A part of a category a source answers for: the keys starting with `prefix` ('' is the whole category). */
export interface StorageReferenceClaim {
readonly category: ServedCategory;
readonly prefix: string;
}
export interface StorageReferenceSource {
readonly id: StorageReferenceSourceId;
/**
* Every category the source's columns can point into, not only the one they
* usually do: a trip cover may name `/uploads/places/x`, and that object
* must stay. A source asked about a category it does not claim is not asked.
*/
readonly claims: ReadonlyArray<StorageReferenceClaim>;
/**
* Of these category-relative keys (as `StorageService.list` yields them, at
* most a page of 500), the ones a row still names. A row that is trashed,
* pending or otherwise not gone counts. Throwing stops the pass for the
* category; it never deletes on a failed answer.
*/
referenced(category: ServedCategory, keys: readonly string[]): Promise<Set<string>>;
}
/** One object a stored value names. */
export interface StorageKeyRef {
category: string;
key: string;
}
/** What a column's value can mean besides the `/uploads/<category>/<key>` and `<category>/<key>` spellings. */
export interface StorageKeyOptions {
/** The column holds a bare name of this category (`users.avatar`, `trip_files.filename`). */
bare?: ServedCategory;
/** Legacy rows may carry any path before the name; the last segment names an object of `bare` too. */
basename?: boolean;
}
const UPLOADS_PATH = /(?:^|\/)uploads\/([a-z0-9][a-z0-9-]*)\//gi;
const KEY_END = /[?#"'<>\r\n]/;
const SERVED = new Set<string>(SERVED_CATEGORIES);
/**
* Every `/uploads/<category>/<key>` in a value, wherever it stands: a path, an
* absolute URL of this instance (`https://host/uploads/covers/x.jpg`), a leading
* `uploads/` without the slash, several in one text. A query or a fragment is
* dropped, the key is percent-decoded and trimmed. A key with a blank in it is
* reported both whole and cut at the blank, since a value may be a sentence:
* reporting one key too many only keeps an object, one too few would lose it.
*/
export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] {
if (typeof value !== 'string' || value === '') return [];
const matches = [...value.matchAll(UPLOADS_PATH)];
return matches.flatMap((match, i) => {
const start = match.index + match[0].length;
const next = matches[i + 1];
const tail = value.slice(start, next ? next.index : value.length);
const end = tail.search(KEY_END);
const category = match[1]!.toLowerCase();
return keyVariants(end === -1 ? tail : tail.slice(0, end)).map((key) => ({ category, key }));
});
}
/**
* Every object a stored value can name: the upload paths above, the relative
* `<category>/<key>` spelling (`journeys.cover_image` holds `covers/x`), and
* with `options` the bare or legacy-prefixed name of one category. A value
* that names nothing this way (an https URL elsewhere, a data: URL, garbage)
* answers an empty list: it references nothing a pass can find.
*/
export function storageKeysIn(value: string | null | undefined, options: StorageKeyOptions = {}): StorageKeyRef[] {
if (typeof value !== 'string') return [];
const trimmed = stripSuffix(value).trim();
if (trimmed === '') return [];
const refs = uploadKeysIn(value);
const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/i.exec(trimmed);
if (relative && SERVED.has(relative[1]!.toLowerCase())) {
const category = relative[1]!.toLowerCase();
for (const key of keyVariants(relative[2]!)) refs.push({ category, key });
}
if (options.bare) {
const bare = options.bare;
if (!/[/\\:]/.test(trimmed)) {
for (const key of keyVariants(trimmed)) refs.push({ category: bare, key });
} else if (options.basename) {
const last = trimmed.slice(Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')) + 1);
for (const key of keyVariants(last)) refs.push({ category: bare, key });
}
}
return refs;
}
/**
* The candidates of `category` that any of `values` names, compared without
* regard to case: a local volume on Windows or macOS serves `A.jpg` for `a.jpg`,
* so an object is kept when a value names it in either spelling. The helper a
* source builds `referenced()` from, so no source parses with a regex of its own.
*/
export function referencedAmong(
category: ServedCategory,
candidates: readonly string[],
values: Iterable<string | null | undefined>,
options: StorageKeyOptions = {},
): Set<string> {
const named = new Set<string>();
for (const value of values) {
for (const ref of storageKeysIn(value, options)) {
if (ref.category === category) named.add(ref.key.toLowerCase());
}
}
return new Set(candidates.filter((key) => named.has(key.toLowerCase())));
}
function stripSuffix(value: string): string {
const cut = value.search(/[?#]/);
return cut === -1 ? value : value.slice(0, cut);
}
/** The raw key, decoded, and cut at a blank; never an empty key or one that climbs out of its category. */
function keyVariants(raw: string): string[] {
const forms = new Set<string>();
const add = (key: string) => {
const clean = key.trim().replace(/^\/+/, '');
if (clean === '' || clean.split('/').some((segment) => segment === '..' || segment === '.')) return;
forms.add(clean);
};
const whole = stripSuffix(raw);
for (const form of [whole, decoded(whole)]) {
add(form);
add(form.trim().split(/\s/)[0]!);
}
return [...forms];
}
function decoded(value: string): string {
try {
return decodeURIComponent(value);
} catch {
return value;
}
}
+22 -4
View File
@@ -1,11 +1,17 @@
import { AppSettings } from '../../db/entities/AppSettings.entity';
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
import { AppConfigModule } from '../app-config/app-config.module';
import { AuditModule } from '../audit/audit.module';
import { DatabaseLifecycleModule } from '../database/database-lifecycle.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { StorageAdminController } from './storage-admin.controller';
import { StorageAdminService } from './storage-admin.service';
import { StorageEventsService } from './storage-events.service';
import { StorageJobsService } from './storage-jobs.service';
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
import { StorageReclaimController } from './storage-reclaim.controller';
import { StorageReclaimJob } from './storage-reclaim.job';
import { StorageReclaimService } from './storage-reclaim.service';
import { StorageRegistryService } from './storage-registry.service';
import { StorageStatsService } from './storage-stats.service';
import { StorageUsageScanJob } from './storage-usage-scan.job';
@@ -27,11 +33,20 @@ import { Module } from '@nestjs/common';
* may cache drivers or trigger reloads. MikroOrmModule.forFeature registers
* AppSettingsRepository for StorageRegistryService/StorageAdminService/
* StorageStatsService's @InjectRepository (Plan 3i, storage task — SR/SA/SS's
* 14 call sites, all on app_settings).
* 14 call sites, all on app_settings). The reclaim pass adds its marks table
* and follows database reopens (DatabaseLifecycleModule); the domains whose
* rows name stored objects register their reference sources with the exported
* StorageReclaimRegistryService, so storage never imports them.
*/
@Module({
imports: [AppConfigModule, AuditModule, SchedulingModule, MikroOrmModule.forFeature([AppSettings])],
controllers: [StorageAdminController],
imports: [
AppConfigModule,
AuditModule,
DatabaseLifecycleModule,
SchedulingModule,
MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks]),
],
controllers: [StorageAdminController, StorageReclaimController],
providers: [
StorageRegistryService,
StorageService,
@@ -40,7 +55,10 @@ import { Module } from '@nestjs/common';
StorageJobsService,
StorageStatsService,
StorageUsageScanJob,
StorageReclaimRegistryService,
StorageReclaimService,
StorageReclaimJob,
],
exports: [StorageService, StorageEventsService],
exports: [StorageService, StorageEventsService, StorageReclaimRegistryService],
})
export class StorageModule {}