mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
feat(storage): find and reclaim upload objects no row names any more
Domains whose rows name stored objects register a reference source with StorageReclaimRegistryService; storage never imports them. A ledger in storage decides per category whether it is reclaimed at all and which sources must have answered first, and a category whose resolved location overlaps another one is left alone. A nightly pass at 04:05 lists the root level of files, journey, covers, avatars and places, skips objects younger than a day or without a modification time, asks every source in pages of 500 and marks the rest. Only with deletion switched on, and only for marks a week old, it asks every source about the single key again and deletes through StorageService. A category is skipped and its marks start over when a source is missing or fails, the listing fails, or more than a fifth of its objects look unreferenced. A pass shares the one-job slot with backfills and migrations, never runs on a demo instance and stops when the database is reopened. Admins switch deletion and run a pass on demand under /api/admin/storage/reclaim; both are audited, and GET /api/admin/storage carries the switch and the last report. Deletion is off after install and upgrade.
This commit is contained in:
@@ -31,6 +31,8 @@ const ACTION_LABELS: Record<string, string> = {
|
||||
'admin.storage_backfill': 'started a storage sync',
|
||||
'admin.storage_backfill_cancel': 'cancelled a storage sync',
|
||||
'admin.storage_stats_refresh': 'refreshed storage usage stats',
|
||||
'admin.storage_reclaim_update': 'switched storage reclaim deletion',
|
||||
'admin.storage_reclaim_run': 'ran a storage reclaim pass',
|
||||
'admin.places_contribute_decision': 'decided on sharing place details with TREK Places',
|
||||
'admin.places_contribute_withdraw': 'withdrew the place details this installation shared',
|
||||
'immich.private_ip_configured': 'configured Immich with private IP',
|
||||
|
||||
@@ -1,4 +1,10 @@
|
||||
import { storageConfigPutSchema, storageMigrationRequestSchema, storageTestRequestSchema } from '@trek/shared';
|
||||
import {
|
||||
storageConfigPutSchema,
|
||||
storageMigrationRequestSchema,
|
||||
storageReclaimRunRequestSchema,
|
||||
storageReclaimSettingsRequestSchema,
|
||||
storageTestRequestSchema,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { createZodDto } from 'nestjs-zod';
|
||||
|
||||
@@ -12,3 +18,5 @@ import { createZodDto } from 'nestjs-zod';
|
||||
export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {}
|
||||
export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {}
|
||||
export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {}
|
||||
export class StorageReclaimSettingsDto extends createZodDto(storageReclaimSettingsRequestSchema) {}
|
||||
export class StorageReclaimRunDto extends createZodDto(storageReclaimRunRequestSchema) {}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { getSeedConfigPath } from './storage-paths';
|
||||
import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe';
|
||||
import { readReclaimState } from './storage-reclaim.settings';
|
||||
import { BACKENDS_KEY, CATEGORIES_KEY, VERSION_KEY, StorageRegistryService } from './storage-registry.service';
|
||||
import {
|
||||
assertNoMaskSentinels,
|
||||
@@ -63,6 +64,7 @@ export class StorageAdminService {
|
||||
health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) },
|
||||
seedFilePresent: fs.existsSync(getSeedConfigPath()),
|
||||
usage: await this.stats.readUsage(),
|
||||
reclaim: await readReclaimState(this.appSettings),
|
||||
backfills: this.jobs.statuses(),
|
||||
migrations: this.jobs.migrationStatuses(),
|
||||
version: await this.registry.currentConfigVersion(),
|
||||
|
||||
@@ -41,15 +41,38 @@ export class StorageJobsService {
|
||||
private readonly jobs = new Map<string, ActiveJob>();
|
||||
private readonly migrations = new Map<string, ActiveMigration>();
|
||||
private ttlMs = 10 * 60_000;
|
||||
/** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */
|
||||
private exclusive: string | null = null;
|
||||
|
||||
/** One storage job at a time, across both backfills and category migrations. */
|
||||
/** One storage job at a time, across backfills, category migrations and the reclaim pass. */
|
||||
private anyJobRunning(): boolean {
|
||||
return (
|
||||
this.exclusive !== null ||
|
||||
[...this.jobs.values()].some((j) => j.status.status === 'running') ||
|
||||
[...this.migrations.values()].some((m) => m.status.status === 'running')
|
||||
);
|
||||
}
|
||||
|
||||
/** The refusal a start meets while the slot is taken, naming what holds it. */
|
||||
private busyError(fallback: string): BackfillBusyError {
|
||||
return new BackfillBusyError(this.exclusive ? `${this.exclusive} is running, one storage job at a time` : fallback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the one-job slot for a pass that runs in the caller (the reclaim
|
||||
* pass): false while a backfill, a migration or another such pass runs.
|
||||
* The caller releases it in a finally.
|
||||
*/
|
||||
claimExclusive(name: string): boolean {
|
||||
if (this.anyJobRunning()) return false;
|
||||
this.exclusive = name;
|
||||
return true;
|
||||
}
|
||||
|
||||
releaseExclusive(name: string): void {
|
||||
if (this.exclusive === name) this.exclusive = null;
|
||||
}
|
||||
|
||||
constructor(private readonly registry: StorageRegistryService) {}
|
||||
|
||||
/** Test-only factory for a short TTL, since a second constructor param would confuse Nest DI. */
|
||||
@@ -74,7 +97,7 @@ export class StorageJobsService {
|
||||
throw new BackfillTargetError(`'${mirrorName}' is not a mirror backend`);
|
||||
}
|
||||
if (this.anyJobRunning()) {
|
||||
throw new BackfillBusyError('a sync is already running — one backfill at a time');
|
||||
throw this.busyError('a sync is already running — one backfill at a time');
|
||||
}
|
||||
const prefixes = resolved.map((r) => r.keyPrefix);
|
||||
|
||||
@@ -156,7 +179,7 @@ export class StorageJobsService {
|
||||
const target = this.registry.driverByName(to);
|
||||
if (!target) throw new MigrationTargetError(`no backend named '${to}'`);
|
||||
if (this.anyJobRunning()) {
|
||||
throw new BackfillBusyError('a sync is already running — one storage job at a time');
|
||||
throw this.busyError('a sync is already running — one storage job at a time');
|
||||
}
|
||||
// The DESTINATION prefix, computed for `to` before the flip — may differ
|
||||
// from `current.keyPrefix` (audit #8: a mode-A '' source migrating onto a
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import type { StorageReferenceSource, StorageReferenceSourceId } from './storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Where the domains whose rows name stored objects register their reference
|
||||
* source (`storage-reclaim.types.ts`), the StorageEventsService shape: an owner
|
||||
* imports StorageModule, injects this and calls `register` from its own
|
||||
* `onApplicationBootstrap`, so storage never imports the owner back.
|
||||
*
|
||||
* Registering never throws: the e2e suites boot partial module graphs, and a
|
||||
* graph without a source is no error here. The reclaim pass checks the
|
||||
* registered ids against its ledger and skips every category that lacks one,
|
||||
* and a full-app test pins that the app registers exactly what the ledger asks.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimRegistryService {
|
||||
private readonly registered: StorageReferenceSource[] = [];
|
||||
|
||||
register(source: StorageReferenceSource): void {
|
||||
// A provider listed in two modules bootstraps twice: one entry is enough.
|
||||
if (this.registered.includes(source)) return;
|
||||
if (this.registered.some((existing) => existing.id === source.id)) {
|
||||
// Two sources under one id are both asked; neither may hide the other.
|
||||
logWarn(`[storage] a second reference source registered as '${source.id}', both are asked`);
|
||||
}
|
||||
this.registered.push(source);
|
||||
}
|
||||
|
||||
sources(): readonly StorageReferenceSource[] {
|
||||
return this.registered;
|
||||
}
|
||||
|
||||
ids(): Set<StorageReferenceSourceId> {
|
||||
return new Set(this.registered.map((source) => source.id));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { User } from '../../types';
|
||||
import { AuditService } from '../audit/audit.service';
|
||||
import { getClientIp } from '../audit/client-ip';
|
||||
import { AdminGuard } from '../auth-core/admin.guard';
|
||||
import { CurrentUser } from '../auth-core/current-user.decorator';
|
||||
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
|
||||
import { ManagedForbidden } from '../common/managed';
|
||||
import { ResponseContract } from '../common/response-contract';
|
||||
import { StorageReclaimRunDto, StorageReclaimSettingsDto } from './storage-admin.dto';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { Body, Controller, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common';
|
||||
import {
|
||||
storageReclaimReportSchema,
|
||||
storageReclaimStateSchema,
|
||||
type StorageReclaimReport,
|
||||
type StorageReclaimState,
|
||||
} from '@trek/shared';
|
||||
|
||||
import type { Request } from 'express';
|
||||
|
||||
/**
|
||||
* /api/admin/storage/reclaim: the admin's switch for the reclaim pass and a
|
||||
* pass on demand. The state itself rides on GET /api/admin/storage. Deleting
|
||||
* stays off until an admin turns it on here; a pass then still deletes only
|
||||
* what has been unreferenced for a week (StorageReclaimService).
|
||||
*/
|
||||
@Controller('api/admin/storage/reclaim')
|
||||
@UseGuards(JwtAuthGuard, AdminGuard)
|
||||
@ManagedForbidden('storage backends and their contents are hoster-level configuration')
|
||||
export class StorageReclaimController {
|
||||
constructor(
|
||||
private readonly reclaim: StorageReclaimService,
|
||||
private readonly audit: AuditService,
|
||||
) {}
|
||||
|
||||
@Put()
|
||||
@ResponseContract(storageReclaimStateSchema)
|
||||
async update(
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: StorageReclaimSettingsDto,
|
||||
@Req() req: Request,
|
||||
): Promise<StorageReclaimState> {
|
||||
const state = await this.reclaim.setDeleteEnabled(body.deleteEnabled);
|
||||
await this.audit.writeAudit({
|
||||
userId: user.id,
|
||||
action: 'admin.storage_reclaim_update',
|
||||
ip: getClientIp(req),
|
||||
details: { deleteEnabled: body.deleteEnabled },
|
||||
});
|
||||
return state;
|
||||
}
|
||||
|
||||
/** One pass now, honouring the switch unless `reportOnly`. 409 on a demo or while another storage job runs. */
|
||||
@Post('run')
|
||||
@HttpCode(200)
|
||||
@ResponseContract(storageReclaimReportSchema)
|
||||
async run(
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: StorageReclaimRunDto,
|
||||
@Req() req: Request,
|
||||
): Promise<StorageReclaimReport> {
|
||||
const report = await this.reclaim.run({ reportOnly: body.reportOnly === true });
|
||||
const rows = Object.values(report.categories);
|
||||
await this.audit.writeAudit({
|
||||
userId: user.id,
|
||||
action: 'admin.storage_reclaim_run',
|
||||
ip: getClientIp(req),
|
||||
details: {
|
||||
mode: report.mode,
|
||||
deleted: rows.reduce((sum, row) => sum + row.deleted.objects, 0),
|
||||
bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0),
|
||||
},
|
||||
});
|
||||
return report;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { logDebug, logError, logInfo } from '../audit/audit-log.logger';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
import type { StorageReclaimReport } from '@trek/shared';
|
||||
|
||||
/**
|
||||
* The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00 and
|
||||
* before the usage scan at 04:15, so the next usage figures already show what
|
||||
* it freed. No pass at boot: a pass that may delete waits for its hour.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
private readonly registrar: CronRegistrarService,
|
||||
private readonly reclaim: StorageReclaimService,
|
||||
private readonly runtimeEnv: RuntimeEnvService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('storage-reclaim', '5 4 * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: each category's marks and each delete
|
||||
* stand on their own (StorageReclaimService.run).
|
||||
*/
|
||||
async tick(now: Date = new Date()): Promise<void> {
|
||||
if (this.runtimeEnv.isDemoMode()) return;
|
||||
try {
|
||||
const report = await this.reclaim.run({ now });
|
||||
const line = summary(report);
|
||||
if (line) logInfo(`Storage reclaim: ${line}`);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof DomainError && err.getStatus() === 409) {
|
||||
logDebug(`Storage reclaim skipped: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
logError(`Storage reclaim: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** One line when the pass found, deleted or refused something; null when there was nothing to say. */
|
||||
function summary(report: StorageReclaimReport): string | null {
|
||||
const rows = Object.values(report.categories);
|
||||
const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0);
|
||||
const deleted = rows.reduce((sum, row) => sum + row.deleted.objects, 0);
|
||||
const bytes = rows.reduce((sum, row) => sum + row.deleted.bytes, 0);
|
||||
const braked = Object.entries(report.categories)
|
||||
.filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed')
|
||||
.map(([category, row]) => `${category} (${row.skipped})`);
|
||||
const parts: string[] = [];
|
||||
if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`);
|
||||
if (deleted > 0) parts.push(`deleted ${deleted} (${bytes} bytes)`);
|
||||
if (braked.length > 0) parts.push(`skipped ${braked.join(', ')}`);
|
||||
if (report.stopped) parts.push(`stopped: ${report.stopped}`);
|
||||
return parts.length > 0 ? parts.join('; ') : null;
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import type { StorageReclaimableCategory, StorageReferenceSourceId } from './storage-reclaim.types';
|
||||
import type { BackendSnapshot } from './storage-registry.service';
|
||||
import type { ServedCategory } from './storage.types';
|
||||
|
||||
import path from 'node:path';
|
||||
|
||||
export type ReclaimRule =
|
||||
| { readonly reclaim: false; readonly reason: string }
|
||||
| { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] };
|
||||
|
||||
/** The owners of columns that hold a free-form upload path, which can point into any category. */
|
||||
const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos'];
|
||||
|
||||
/**
|
||||
* What the reclaim pass may do per category, and which reference sources must
|
||||
* have registered before it may call one object of it unreferenced. A Record,
|
||||
* so a new category does not compile until it is decided here (the
|
||||
* KEPT_ON_DEMO precedent). A category missing one of its sources is skipped,
|
||||
* never swept on a partial answer.
|
||||
*/
|
||||
export const RECLAIM_LEDGER: Readonly<Record<ServedCategory, ReclaimRule>> = {
|
||||
// trip_files.filename (trashed rows included) holds the bare name; collab
|
||||
// note files, chat images and doc-sync copies are trip_files rows too.
|
||||
files: { reclaim: true, requiredSources: ['files', ...FREE_FORM_SOURCES] },
|
||||
// trek_photos.file_path and thumbnail_path (posters), journeys.cover_image.
|
||||
journey: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
// Trip, collection and journey covers; a trip copy and a journey share one object.
|
||||
covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] },
|
||||
// places.image_url, collection_places.image_url and the contribution outbox.
|
||||
places: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
photos: {
|
||||
reclaim: false,
|
||||
reason:
|
||||
'the legacy photo folder is served to every signed-in user by name, holds the place-photo caches in its ' +
|
||||
'google/ and trek/ subtrees, and nothing writes to it any more',
|
||||
},
|
||||
'photos-google': { reclaim: false, reason: 'a cache the place-photo sweep keeps against its meta rows' },
|
||||
'photos-trek': { reclaim: false, reason: 'a cache with a one-hour lifetime and a sweep of its own' },
|
||||
backups: { reclaim: false, reason: 'no row names a backup; auto-backup rotates them by name' },
|
||||
};
|
||||
|
||||
/** Every source id some category of the ledger waits for: what a full app must register. */
|
||||
export function requiredReclaimSources(): Set<StorageReferenceSourceId> {
|
||||
const ids = new Set<StorageReferenceSourceId>();
|
||||
for (const rule of Object.values(RECLAIM_LEDGER)) {
|
||||
if (rule.reclaim) for (const id of rule.requiredSources) ids.add(id);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
/** The categories the ledger lets a pass delete from. */
|
||||
export function reclaimableCategories(): StorageReclaimableCategory[] {
|
||||
return (Object.keys(RECLAIM_LEDGER) as ServedCategory[]).filter(
|
||||
(category): category is StorageReclaimableCategory => RECLAIM_LEDGER[category].reclaim,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Only the root level of a category is swept: `files/assistant/` belongs to the
|
||||
* assistant's retention, `journey/thumbs/` to the thumbnail sweep, and a nested
|
||||
* key elsewhere was not written by TREK.
|
||||
*/
|
||||
export function inReclaimScope(key: string): boolean {
|
||||
return key !== '' && !key.includes('/');
|
||||
}
|
||||
|
||||
export interface CategoryAssignment {
|
||||
backendName: string;
|
||||
keyPrefix: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The categories whose objects may sit where another category's do, decided
|
||||
* from the RESOLVED backends rather than their names: two backends can share a
|
||||
* root or a bucket, `TREK_PLACE_PHOTO_DIR` can point inside the uploads folder,
|
||||
* the legacy `photos/` folder holds the two photo caches, and `backups` on a
|
||||
* shared backend is that backend's whole root. A category is excluded when its
|
||||
* location, or a mirror member's copy of it, contains another category's (equal
|
||||
* counts), or when a location cannot be resolved at all.
|
||||
*/
|
||||
export function sharedLocationCategories(
|
||||
assignments: ReadonlyMap<ServedCategory, CategoryAssignment>,
|
||||
backends: readonly BackendSnapshot[],
|
||||
): Set<ServedCategory> {
|
||||
const byName = new Map(backends.map((backend) => [backend.name, backend]));
|
||||
const locations = new Map<ServedCategory, string[] | null>();
|
||||
for (const [category, assignment] of assignments) {
|
||||
locations.set(category, locationsOf(assignment, byName));
|
||||
}
|
||||
const shared = new Set<ServedCategory>();
|
||||
for (const [category, own] of locations) {
|
||||
if (own === null) {
|
||||
shared.add(category);
|
||||
continue;
|
||||
}
|
||||
for (const [other, theirs] of locations) {
|
||||
if (other === category) continue;
|
||||
if (theirs === null || theirs.some((location) => own.some((mine) => location.startsWith(mine)))) {
|
||||
shared.add(category);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return shared;
|
||||
}
|
||||
|
||||
function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap<string, BackendSnapshot>): string[] | null {
|
||||
const backend = backends.get(assignment.backendName);
|
||||
if (!backend) return null;
|
||||
const members =
|
||||
backend.type === 'mirror'
|
||||
? [backend.options.primary, ...asList(backend.options.replicas)].map((name) => backends.get(String(name)))
|
||||
: [backend];
|
||||
const bases: string[] = [];
|
||||
for (const member of members) {
|
||||
const base = member ? baseOf(member) : null;
|
||||
if (base === null) return null;
|
||||
bases.push(base);
|
||||
}
|
||||
return bases.map((base) => base + assignment.keyPrefix.toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a backend keeps its objects, lowercased: comparing more paths as equal
|
||||
* than the volume does only excludes more, never less.
|
||||
*/
|
||||
function baseOf(backend: BackendSnapshot): string | null {
|
||||
const { options } = backend;
|
||||
if (backend.type === 'local') {
|
||||
if (typeof options.root !== 'string' || options.root === '') return null;
|
||||
return `local:${trimSlashes(path.resolve(options.root).replace(/\\/g, '/'))}/`.toLowerCase();
|
||||
}
|
||||
if (backend.type === 's3') {
|
||||
if (typeof options.endpoint !== 'string' || typeof options.bucket !== 'string') return null;
|
||||
const keyPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : '';
|
||||
const base = `s3:${trimSlashes(options.endpoint)}/${options.bucket}/`;
|
||||
return (keyPrefix === '' ? base : `${base}${keyPrefix}/`).toLowerCase();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function asList(value: string | number | string[] | undefined): string[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function trimSlashes(value: string): string {
|
||||
let start = 0;
|
||||
let end = value.length;
|
||||
while (start < end && value[start] === '/') start += 1;
|
||||
while (end > start && value[end - 1] === '/') end -= 1;
|
||||
return value.slice(start, end);
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
import { AppSettings } from '../../db/entities/AppSettings.entity';
|
||||
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
|
||||
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
|
||||
import type {
|
||||
ReclaimMarkRow,
|
||||
StorageReclaimMarksRepository,
|
||||
} from '../../db/repositories/StorageReclaimMarks.repository';
|
||||
import { dbNow, parseDbTimestamp } from '../../db/types';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
|
||||
import { inReclaimScope, RECLAIM_LEDGER, sharedLocationCategories } from './storage-reclaim.ledger';
|
||||
import {
|
||||
RECLAIM_DELETE_KEY,
|
||||
RECLAIM_REPORT_KEY,
|
||||
readReclaimDeleteEnabled,
|
||||
readReclaimState,
|
||||
} from './storage-reclaim.settings';
|
||||
import type { StorageReferenceSource } from './storage-reclaim.types';
|
||||
import { StorageRegistryService } from './storage-registry.service';
|
||||
import { StorageService } from './storage.service';
|
||||
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnModuleInit } from '@nestjs/common';
|
||||
import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared';
|
||||
|
||||
const HOUR_MS = 60 * 60 * 1000;
|
||||
/** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */
|
||||
const RECLAIM_GRACE_MS = 24 * HOUR_MS;
|
||||
/** How long an object must stay unreferenced, on every pass, before a pass may delete it. */
|
||||
const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS;
|
||||
/** More unreferenced than this share of a category points at a bucket another install shares: hands off. */
|
||||
const RECLAIM_BRAKE_SHARE = 0.2;
|
||||
/** Keys per reference question, the page the sources are promised. */
|
||||
const SOURCE_PAGE = 500;
|
||||
/** Marks per statement: four bound values each, well under SQLite's limit. */
|
||||
const MARK_BATCH = 100;
|
||||
/** Deletes in flight at once. */
|
||||
const DELETE_BATCH = 20;
|
||||
const JOB_NAME = 'a storage reclaim';
|
||||
|
||||
interface Candidate {
|
||||
key: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
interface PassContext {
|
||||
now: number;
|
||||
deleting: boolean;
|
||||
sources: readonly StorageReferenceSource[];
|
||||
shared: ReadonlySet<ServedCategory>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
interface ReclaimRunOptions {
|
||||
/** Delete nothing, whatever the switch says. */
|
||||
reportOnly?: boolean;
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */
|
||||
class ReclaimStopped extends Error {}
|
||||
|
||||
/**
|
||||
* The reclaim pass (#2924): objects in upload storage that no row names any
|
||||
* more, found in two phases and, only when the admin switched deletion on,
|
||||
* deleted. A pass lists each category the ledger allows, skips objects younger
|
||||
* than a day (the bytes of an upload land before its row), asks every
|
||||
* registered reference source which keys are still named, and marks the rest
|
||||
* with the time it first saw them. An object goes only once its mark is a week
|
||||
* old, after one more question to every source about that single key, through
|
||||
* `StorageService.delete` (so replicas follow and a demo keeps its bytes).
|
||||
*
|
||||
* Brakes, each failing closed: a category whose sources are not all
|
||||
* registered, whose objects share a location with another category's, whose
|
||||
* listing or lookups fail, or where more than a fifth of the objects look
|
||||
* unreferenced is skipped and its marks start over. A pass never runs beside
|
||||
* a backfill or a category migration (the one-job slot of StorageJobsService),
|
||||
* never on a demo instance, and stops when the database is reopened under it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimService implements OnModuleInit {
|
||||
private generation = 0;
|
||||
|
||||
constructor(
|
||||
private readonly storage: StorageService,
|
||||
private readonly registry: StorageRegistryService,
|
||||
private readonly references: StorageReclaimRegistryService,
|
||||
private readonly jobs: StorageJobsService,
|
||||
private readonly lifecycle: DatabaseLifecycle,
|
||||
private readonly runtimeEnv: RuntimeEnvService,
|
||||
@InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository,
|
||||
@InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
this.lifecycle.onReopened(async () => {
|
||||
this.generation += 1;
|
||||
});
|
||||
}
|
||||
|
||||
async state(): Promise<StorageReclaimState> {
|
||||
return await readReclaimState(this.appSettings);
|
||||
}
|
||||
|
||||
async setDeleteEnabled(enabled: boolean): Promise<StorageReclaimState> {
|
||||
await this.appSettings.setValue(RECLAIM_DELETE_KEY, enabled ? 'true' : 'false');
|
||||
return await this.state();
|
||||
}
|
||||
|
||||
/**
|
||||
* One pass over every category, its report persisted and returned. Refuses
|
||||
* with a 409 on a demo instance and while another storage job holds the slot.
|
||||
*/
|
||||
async run(options: ReclaimRunOptions = {}): Promise<StorageReclaimReport> {
|
||||
if (this.runtimeEnv.isDemoMode()) {
|
||||
throw new DomainError(409, 'storage reclaim does not run on a demo instance', { code: 'reclaim_demo' });
|
||||
}
|
||||
if (!this.jobs.claimExclusive(JOB_NAME)) {
|
||||
throw new DomainError(409, 'another storage job is running, one storage job at a time', {
|
||||
code: 'reclaim_busy',
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await this.pass(options.reportOnly === true, options.now ?? new Date());
|
||||
} finally {
|
||||
this.jobs.releaseExclusive(JOB_NAME);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: each category's marks and each delete
|
||||
* stand on their own, and a pass cut short leaves the work done so far valid.
|
||||
*/
|
||||
private async pass(reportOnly: boolean, now: Date): Promise<StorageReclaimReport> {
|
||||
const wallStart = Date.now();
|
||||
const ctx: PassContext = {
|
||||
now: now.getTime(),
|
||||
deleting: !reportOnly && (await readReclaimDeleteEnabled(this.appSettings)),
|
||||
sources: [...this.references.sources()],
|
||||
shared: this.sharedLocations(),
|
||||
generation: this.generation,
|
||||
};
|
||||
const categories = {} as Record<ServedCategory, StorageReclaimCategoryReport>;
|
||||
let stopped: StorageReclaimReport['stopped'] = null;
|
||||
await SERVED_CATEGORIES.reduce<Promise<void>>(async (previous, category) => {
|
||||
await previous;
|
||||
if (stopped === null) {
|
||||
try {
|
||||
categories[category] = await this.reclaimCategory(category, ctx);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (!(err instanceof ReclaimStopped)) throw err;
|
||||
stopped = 'database_reopened';
|
||||
}
|
||||
}
|
||||
categories[category] = emptyReport(null, 0);
|
||||
}, Promise.resolve());
|
||||
const report: StorageReclaimReport = {
|
||||
startedAt: ctx.now,
|
||||
finishedAt: ctx.now + (Date.now() - wallStart),
|
||||
mode: ctx.deleting ? 'delete' : 'report',
|
||||
stopped,
|
||||
categories,
|
||||
};
|
||||
await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, JSON.stringify(report));
|
||||
return report;
|
||||
}
|
||||
|
||||
/** The categories whose resolved location overlaps another's, from the live registry. */
|
||||
private sharedLocations(): Set<ServedCategory> {
|
||||
const assignments = new Map(
|
||||
SERVED_CATEGORIES.map((category) => {
|
||||
const { backendName, keyPrefix } = this.registry.resolve(category);
|
||||
return [category, { backendName, keyPrefix }] as const;
|
||||
}),
|
||||
);
|
||||
return sharedLocationCategories(assignments, this.registry.snapshot().backends);
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a sweep in passes: marks are written per batch and every
|
||||
* delete stands alone, so a failure part way keeps what came before.
|
||||
*/
|
||||
private async reclaimCategory(category: ServedCategory, ctx: PassContext): Promise<StorageReclaimCategoryReport> {
|
||||
const rule = RECLAIM_LEDGER[category];
|
||||
if (!rule.reclaim) return emptyReport('not_reclaimed', 0);
|
||||
const sources = ctx.sources.filter((source) => claimsAny(source, category));
|
||||
const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id));
|
||||
if (rule.requiredSources.some((id) => !answering.has(id))) return this.skip(category, 'sources_missing', 0);
|
||||
if (ctx.shared.has(category)) return this.skip(category, 'shared_location', 0);
|
||||
|
||||
let listed: { objects: number; old: Candidate[] };
|
||||
try {
|
||||
listed = await this.listOld(category, ctx.now);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`);
|
||||
return this.skip(category, 'list_failed', 0);
|
||||
}
|
||||
let unreferenced: Candidate[];
|
||||
try {
|
||||
unreferenced = await this.unreferencedOf(category, listed.old, sources);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`);
|
||||
return this.skip(category, 'source_failed', listed.objects);
|
||||
}
|
||||
this.assertSameDatabase(ctx);
|
||||
const unreferencedCount = countOf(unreferenced);
|
||||
if (unreferenced.length > listed.objects * RECLAIM_BRAKE_SHARE) {
|
||||
return {
|
||||
...(await this.skip(category, 'too_many_unreferenced', listed.objects)),
|
||||
unreferenced: unreferencedCount,
|
||||
};
|
||||
}
|
||||
|
||||
const marks = new Map((await this.marks.marksOf(category)).map((mark) => [mark.key, mark]));
|
||||
await this.writeMarks(category, unreferenced, marks, ctx.now);
|
||||
const cutoff = ctx.now - RECLAIM_DELAY_MS;
|
||||
const eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff));
|
||||
const deleted = ctx.deleting ? await this.deleteEligible(category, eligible, sources, ctx) : [];
|
||||
return {
|
||||
objects: listed.objects,
|
||||
unreferenced: unreferencedCount,
|
||||
eligible: countOf(eligible),
|
||||
deleted: countOf(deleted),
|
||||
skipped: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** A category this pass cannot judge: nothing is deleted and its marks start over. */
|
||||
private async skip(
|
||||
category: ServedCategory,
|
||||
reason: NonNullable<StorageReclaimCategoryReport['skipped']>,
|
||||
objects: number,
|
||||
): Promise<StorageReclaimCategoryReport> {
|
||||
await this.marks.deleteCategory(category);
|
||||
return emptyReport(reason, objects);
|
||||
}
|
||||
|
||||
/** The root-level objects of a category, and of them the ones past the grace period. */
|
||||
private async listOld(category: ServedCategory, now: number): Promise<{ objects: number; old: Candidate[] }> {
|
||||
let objects = 0;
|
||||
const old: Candidate[] = [];
|
||||
for await (const stat of this.storage.list(category)) {
|
||||
if (!inReclaimScope(stat.key)) continue;
|
||||
objects += 1;
|
||||
// A missing or zero modification time reads as infinitely old: such an object is kept.
|
||||
if (!(stat.mtimeMs > 0) || now - stat.mtimeMs < RECLAIM_GRACE_MS) continue;
|
||||
old.push({ key: stat.key, size: stat.size });
|
||||
}
|
||||
return { objects, old };
|
||||
}
|
||||
|
||||
/** The candidates no source names, asked a page at a time. */
|
||||
private async unreferencedOf(
|
||||
category: ServedCategory,
|
||||
candidates: readonly Candidate[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
): Promise<Candidate[]> {
|
||||
return chunk(candidates, SOURCE_PAGE).reduce<Promise<Candidate[]>>(async (previous, page) => {
|
||||
const done = await previous;
|
||||
const referenced = await this.referencedIn(
|
||||
category,
|
||||
page.map((candidate) => candidate.key),
|
||||
sources,
|
||||
);
|
||||
done.push(...page.filter((candidate) => !referenced.has(candidate.key.toLowerCase())));
|
||||
return done;
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* The keys some source still names, lowercased. Every key is claimed: the
|
||||
* ledger's required sources answer for the whole category, and a category
|
||||
* missing one never gets this far.
|
||||
*/
|
||||
private async referencedIn(
|
||||
category: ServedCategory,
|
||||
keys: readonly string[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
): Promise<Set<string>> {
|
||||
const answers = await Promise.all(
|
||||
sources.map(async (source) => {
|
||||
const asked = keys.filter((key) => claimsKey(source, category, key));
|
||||
return asked.length === 0 ? new Set<string>() : await source.referenced(category, asked);
|
||||
}),
|
||||
);
|
||||
const referenced = new Set<string>();
|
||||
for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase());
|
||||
return referenced;
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent a mark batch stands on its own: a pass that stops between
|
||||
* two leaves marks that are each true.
|
||||
*/
|
||||
private async writeMarks(
|
||||
category: ServedCategory,
|
||||
unreferenced: readonly Candidate[],
|
||||
marks: ReadonlyMap<string, ReclaimMarkRow>,
|
||||
now: number,
|
||||
): Promise<void> {
|
||||
const seenAt = dbNow(new Date(now));
|
||||
const fresh = unreferenced.filter((candidate) => !marks.has(candidate.key));
|
||||
const still = new Set(unreferenced.map((candidate) => candidate.key));
|
||||
const stale = [...marks.keys()].filter((key) => !still.has(key));
|
||||
await chunk(fresh, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
|
||||
await previous;
|
||||
await this.marks.insertMissing(category, page, seenAt);
|
||||
}, Promise.resolve());
|
||||
await chunk(stale, MARK_BATCH).reduce<Promise<void>>(async (previous, page) => {
|
||||
await previous;
|
||||
await this.marks.deleteKeys(category, page);
|
||||
}, Promise.resolve());
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent one delete per object, each with its own re-check, so a
|
||||
* failure leaves the others' work standing.
|
||||
*/
|
||||
private async deleteEligible(
|
||||
category: ServedCategory,
|
||||
eligible: readonly Candidate[],
|
||||
sources: readonly StorageReferenceSource[],
|
||||
ctx: PassContext,
|
||||
): Promise<Candidate[]> {
|
||||
return chunk(eligible, DELETE_BATCH).reduce<Promise<Candidate[]>>(async (previous, batch) => {
|
||||
const done = await previous;
|
||||
const results = await Promise.all(batch.map((candidate) => this.deleteOne(category, candidate, sources, ctx)));
|
||||
done.push(...batch.filter((_candidate, i) => results[i]));
|
||||
return done;
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks every source about this one key again, checks the object is still the
|
||||
* old one the pass judged, then deletes it and its mark. False keeps it.
|
||||
*
|
||||
* @txIndependent the object goes first and the mark after, so a mark never
|
||||
* outlives a delete that failed.
|
||||
*/
|
||||
private async deleteOne(
|
||||
category: ServedCategory,
|
||||
candidate: Candidate,
|
||||
sources: readonly StorageReferenceSource[],
|
||||
ctx: PassContext,
|
||||
): Promise<boolean> {
|
||||
let referenced: Set<string>;
|
||||
try {
|
||||
referenced = await this.referencedIn(category, [candidate.key], sources);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim kept a '${category}' object, its re-check failed: ${messageOf(err)}`);
|
||||
return false;
|
||||
}
|
||||
if (referenced.size > 0) {
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return false;
|
||||
}
|
||||
const stat = await this.storage.stat(category, candidate.key);
|
||||
if (stat === null || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS) {
|
||||
// Gone, or written again since it was judged: the next pass looks at it afresh.
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return false;
|
||||
}
|
||||
if (this.storage.keepsOnDelete(category, candidate.key)) return false;
|
||||
this.assertSameDatabase(ctx);
|
||||
try {
|
||||
await this.storage.delete(category, candidate.key);
|
||||
} catch (err) {
|
||||
logError(`[storage] reclaim could not delete a '${category}' object: ${messageOf(err)}`);
|
||||
return false;
|
||||
}
|
||||
await this.marks.deleteKeys(category, [candidate.key]);
|
||||
return true;
|
||||
}
|
||||
|
||||
private assertSameDatabase(ctx: PassContext): void {
|
||||
if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass');
|
||||
}
|
||||
}
|
||||
|
||||
function emptyReport(skipped: StorageReclaimCategoryReport['skipped'], objects: number): StorageReclaimCategoryReport {
|
||||
const none = { objects: 0, bytes: 0 };
|
||||
return { objects, unreferenced: none, eligible: none, deleted: none, skipped };
|
||||
}
|
||||
|
||||
function countOf(candidates: readonly Candidate[]): { objects: number; bytes: number } {
|
||||
return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) };
|
||||
}
|
||||
|
||||
function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean {
|
||||
const firstSeen = parseDbTimestamp(mark?.first_seen_at);
|
||||
return firstSeen !== null && firstSeen.getTime() <= cutoff;
|
||||
}
|
||||
|
||||
/** The source answers for some part of the category. */
|
||||
function claimsAny(source: StorageReferenceSource, category: ServedCategory): boolean {
|
||||
return source.claims.some((claim) => claim.category === category);
|
||||
}
|
||||
|
||||
/** The source answers for the whole category: what the ledger's required sources must do. */
|
||||
function answersFor(source: StorageReferenceSource, category: ServedCategory): boolean {
|
||||
return source.claims.some((claim) => claim.category === category && claim.prefix === '');
|
||||
}
|
||||
|
||||
function claimsKey(source: StorageReferenceSource, category: ServedCategory, key: string): boolean {
|
||||
return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix));
|
||||
}
|
||||
|
||||
function chunk<T>(items: readonly T[], size: number): T[][] {
|
||||
return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size));
|
||||
}
|
||||
|
||||
function messageOf(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import { storageReclaimReportSchema, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared';
|
||||
|
||||
/** The admin's switch: 'true' lets a pass delete. Absent, or anything else, means report only. */
|
||||
export const RECLAIM_DELETE_KEY = 'storage.reclaim_delete';
|
||||
/** The last pass's report, one JSON row (the `storage.usage` pattern). */
|
||||
export const RECLAIM_REPORT_KEY = 'storage.reclaim_report';
|
||||
|
||||
/** Fail closed: only the exact stored 'true' switches deletion on. */
|
||||
export async function readReclaimDeleteEnabled(appSettings: AppSettingsRepository): Promise<boolean> {
|
||||
return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === 'true';
|
||||
}
|
||||
|
||||
/** The stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */
|
||||
async function readReclaimReport(appSettings: AppSettingsRepository): Promise<StorageReclaimReport | null> {
|
||||
const value = await appSettings.getValue(RECLAIM_REPORT_KEY);
|
||||
if (value == null) return null;
|
||||
try {
|
||||
return storageReclaimReportSchema.parse(JSON.parse(value));
|
||||
} catch {
|
||||
logWarn('[storage] the stored reclaim report is unparseable, treating it as never run');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readReclaimState(appSettings: AppSettingsRepository): Promise<StorageReclaimState> {
|
||||
return {
|
||||
deleteEnabled: await readReclaimDeleteEnabled(appSettings),
|
||||
lastRun: await readReclaimReport(appSettings),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
|
||||
|
||||
/**
|
||||
* The contract between the storage reclaim pass and the domains whose rows
|
||||
* name stored objects (#2924). Storage cannot read those rows itself: it may
|
||||
* import no domain that imports it, and it may not inject another domain's
|
||||
* repositories. So each owner registers a source with
|
||||
* `StorageReclaimRegistryService` and answers, for a page of keys, which of
|
||||
* them its rows still name.
|
||||
*
|
||||
* The ids are a closed list: the ledger in storage names, per category, which
|
||||
* of them must have answered before a single object of that category may be
|
||||
* called unreferenced.
|
||||
*/
|
||||
export const STORAGE_REFERENCE_SOURCE_IDS = [
|
||||
'trips',
|
||||
'collections',
|
||||
'places',
|
||||
'journey',
|
||||
'photos',
|
||||
'files',
|
||||
'auth',
|
||||
] as const;
|
||||
export type StorageReferenceSourceId = (typeof STORAGE_REFERENCE_SOURCE_IDS)[number];
|
||||
|
||||
/**
|
||||
* The categories a pass may delete from, at their root level only (the
|
||||
* `files/assistant/` and `journey/thumbs/` subtrees have sweeps of their own).
|
||||
* A source whose columns hold free-form upload paths (a cover, an image url,
|
||||
* a photo path) claims all of them, since such a value can point anywhere.
|
||||
*/
|
||||
export const STORAGE_RECLAIMABLE_CATEGORIES = ['files', 'journey', 'covers', 'avatars', 'places'] as const;
|
||||
export type StorageReclaimableCategory = (typeof STORAGE_RECLAIMABLE_CATEGORIES)[number];
|
||||
|
||||
/** A part of a category a source answers for: the keys starting with `prefix` ('' is the whole category). */
|
||||
export interface StorageReferenceClaim {
|
||||
readonly category: ServedCategory;
|
||||
readonly prefix: string;
|
||||
}
|
||||
|
||||
export interface StorageReferenceSource {
|
||||
readonly id: StorageReferenceSourceId;
|
||||
/**
|
||||
* Every category the source's columns can point into, not only the one they
|
||||
* usually do: a trip cover may name `/uploads/places/x`, and that object
|
||||
* must stay. A source asked about a category it does not claim is not asked.
|
||||
*/
|
||||
readonly claims: ReadonlyArray<StorageReferenceClaim>;
|
||||
/**
|
||||
* Of these category-relative keys (as `StorageService.list` yields them, at
|
||||
* most a page of 500), the ones a row still names. A row that is trashed,
|
||||
* pending or otherwise not gone counts. Throwing stops the pass for the
|
||||
* category; it never deletes on a failed answer.
|
||||
*/
|
||||
referenced(category: ServedCategory, keys: readonly string[]): Promise<Set<string>>;
|
||||
}
|
||||
|
||||
/** One object a stored value names. */
|
||||
export interface StorageKeyRef {
|
||||
category: string;
|
||||
key: string;
|
||||
}
|
||||
|
||||
/** What a column's value can mean besides the `/uploads/<category>/<key>` and `<category>/<key>` spellings. */
|
||||
export interface StorageKeyOptions {
|
||||
/** The column holds a bare name of this category (`users.avatar`, `trip_files.filename`). */
|
||||
bare?: ServedCategory;
|
||||
/** Legacy rows may carry any path before the name; the last segment names an object of `bare` too. */
|
||||
basename?: boolean;
|
||||
}
|
||||
|
||||
const UPLOADS_PATH = /(?:^|\/)uploads\/([a-z0-9][a-z0-9-]*)\//gi;
|
||||
const KEY_END = /[?#"'<>\r\n]/;
|
||||
const SERVED = new Set<string>(SERVED_CATEGORIES);
|
||||
|
||||
/**
|
||||
* Every `/uploads/<category>/<key>` in a value, wherever it stands: a path, an
|
||||
* absolute URL of this instance (`https://host/uploads/covers/x.jpg`), a leading
|
||||
* `uploads/` without the slash, several in one text. A query or a fragment is
|
||||
* dropped, the key is percent-decoded and trimmed. A key with a blank in it is
|
||||
* reported both whole and cut at the blank, since a value may be a sentence:
|
||||
* reporting one key too many only keeps an object, one too few would lose it.
|
||||
*/
|
||||
export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] {
|
||||
if (typeof value !== 'string' || value === '') return [];
|
||||
const matches = [...value.matchAll(UPLOADS_PATH)];
|
||||
return matches.flatMap((match, i) => {
|
||||
const start = match.index + match[0].length;
|
||||
const next = matches[i + 1];
|
||||
const tail = value.slice(start, next ? next.index : value.length);
|
||||
const end = tail.search(KEY_END);
|
||||
const category = match[1]!.toLowerCase();
|
||||
return keyVariants(end === -1 ? tail : tail.slice(0, end)).map((key) => ({ category, key }));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Every object a stored value can name: the upload paths above, the relative
|
||||
* `<category>/<key>` spelling (`journeys.cover_image` holds `covers/x`), and
|
||||
* with `options` the bare or legacy-prefixed name of one category. A value
|
||||
* that names nothing this way (an https URL elsewhere, a data: URL, garbage)
|
||||
* answers an empty list: it references nothing a pass can find.
|
||||
*/
|
||||
export function storageKeysIn(value: string | null | undefined, options: StorageKeyOptions = {}): StorageKeyRef[] {
|
||||
if (typeof value !== 'string') return [];
|
||||
const trimmed = stripSuffix(value).trim();
|
||||
if (trimmed === '') return [];
|
||||
const refs = uploadKeysIn(value);
|
||||
const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/i.exec(trimmed);
|
||||
if (relative && SERVED.has(relative[1]!.toLowerCase())) {
|
||||
const category = relative[1]!.toLowerCase();
|
||||
for (const key of keyVariants(relative[2]!)) refs.push({ category, key });
|
||||
}
|
||||
if (options.bare) {
|
||||
const bare = options.bare;
|
||||
if (!/[/\\:]/.test(trimmed)) {
|
||||
for (const key of keyVariants(trimmed)) refs.push({ category: bare, key });
|
||||
} else if (options.basename) {
|
||||
const last = trimmed.slice(Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')) + 1);
|
||||
for (const key of keyVariants(last)) refs.push({ category: bare, key });
|
||||
}
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
/**
|
||||
* The candidates of `category` that any of `values` names, compared without
|
||||
* regard to case: a local volume on Windows or macOS serves `A.jpg` for `a.jpg`,
|
||||
* so an object is kept when a value names it in either spelling. The helper a
|
||||
* source builds `referenced()` from, so no source parses with a regex of its own.
|
||||
*/
|
||||
export function referencedAmong(
|
||||
category: ServedCategory,
|
||||
candidates: readonly string[],
|
||||
values: Iterable<string | null | undefined>,
|
||||
options: StorageKeyOptions = {},
|
||||
): Set<string> {
|
||||
const named = new Set<string>();
|
||||
for (const value of values) {
|
||||
for (const ref of storageKeysIn(value, options)) {
|
||||
if (ref.category === category) named.add(ref.key.toLowerCase());
|
||||
}
|
||||
}
|
||||
return new Set(candidates.filter((key) => named.has(key.toLowerCase())));
|
||||
}
|
||||
|
||||
function stripSuffix(value: string): string {
|
||||
const cut = value.search(/[?#]/);
|
||||
return cut === -1 ? value : value.slice(0, cut);
|
||||
}
|
||||
|
||||
/** The raw key, decoded, and cut at a blank; never an empty key or one that climbs out of its category. */
|
||||
function keyVariants(raw: string): string[] {
|
||||
const forms = new Set<string>();
|
||||
const add = (key: string) => {
|
||||
const clean = key.trim().replace(/^\/+/, '');
|
||||
if (clean === '' || clean.split('/').some((segment) => segment === '..' || segment === '.')) return;
|
||||
forms.add(clean);
|
||||
};
|
||||
const whole = stripSuffix(raw);
|
||||
for (const form of [whole, decoded(whole)]) {
|
||||
add(form);
|
||||
add(form.trim().split(/\s/)[0]!);
|
||||
}
|
||||
return [...forms];
|
||||
}
|
||||
|
||||
function decoded(value: string): string {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,17 @@
|
||||
import { AppSettings } from '../../db/entities/AppSettings.entity';
|
||||
import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity';
|
||||
import { AppConfigModule } from '../app-config/app-config.module';
|
||||
import { AuditModule } from '../audit/audit.module';
|
||||
import { DatabaseLifecycleModule } from '../database/database-lifecycle.module';
|
||||
import { SchedulingModule } from '../scheduling/scheduling.module';
|
||||
import { StorageAdminController } from './storage-admin.controller';
|
||||
import { StorageAdminService } from './storage-admin.service';
|
||||
import { StorageEventsService } from './storage-events.service';
|
||||
import { StorageJobsService } from './storage-jobs.service';
|
||||
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
|
||||
import { StorageReclaimController } from './storage-reclaim.controller';
|
||||
import { StorageReclaimJob } from './storage-reclaim.job';
|
||||
import { StorageReclaimService } from './storage-reclaim.service';
|
||||
import { StorageRegistryService } from './storage-registry.service';
|
||||
import { StorageStatsService } from './storage-stats.service';
|
||||
import { StorageUsageScanJob } from './storage-usage-scan.job';
|
||||
@@ -27,11 +33,20 @@ import { Module } from '@nestjs/common';
|
||||
* may cache drivers or trigger reloads. MikroOrmModule.forFeature registers
|
||||
* AppSettingsRepository for StorageRegistryService/StorageAdminService/
|
||||
* StorageStatsService's @InjectRepository (Plan 3i, storage task — SR/SA/SS's
|
||||
* 14 call sites, all on app_settings).
|
||||
* 14 call sites, all on app_settings). The reclaim pass adds its marks table
|
||||
* and follows database reopens (DatabaseLifecycleModule); the domains whose
|
||||
* rows name stored objects register their reference sources with the exported
|
||||
* StorageReclaimRegistryService, so storage never imports them.
|
||||
*/
|
||||
@Module({
|
||||
imports: [AppConfigModule, AuditModule, SchedulingModule, MikroOrmModule.forFeature([AppSettings])],
|
||||
controllers: [StorageAdminController],
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
AuditModule,
|
||||
DatabaseLifecycleModule,
|
||||
SchedulingModule,
|
||||
MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks]),
|
||||
],
|
||||
controllers: [StorageAdminController, StorageReclaimController],
|
||||
providers: [
|
||||
StorageRegistryService,
|
||||
StorageService,
|
||||
@@ -40,7 +55,10 @@ import { Module } from '@nestjs/common';
|
||||
StorageJobsService,
|
||||
StorageStatsService,
|
||||
StorageUsageScanJob,
|
||||
StorageReclaimRegistryService,
|
||||
StorageReclaimService,
|
||||
StorageReclaimJob,
|
||||
],
|
||||
exports: [StorageService, StorageEventsService],
|
||||
exports: [StorageService, StorageEventsService, StorageReclaimRegistryService],
|
||||
})
|
||||
export class StorageModule {}
|
||||
|
||||
Reference in New Issue
Block a user