diff --git a/server/src/db/repositories/StorageReclaimMarks.repository.ts b/server/src/db/repositories/StorageReclaimMarks.repository.ts index 1bedc7986..dc39004c1 100644 --- a/server/src/db/repositories/StorageReclaimMarks.repository.ts +++ b/server/src/db/repositories/StorageReclaimMarks.repository.ts @@ -56,8 +56,8 @@ export class StorageReclaimMarksRepository extends TrekRepository { - return await this.nativeDelete({ category }); + /** Drops every mark, for a database a restore brought back: its marks are another history's. */ + async deleteAll(): Promise { + return await this.nativeDelete({}); } } diff --git a/server/src/nest/audit/audit.service.ts b/server/src/nest/audit/audit.service.ts index 38cceab26..157b29003 100644 --- a/server/src/nest/audit/audit.service.ts +++ b/server/src/nest/audit/audit.service.ts @@ -32,7 +32,9 @@ const ACTION_LABELS: Record = { 'admin.storage_backfill_cancel': 'cancelled a storage sync', 'admin.storage_stats_refresh': 'refreshed storage usage stats', 'admin.storage_reclaim_update': 'switched storage reclaim deletion', - 'admin.storage_reclaim_run': 'ran a storage reclaim pass', + 'admin.storage_reclaim_run': 'started a storage reclaim check', + 'admin.storage_reclaim_confirm': 'confirmed deleting a large share of unreferenced storage objects', + 'storage.reclaim_deleted': 'deleted unreferenced storage objects', 'admin.places_contribute_decision': 'decided on sharing place details with TREK Places', 'admin.places_contribute_withdraw': 'withdrew the place details this installation shared', 'immich.private_ip_configured': 'configured Immich with private IP', diff --git a/server/src/nest/auth/auth-storage-refs.service.ts b/server/src/nest/auth/auth-storage-refs.service.ts index 9af40df6c..7f7022095 100644 --- a/server/src/nest/auth/auth-storage-refs.service.ts +++ b/server/src/nest/auth/auth-storage-refs.service.ts @@ -1,5 +1,5 @@ import { Users } from '../../db/entities/Users.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { Injectable } from '@nestjs/common'; /** @@ -14,8 +14,10 @@ export class AuthStorageRefsService extends StorageReclaimSource { readonly claims = [{ category: 'avatars', prefix: '' }] as const; protected override readonly keyOptions = { bare: 'avatars' } as const; - protected async storedValues(): Promise> { - const rows = await this.em.getRepository(Users).find({ avatar: { $ne: null } }, { fields: ['avatar'] }); + protected async storedValues(containing: readonly string[] | null): Promise> { + const rows = await this.em + .getRepository(Users) + .find(likeAny(['avatar'], containing), { fields: ['avatar'] }); return rows.map((row) => row.avatar); } } diff --git a/server/src/nest/backup/backup.impl.ts b/server/src/nest/backup/backup.impl.ts index 2b59032b6..004c10a68 100644 --- a/server/src/nest/backup/backup.impl.ts +++ b/server/src/nest/backup/backup.impl.ts @@ -9,6 +9,7 @@ import { invalidatePermissionsCache } from '../permissions/permissions-cache'; import { snapshotAllPluginDataDbs } from '../plugins/host/plugin-data.service'; import { pluginsCodeRoot, pluginsDataRoot } from '../plugins/paths'; import { stageExtractedPluginTrees, applyStagedRestoreNow } from '../plugins/plugin-backup'; +import { storageOwnerMarkerInstall } from '../storage/storage-reclaim.types'; import type { StorageService } from '../storage/storage.service'; import { StorageInvalidKeyError } from '../storage/storage.types'; import { VALID_INTERVALS } from './auto-backup.settings'; @@ -217,6 +218,10 @@ export async function createBackup( // In mode A the google/trek caches nest under the photos/ prefix — the // category walk would sweep them back in without this skip. if (category === 'photos' && (obj.key.startsWith('google/') || obj.key.startsWith('trek/'))) continue; + // The storage reclaim pass's owner marker names the install that wrote + // it; restored elsewhere, it would make that install's own uploads look + // shared with a foreign one. + if (storageOwnerMarkerInstall(obj.key) !== null) continue; // Local path available (exists on disk right now — the fail-safe half // of getLocalPathOrNull's contract) → push it directly: zero-copy, the // default-install path. Otherwise (a remote/S3 primary, or a local diff --git a/server/src/nest/backup/backup.service.ts b/server/src/nest/backup/backup.service.ts index a7963f6c9..54711f3f9 100644 --- a/server/src/nest/backup/backup.service.ts +++ b/server/src/nest/backup/backup.service.ts @@ -32,8 +32,14 @@ export class BackupService { listBackups() { return svc.listBackups(this.storage); } - createBackup(prefix?: 'backup' | 'auto-backup') { - return svc.createBackup(this.deps, prefix); + /** Marked running while it reads the uploads, so the storage reclaim pass deletes nothing it still archives. */ + async createBackup(prefix?: 'backup' | 'auto-backup') { + this.jobs.beginBackup(); + try { + return await svc.createBackup(this.deps, prefix); + } finally { + this.jobs.endBackup(); + } } restoreFromZip(zipPath: string) { return this.whileRestoring(() => svc.restoreFromZip(this.deps, zipPath)); diff --git a/server/src/nest/collections/collections-storage-refs.service.ts b/server/src/nest/collections/collections-storage-refs.service.ts index 93a8e89f6..2384b3a4a 100644 --- a/server/src/nest/collections/collections-storage-refs.service.ts +++ b/server/src/nest/collections/collections-storage-refs.service.ts @@ -1,6 +1,6 @@ import { CollectionPlaces } from '../../db/entities/CollectionPlaces.entity'; import { Collections } from '../../db/entities/Collections.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -16,13 +16,13 @@ export class CollectionsStorageRefsService extends StorageReclaimSource { readonly id = 'collections'; readonly claims = EVERY_RECLAIMABLE_CATEGORY; - protected async storedValues(): Promise> { + protected async storedValues(containing: readonly string[] | null): Promise> { const covers = await this.em .getRepository(Collections) - .find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + .find(likeAny(['cover_image'], containing), { fields: ['cover_image'] }); const images = await this.em .getRepository(CollectionPlaces) - .find({ image_url: { $ne: null } }, { fields: ['image_url'] }); + .find(likeAny(['image_url'], containing), { fields: ['image_url'] }); return [...covers.map((row) => row.cover_image), ...images.map((row) => row.image_url)]; } } diff --git a/server/src/nest/common/validate-managed-routes.ts b/server/src/nest/common/validate-managed-routes.ts index 1a6232f27..a2b0ce0b4 100644 --- a/server/src/nest/common/validate-managed-routes.ts +++ b/server/src/nest/common/validate-managed-routes.ts @@ -134,6 +134,7 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [ 'StorageAdminController.statsRefresh', 'StorageAdminController.test', 'StorageAdminController.update', + 'StorageReclaimController.confirm', 'StorageReclaimController.run', 'StorageReclaimController.update', ]; diff --git a/server/src/nest/files/files-storage-refs.service.ts b/server/src/nest/files/files-storage-refs.service.ts index 83df3de53..908c4211a 100644 --- a/server/src/nest/files/files-storage-refs.service.ts +++ b/server/src/nest/files/files-storage-refs.service.ts @@ -1,5 +1,5 @@ import { TripFiles } from '../../db/entities/TripFiles.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { Injectable } from '@nestjs/common'; /** @@ -19,8 +19,10 @@ export class FilesStorageRefsService extends StorageReclaimSource { readonly claims = [{ category: 'files', prefix: '' }] as const; protected override readonly keyOptions = { bare: 'files', basename: true } as const; - protected async storedValues(): Promise> { - const rows = await this.em.getRepository(TripFiles).find({}, { fields: ['filename'] }); + protected async storedValues(containing: readonly string[] | null): Promise> { + const rows = await this.em + .getRepository(TripFiles) + .find(likeAny(['filename'], containing), { fields: ['filename'] }); return rows.map((row) => row.filename); } } diff --git a/server/src/nest/journey/journey-storage-refs.service.ts b/server/src/nest/journey/journey-storage-refs.service.ts index a591c246a..71a8bd184 100644 --- a/server/src/nest/journey/journey-storage-refs.service.ts +++ b/server/src/nest/journey/journey-storage-refs.service.ts @@ -1,5 +1,5 @@ import { Journeys } from '../../db/entities/Journeys.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -15,10 +15,10 @@ export class JourneyStorageRefsService extends StorageReclaimSource { readonly id = 'journey'; readonly claims = EVERY_RECLAIMABLE_CATEGORY; - protected async storedValues(): Promise> { + protected async storedValues(containing: readonly string[] | null): Promise> { const rows = await this.em .getRepository(Journeys) - .find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + .find(likeAny(['cover_image'], containing), { fields: ['cover_image'] }); return rows.map((row) => row.cover_image); } } diff --git a/server/src/nest/photos/photos-storage-refs.service.ts b/server/src/nest/photos/photos-storage-refs.service.ts index b8f8ad992..3eec9cbd9 100644 --- a/server/src/nest/photos/photos-storage-refs.service.ts +++ b/server/src/nest/photos/photos-storage-refs.service.ts @@ -1,5 +1,5 @@ import { TrekPhotos } from '../../db/entities/TrekPhotos.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -20,13 +20,12 @@ export class PhotosStorageRefsService extends StorageReclaimSource { readonly claims = EVERY_RECLAIMABLE_CATEGORY; protected override readonly keyOptions = { bare: 'journey', basename: true } as const; - protected async storedValues(): Promise> { + protected async storedValues(containing: readonly string[] | null): Promise> { const rows = await this.em .getRepository(TrekPhotos) - .find( - { $or: [{ file_path: { $ne: null } }, { thumbnail_path: { $ne: null } }] }, - { fields: ['file_path', 'thumbnail_path'] }, - ); + .find(likeAny(['file_path', 'thumbnail_path'], containing), { + fields: ['file_path', 'thumbnail_path'], + }); return rows.flatMap((row) => [row.file_path, row.thumbnail_path]); } } diff --git a/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts b/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts index 36da56c41..970c6016e 100644 --- a/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts +++ b/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts @@ -1,5 +1,5 @@ import { PlaceContribOutbox } from '../../db/entities/PlaceContribOutbox.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -15,8 +15,10 @@ export class PlaceContribStorageRefsService extends StorageReclaimSource { readonly id = 'place-contrib'; readonly claims = EVERY_RECLAIMABLE_CATEGORY; - protected async storedValues(): Promise> { - const rows = await this.em.getRepository(PlaceContribOutbox).find({}, { fields: ['value'] }); + protected async storedValues(containing: readonly string[] | null): Promise> { + const rows = await this.em + .getRepository(PlaceContribOutbox) + .find(likeAny(['value'], containing), { fields: ['value'] }); return rows.map((row) => row.value); } } diff --git a/server/src/nest/places/places-storage-refs.service.ts b/server/src/nest/places/places-storage-refs.service.ts index 739daa7b4..6e461e1f8 100644 --- a/server/src/nest/places/places-storage-refs.service.ts +++ b/server/src/nest/places/places-storage-refs.service.ts @@ -1,5 +1,5 @@ import { Places } from '../../db/entities/Places.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -16,8 +16,10 @@ export class PlacesStorageRefsService extends StorageReclaimSource { readonly id = 'places'; readonly claims = EVERY_RECLAIMABLE_CATEGORY; - protected async storedValues(): Promise> { - const rows = await this.em.getRepository(Places).find({ image_url: { $ne: null } }, { fields: ['image_url'] }); + protected async storedValues(containing: readonly string[] | null): Promise> { + const rows = await this.em + .getRepository(Places) + .find(likeAny(['image_url'], containing), { fields: ['image_url'] }); return rows.map((row) => row.image_url); } } diff --git a/server/src/nest/storage/drivers/local.driver.ts b/server/src/nest/storage/drivers/local.driver.ts index 97b34be3d..02b0b3f65 100644 --- a/server/src/nest/storage/drivers/local.driver.ts +++ b/server/src/nest/storage/drivers/local.driver.ts @@ -5,6 +5,7 @@ import { StorageInvalidKeyError, StorageNotFoundError, type ByteRange, + type ListOptions, type LocalTempFile, type ObjectStat, type StorageDriver, @@ -175,13 +176,13 @@ export class LocalDriver implements StorageDriver { } } - async *list(prefix: string): AsyncIterable { + async *list(prefix: string, opts: ListOptions = {}): AsyncIterable { assertValidPrefix(prefix); const root = this.root(); - yield* this.walk(prefix ? path.resolve(root, prefix) : root, root); + yield* this.walk(prefix ? path.resolve(root, prefix) : root, root, opts.shallow === true); } - private async *walk(dir: string, root: string): AsyncIterable { + private async *walk(dir: string, root: string, shallow: boolean): AsyncIterable { let entries: fs.Dirent[]; try { entries = await fs.promises.readdir(dir, { withFileTypes: true }); @@ -195,7 +196,7 @@ export class LocalDriver implements StorageDriver { if (entry.name.startsWith('.')) continue; const entryPath = path.join(dir, entry.name); if (entry.isDirectory()) { - yield* this.walk(entryPath, root); + if (!shallow) yield* this.walk(entryPath, root, false); } else if (entry.isFile()) { const st = await fs.promises.stat(entryPath); yield { diff --git a/server/src/nest/storage/drivers/mirror.driver.ts b/server/src/nest/storage/drivers/mirror.driver.ts index a13d98b9c..1f073fe3c 100644 --- a/server/src/nest/storage/drivers/mirror.driver.ts +++ b/server/src/nest/storage/drivers/mirror.driver.ts @@ -6,6 +6,7 @@ import { StorageInvalidKeyError, StorageNotFoundError, type ByteRange, + type ListOptions, type LocalTempFile, type ObjectStat, type PutOptions, @@ -195,11 +196,11 @@ export class MirrorDriver implements StorageDriver { } } - async *list(prefix: string): AsyncIterable { + async *list(prefix: string, opts: ListOptions = {}): AsyncIterable { assertValidPrefix(prefix); let yieldedAny = false; try { - for await (const stat of this.primary.list(prefix)) { + for await (const stat of this.primary.list(prefix, opts)) { yieldedAny = true; yield stat; } @@ -210,7 +211,7 @@ export class MirrorDriver implements StorageDriver { if (yieldedAny || !shouldFallback(err)) throw err; for (const replica of this.replicas) { try { - for await (const stat of replica.list(prefix)) yield stat; + for await (const stat of replica.list(prefix, opts)) yield stat; return; } catch { /* try next replica */ diff --git a/server/src/nest/storage/drivers/s3.driver.ts b/server/src/nest/storage/drivers/s3.driver.ts index d423916d5..b010868a5 100644 --- a/server/src/nest/storage/drivers/s3.driver.ts +++ b/server/src/nest/storage/drivers/s3.driver.ts @@ -5,6 +5,7 @@ import { StorageNotFoundError, isLocalTempFile, type ByteRange, + type ListOptions, type LocalTempFile, type ObjectStat, type PutOptions, @@ -403,13 +404,15 @@ export class S3Driver implements StorageDriver { } } - async *list(prefix: string): AsyncIterable { + async *list(prefix: string, opts: ListOptions = {}): AsyncIterable { assertValidPrefix(prefix); const s3 = await this.client(); + // A delimiter makes S3 fold every deeper key into CommonPrefixes, which are not read. + const shallow = opts.shallow === true ? { Delimiter: '/' } : {}; let pages: AsyncIterable<{ Contents?: Array<{ Key?: string; Size?: number; LastModified?: Date }> }>; try { pages = await this.deadlined( - s3.ListObjectsV2({ Bucket: this.bucket, Prefix: this.keyPrefix + prefix, paginate: 'iterator' }), + s3.ListObjectsV2({ Bucket: this.bucket, Prefix: this.keyPrefix + prefix, paginate: 'iterator', ...shallow }), `list '${prefix}'`, ); } catch (err) { diff --git a/server/src/nest/storage/storage-admin.dto.ts b/server/src/nest/storage/storage-admin.dto.ts index 834c5c91b..5d4493a34 100644 --- a/server/src/nest/storage/storage-admin.dto.ts +++ b/server/src/nest/storage/storage-admin.dto.ts @@ -1,7 +1,7 @@ import { storageConfigPutSchema, storageMigrationRequestSchema, - storageReclaimRunRequestSchema, + storageReclaimConfirmRequestSchema, storageReclaimSettingsRequestSchema, storageTestRequestSchema, } from '@trek/shared'; @@ -19,4 +19,4 @@ export class StorageConfigDto extends createZodDto(storageConfigPutSchema) {} export class StorageTestRequestDto extends createZodDto(storageTestRequestSchema) {} export class StorageMigrationRequestDto extends createZodDto(storageMigrationRequestSchema) {} export class StorageReclaimSettingsDto extends createZodDto(storageReclaimSettingsRequestSchema) {} -export class StorageReclaimRunDto extends createZodDto(storageReclaimRunRequestSchema) {} +export class StorageReclaimConfirmDto extends createZodDto(storageReclaimConfirmRequestSchema) {} diff --git a/server/src/nest/storage/storage-admin.service.ts b/server/src/nest/storage/storage-admin.service.ts index 441e12493..076315914 100644 --- a/server/src/nest/storage/storage-admin.service.ts +++ b/server/src/nest/storage/storage-admin.service.ts @@ -4,7 +4,7 @@ import { UnitOfWork } from '../database/unit-of-work'; import { StorageJobsService } from './storage-jobs.service'; import { getSeedConfigPath } from './storage-paths'; import { ephemeralDriverFor, probeDriver, type ProbeTargetResult } from './storage-probe'; -import { readReclaimState } from './storage-reclaim.settings'; +import { StorageReclaimService } from './storage-reclaim.service'; import { BACKENDS_KEY, CATEGORIES_KEY, VERSION_KEY, StorageRegistryService } from './storage-registry.service'; import { assertNoMaskSentinels, @@ -44,6 +44,7 @@ export class StorageAdminService { private readonly jobs: StorageJobsService, private readonly stats: StorageStatsService, private readonly uow: UnitOfWork, + private readonly reclaim: StorageReclaimService, ) {} /** The effective world — secrets masked, categories cross-referenced per backend. */ @@ -64,7 +65,7 @@ export class StorageAdminService { health: { replicaFailures: this.storage.health().replicaFailures.map((f) => ({ ...f })) }, seedFilePresent: fs.existsSync(getSeedConfigPath()), usage: await this.stats.readUsage(), - reclaim: await readReclaimState(this.appSettings), + reclaim: await this.reclaim.state(), backfills: this.jobs.statuses(), migrations: this.jobs.migrationStatuses(), version: await this.registry.currentConfigVersion(), diff --git a/server/src/nest/storage/storage-jobs.service.ts b/server/src/nest/storage/storage-jobs.service.ts index 8c89929c0..595e1a17c 100644 --- a/server/src/nest/storage/storage-jobs.service.ts +++ b/server/src/nest/storage/storage-jobs.service.ts @@ -45,6 +45,8 @@ export class StorageJobsService { private exclusive: string | null = null; /** Restores in progress: no exclusive pass starts while one runs, and one in flight stops. */ private restores = 0; + /** Backups in progress: the reclaim pass deletes nothing while one reads the uploads. */ + private backups = 0; /** Settles once the exclusive pass holding the slot releases it; settled while none does. */ private exclusiveReleased: Promise = Promise.resolve(); private onExclusiveReleased: (() => void) | null = null; @@ -90,6 +92,33 @@ export class StorageJobsService { return this.restores > 0; } + /** True while a backup reads the uploads: the reclaim pass deletes nothing then, the archive may still want them. */ + get backupRunning(): boolean { + return this.backups > 0; + } + + /** The exclusive pass holding the one-job slot now, by name; null while none does. */ + get exclusiveHolder(): string | null { + return this.exclusive; + } + + /** + * Settles once no exclusive pass holds the slot (at once when none does): the + * nightly usage scan waits here, so its figures follow what a reclaim pass freed. + */ + async exclusiveIdle(): Promise { + await this.exclusiveReleased; + } + + /** A backup started reading the uploads; the caller calls `endBackup` in a finally. */ + beginBackup(): void { + this.backups += 1; + } + + endBackup(): void { + if (this.backups > 0) this.backups -= 1; + } + /** * Marks a restore as running and waits until an exclusive pass in flight * (the reclaim pass, which stops at its next check of `restoreRunning`) has diff --git a/server/src/nest/storage/storage-reclaim-source.service.ts b/server/src/nest/storage/storage-reclaim-source.service.ts index 09ec2812d..a80976f75 100644 --- a/server/src/nest/storage/storage-reclaim-source.service.ts +++ b/server/src/nest/storage/storage-reclaim-source.service.ts @@ -1,27 +1,32 @@ import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; import { - referencedAmong, + keySearchCore, + keysNamedIn, type StorageKeyOptions, type StorageReferenceClaim, type StorageReferenceSource, type StorageReferenceSourceId, } from './storage-reclaim.types'; import type { ServedCategory } from './storage.types'; -import { EntityManager } from '@mikro-orm/core'; +import { EntityManager, type FilterQuery } from '@mikro-orm/core'; import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; /** * The shape every owner's reference source shares: it registers itself with - * the reclaim registry once the app is up, and answers a page of keys by - * reading the values its own columns hold and parsing them with - * `referencedAmong`, so no source carries a parser of its own. + * the reclaim registry once the app is up, and answers by reading the values + * its own columns hold and parsing them with `keysNamedIn`, so no source + * carries a parser of its own. * * An owner extends this in its own domain, names its id and claims, and reads * its columns through `em.getRepository()`; storage never - * learns which tables those are. The values are read whole on every call: - * a column that takes any string (an absolute URL of this instance, a path - * with a query, another category's object) cannot be matched with `IN`, and - * a re-check right before a delete must see the rows as they are now. + * learns which tables those are. A column that takes any string (an absolute + * URL of this instance, a path with a query, another category's object) + * cannot be matched with `IN`, so the values are read and parsed here: + * - `namedKeys` reads every value once, for a pass that judges a whole category; + * - `referenced` reads only the rows whose value holds one of the keys' plain + * runs (`keySearchCore`, a `LIKE '%run%'` each), fresh, for the question a + * delete asks last and for a release right after a user's change. A key with + * no such run reads everything, as `namedKeys` does. */ @Injectable() export abstract class StorageReclaimSource implements StorageReferenceSource, OnApplicationBootstrap { @@ -39,11 +44,45 @@ export abstract class StorageReclaimSource implements StorageReferenceSource, On this.registry.register(this); } - async referenced(category: ServedCategory, keys: readonly string[]): Promise> { - if (keys.length === 0 || !this.claims.some((claim) => claim.category === category)) return new Set(); - return referencedAmong(category, keys, await this.storedValues(), this.keyOptions); + async namedKeys(category: ServedCategory): Promise> { + if (!this.claimsCategory(category)) return new Set(); + return keysNamedIn(category, await this.storedValues(null), this.keyOptions); } - /** Every value the owner's columns hold that may name a stored object; null and empty ones may be among them. */ - protected abstract storedValues(): Promise>; + async referenced(category: ServedCategory, keys: readonly string[]): Promise> { + if (keys.length === 0 || !this.claimsCategory(category)) return new Set(); + const cores = keys.map(keySearchCore); + const narrow = cores.every((core): core is string => core !== null) ? [...new Set(cores)] : null; + const named = keysNamedIn(category, await this.storedValues(narrow), this.keyOptions); + return new Set(keys.filter((key) => named.has(key.toLowerCase()))); + } + + /** + * The values the owner's columns hold that may name a stored object; null + * and empty ones may be among them. With `containing`, only the rows whose + * value holds one of these lowercase runs are needed (`likeAny` builds the + * filter); more rows are fine, fewer would lose an object. + */ + protected abstract storedValues(containing: readonly string[] | null): Promise>; + + private claimsCategory(category: ServedCategory): boolean { + return this.claims.some((claim) => claim.category === category); + } +} + +/** + * The filter a source narrows its columns with: rows where one of `fields` + * holds one of the runs, or every row where one of them is set when there are + * none to narrow by. The runs are `[a-z0-9-]` only, so they need no LIKE + * escaping; SQLite's LIKE ignores ASCII case, as the parser does. + */ +export function likeAny( + fields: ReadonlyArray, + containing: readonly string[] | null, +): FilterQuery { + const terms = + containing === null + ? fields.map((field) => ({ [field]: { $ne: null } })) + : fields.flatMap((field) => containing.map((run) => ({ [field]: { $like: `%${run}%` } }))); + return { $or: terms } as unknown as FilterQuery; } diff --git a/server/src/nest/storage/storage-reclaim.controller.ts b/server/src/nest/storage/storage-reclaim.controller.ts index 9a150af1e..6a92d4e6f 100644 --- a/server/src/nest/storage/storage-reclaim.controller.ts +++ b/server/src/nest/storage/storage-reclaim.controller.ts @@ -6,23 +6,19 @@ import { CurrentUser } from '../auth-core/current-user.decorator'; import { JwtAuthGuard } from '../auth-core/jwt-auth.guard'; import { ManagedForbidden } from '../common/managed'; import { ResponseContract } from '../common/response-contract'; -import { StorageReclaimRunDto, StorageReclaimSettingsDto } from './storage-admin.dto'; +import { StorageReclaimConfirmDto, StorageReclaimSettingsDto } from './storage-admin.dto'; import { StorageReclaimService } from './storage-reclaim.service'; import { Body, Controller, HttpCode, Post, Put, Req, UseGuards } from '@nestjs/common'; -import { - storageReclaimReportSchema, - storageReclaimStateSchema, - type StorageReclaimReport, - type StorageReclaimState, -} from '@trek/shared'; +import { storageReclaimRunStartedSchema, storageReclaimStateSchema, type StorageReclaimState } from '@trek/shared'; import type { Request } from 'express'; /** - * /api/admin/storage/reclaim: the admin's switch for the reclaim pass and a - * pass on demand. The state itself rides on GET /api/admin/storage. Deleting - * stays off until an admin turns it on here; a pass then still deletes only - * what has been unreferenced for a week (StorageReclaimService). + * /api/admin/storage/reclaim: the admin's switch for the reclaim pass, a check + * on demand and the confirmation a category with a large unreferenced share + * waits for. The state itself rides on GET /api/admin/storage. Deleting stays + * off until an admin turns it on here; a pass then still deletes only what has + * been unreferenced for a week (StorageReclaimService). */ @Controller('api/admin/storage/reclaim') @UseGuards(JwtAuthGuard, AdminGuard) @@ -50,27 +46,47 @@ export class StorageReclaimController { return state; } - /** One pass now, honouring the switch unless `reportOnly`. 409 on a demo or while another storage job runs. */ + /** + * Starts a report-only pass in the background and answers 202 at once; the + * admin state says `running` until its report lands. 409 on a demo or while + * another storage job, a reclaim pass included, runs. + */ @Post('run') - @HttpCode(200) - @ResponseContract(storageReclaimReportSchema) - async run( - @CurrentUser() user: User, - @Body() body: StorageReclaimRunDto, - @Req() req: Request, - ): Promise { - const report = await this.reclaim.run({ reportOnly: body.reportOnly === true }); - const rows = Object.values(report.categories); + @HttpCode(202) + @ResponseContract(storageReclaimRunStartedSchema) + async run(@CurrentUser() user: User, @Req() req: Request): Promise<{ started: true }> { + this.reclaim.start(); await this.audit.writeAudit({ userId: user.id, action: 'admin.storage_reclaim_run', ip: getClientIp(req), + details: { mode: 'report' }, + }); + return { started: true }; + } + + /** + * Lets the pass delete in a category whose unreferenced share is over a + * fifth, up to the count the last report showed. 409 when nothing waits. + */ + @Post('confirm') + @HttpCode(200) + @ResponseContract(storageReclaimStateSchema) + async confirm( + @CurrentUser() user: User, + @Body() body: StorageReclaimConfirmDto, + @Req() req: Request, + ): Promise { + const state = await this.reclaim.confirm(body.category); + await this.audit.writeAudit({ + userId: user.id, + action: 'admin.storage_reclaim_confirm', + ip: getClientIp(req), details: { - mode: report.mode, - deleted: rows.reduce((sum, row) => sum + row.deleted.objects, 0), - bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0), + category: body.category, + unreferenced: state.lastRun?.categories[body.category].unreferenced.objects ?? 0, }, }); - return report; + return state; } } diff --git a/server/src/nest/storage/storage-reclaim.job.ts b/server/src/nest/storage/storage-reclaim.job.ts index 406339247..7f907acc7 100644 --- a/server/src/nest/storage/storage-reclaim.job.ts +++ b/server/src/nest/storage/storage-reclaim.job.ts @@ -2,14 +2,15 @@ import { RuntimeEnvService } from '../app-config/runtime-env.service'; import { logDebug, logError, logInfo } from '../audit/audit-log.logger'; import { DomainError } from '../common/domain-error'; import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { reportSummary } from './storage-reclaim.report'; import { StorageReclaimService } from './storage-reclaim.service'; import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; -import type { StorageReclaimReport } from '@trek/shared'; /** - * The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00 and - * before the usage scan at 04:15, so the next usage figures already show what - * it freed. No pass at boot: a pass that may delete waits for its hour. + * The nightly reclaim pass, 04:05: after the thumbnail sweep at 04:00. The + * usage scan at 04:15 waits for a pass still running (StorageUsageScanJob), so + * its figures show what the pass freed. No pass at boot: a pass that may + * delete waits for its hour. A pass that deletes audits itself. */ @Injectable() export class StorageReclaimJob implements OnApplicationBootstrap { @@ -32,7 +33,7 @@ export class StorageReclaimJob implements OnApplicationBootstrap { if (this.runtimeEnv.isDemoMode()) return; try { const report = await this.reclaim.run({ now }); - const line = summary(report); + const line = reportSummary(report); if (line) logInfo(`Storage reclaim: ${line}`); } catch (err: unknown) { if (err instanceof DomainError && err.getStatus() === 409) { @@ -43,20 +44,3 @@ export class StorageReclaimJob implements OnApplicationBootstrap { } } } - -/** One line when the pass found, deleted or refused something; null when there was nothing to say. */ -function summary(report: StorageReclaimReport): string | null { - const rows = Object.values(report.categories); - const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0); - const deleted = rows.reduce((sum, row) => sum + row.deleted.objects, 0); - const bytes = rows.reduce((sum, row) => sum + row.deleted.bytes, 0); - const braked = Object.entries(report.categories) - .filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed') - .map(([category, row]) => `${category} (${row.skipped})`); - const parts: string[] = []; - if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`); - if (deleted > 0) parts.push(`deleted ${deleted} (${bytes} bytes)`); - if (braked.length > 0) parts.push(`skipped ${braked.join(', ')}`); - if (report.stopped) parts.push(`stopped: ${report.stopped}`); - return parts.length > 0 ? parts.join('; ') : null; -} diff --git a/server/src/nest/storage/storage-reclaim.ledger.ts b/server/src/nest/storage/storage-reclaim.ledger.ts index 36b0b7747..76822e420 100644 --- a/server/src/nest/storage/storage-reclaim.ledger.ts +++ b/server/src/nest/storage/storage-reclaim.ledger.ts @@ -1,7 +1,8 @@ import type { StorageReclaimableCategory, StorageReferenceSourceId } from './storage-reclaim.types'; -import type { BackendSnapshot } from './storage-registry.service'; -import type { ServedCategory } from './storage.types'; +import type { BackendSnapshot, StorageRegistryService } from './storage-registry.service'; +import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; +import fs from 'node:fs'; import path from 'node:path'; export type ReclaimRule = @@ -113,6 +114,17 @@ export function sharedLocationCategories( return shared; } +/** The categories whose resolved location overlaps another's, from the live registry. */ +export function sharedLocationsOf(registry: StorageRegistryService): Set { + const assignments = new Map( + SERVED_CATEGORIES.map((category) => { + const { backendName, keyPrefix } = registry.resolve(category); + return [category, { backendName, keyPrefix }] as const; + }), + ); + return sharedLocationCategories(assignments, registry.snapshot().backends); +} + function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap): string[] | null { const backend = backends.get(assignment.backendName); if (!backend) return null; @@ -120,34 +132,54 @@ function locationsOf(assignment: CategoryAssignment, backends: ReadonlyMap backends.get(String(name))) : [backend]; - const bases: string[] = []; + const locations: string[] = []; for (const member of members) { - const base = member ? baseOf(member) : null; - if (base === null) return null; - bases.push(base); + const location = member ? locationOf(member, assignment.keyPrefix) : null; + if (location === null) return null; + locations.push(location); } - return bases.map((base) => base + assignment.keyPrefix.toLowerCase()); + return locations; } /** - * Where a backend keeps its objects, lowercased: comparing more paths as equal - * than the volume does only excludes more, never less. + * Where a backend keeps a key prefix's objects, lowercased: comparing more + * paths as equal than the volume does only excludes more, never less. A local + * root is resolved through its symlinks as LocalDriver resolves it (the image + * links /app/server/uploads to /app/uploads), and the prefix's own folder too + * when it exists; a root that cannot be resolved reads as unknown, so shared. */ -function baseOf(backend: BackendSnapshot): string | null { +function locationOf(backend: BackendSnapshot, keyPrefix: string): string | null { const { options } = backend; if (backend.type === 'local') { if (typeof options.root !== 'string' || options.root === '') return null; - return `local:${trimSlashes(path.resolve(options.root).replace(/\\/g, '/'))}/`.toLowerCase(); + const root = realOrNull(path.resolve(options.root)); + if (root === null) return null; + const prefixed = keyPrefix === '' ? null : realOrNull(path.join(root, keyPrefix)); + const base = prefixed === null ? `${slashed(root)}${keyPrefix}` : slashed(prefixed); + return `local:/${trimSlashes(base)}${base.endsWith('/') ? '/' : ''}`.toLowerCase(); } if (backend.type === 's3') { if (typeof options.endpoint !== 'string' || typeof options.bucket !== 'string') return null; - const keyPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : ''; + const bucketPrefix = typeof options.keyPrefix === 'string' ? trimSlashes(options.keyPrefix) : ''; const base = `s3:${trimSlashes(options.endpoint)}/${options.bucket}/`; - return (keyPrefix === '' ? base : `${base}${keyPrefix}/`).toLowerCase(); + return `${bucketPrefix === '' ? base : `${base}${bucketPrefix}/`}${keyPrefix}`.toLowerCase(); } return null; } +function realOrNull(target: string): string | null { + try { + return fs.realpathSync(target); + } catch { + return null; + } +} + +/** A resolved folder with forward slashes and one trailing slash. */ +function slashed(folder: string): string { + return `${folder.replace(/\\/g, '/').replace(/\/+$/, '')}/`; +} + function asList(value: string | number | string[] | undefined): string[] { return Array.isArray(value) ? value : []; } diff --git a/server/src/nest/storage/storage-reclaim.report.ts b/server/src/nest/storage/storage-reclaim.report.ts new file mode 100644 index 000000000..e4f72f2d3 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim.report.ts @@ -0,0 +1,68 @@ +import type { StorageReclaimCategoryReport, StorageReclaimReport } from '@trek/shared'; + +/** One object a pass looks at. */ +export interface ReclaimCandidate { + key: string; + size: number; +} + +/** + * What a pass has found and done in one category so far. It is filled in as + * the pass goes, so a pass that stops part way still reports what it deleted. + */ +export interface CategoryProgress { + objects: number; + unreferenced: ReclaimCandidate[]; + eligible: ReclaimCandidate[]; + deleted: ReclaimCandidate[]; + confirmationRequired: boolean; + skipped: StorageReclaimCategoryReport['skipped']; +} + +export function newProgress(): CategoryProgress { + return { objects: 0, unreferenced: [], eligible: [], deleted: [], confirmationRequired: false, skipped: null }; +} + +export function reportOf(progress: CategoryProgress): StorageReclaimCategoryReport { + return { + objects: progress.objects, + unreferenced: countOf(progress.unreferenced), + eligible: countOf(progress.eligible), + deleted: countOf(progress.deleted), + confirmationRequired: progress.confirmationRequired, + skipped: progress.skipped, + }; +} + +function countOf(candidates: readonly ReclaimCandidate[]): { objects: number; bytes: number } { + return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) }; +} + +/** What a report says the pass deleted, over every category. */ +export function deletedTotal(report: StorageReclaimReport): { objects: number; bytes: number } { + const rows = Object.values(report.categories); + return { + objects: rows.reduce((sum, row) => sum + row.deleted.objects, 0), + bytes: rows.reduce((sum, row) => sum + row.deleted.bytes, 0), + }; +} + +/** One line when the pass found, deleted or refused something; null when there was nothing to say. */ +export function reportSummary(report: StorageReclaimReport): string | null { + const rows = Object.values(report.categories); + const unreferenced = rows.reduce((sum, row) => sum + row.unreferenced.objects, 0); + const deleted = deletedTotal(report); + const skipped = Object.entries(report.categories) + .filter(([, row]) => row.skipped !== null && row.skipped !== 'not_reclaimed') + .map(([category, row]) => `${category} (${row.skipped})`); + const waiting = Object.entries(report.categories) + .filter(([, row]) => row.confirmationRequired) + .map(([category]) => category); + const parts: string[] = []; + if (unreferenced > 0) parts.push(`${unreferenced} unreferenced object(s)`); + if (deleted.objects > 0) parts.push(`deleted ${deleted.objects} (${deleted.bytes} bytes)`); + if (waiting.length > 0) parts.push(`waiting for an admin to confirm ${waiting.join(', ')}`); + if (skipped.length > 0) parts.push(`skipped ${skipped.join(', ')}`); + if (report.stopped) parts.push(`stopped: ${report.stopped}`); + return parts.length > 0 ? parts.join('; ') : null; +} diff --git a/server/src/nest/storage/storage-reclaim.service.ts b/server/src/nest/storage/storage-reclaim.service.ts index 9d9b0205d..67c163b97 100644 --- a/server/src/nest/storage/storage-reclaim.service.ts +++ b/server/src/nest/storage/storage-reclaim.service.ts @@ -7,51 +7,71 @@ import type { } from '../../db/repositories/StorageReclaimMarks.repository'; import { dbNow, parseDbTimestamp } from '../../db/types'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -import { logError } from '../audit/audit-log.logger'; +import { logError, logInfo, logWarn } from '../audit/audit-log.logger'; +import { AuditService } from '../audit/audit.service'; import { DomainError } from '../common/domain-error'; import { DatabaseLifecycle } from '../database/database-lifecycle.service'; +import { withRequestContext } from '../database/request-context'; +import { UnitOfWork } from '../database/unit-of-work'; import { StorageJobsService } from './storage-jobs.service'; import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; -import { inReclaimScope, RECLAIM_LEDGER, sharedLocationCategories } from './storage-reclaim.ledger'; +import { inReclaimScope, RECLAIM_LEDGER, sharedLocationsOf } from './storage-reclaim.ledger'; import { + deletedTotal, + newProgress, + reportOf, + type CategoryProgress, + type ReclaimCandidate, +} from './storage-reclaim.report'; +import { + INSTALL_ID_KEY, RECLAIM_DELETE_KEY, + RECLAIM_DELETE_REPORT_KEY, RECLAIM_REPORT_KEY, + RECLAIM_RESET_KEYS, + readReclaimConfirmations, readReclaimDeleteEnabled, - readReclaimState, + readReclaimReport, + writeReclaimConfirmations, } from './storage-reclaim.settings'; -import type { StorageReferenceSource } from './storage-reclaim.types'; +import { storageOwnerMarkerInstall, storageOwnerMarkerKey, type StorageReferenceSource } from './storage-reclaim.types'; import { StorageRegistryService } from './storage-registry.service'; import { StorageService } from './storage.service'; import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; +import { MikroORM } from '@mikro-orm/core'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { Injectable, type OnModuleInit } from '@nestjs/common'; +import { Injectable, type OnApplicationBootstrap, type OnModuleInit } from '@nestjs/common'; import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared'; +import { randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; + const HOUR_MS = 60 * 60 * 1000; /** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */ const RECLAIM_GRACE_MS = 24 * HOUR_MS; /** How long an object must stay unreferenced, on every pass, before a pass may delete it. */ const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS; -/** More unreferenced than this share of a category points at a bucket another install shares: hands off. */ -const RECLAIM_BRAKE_SHARE = 0.2; -/** Keys per reference question, the page the sources are promised. */ -const SOURCE_PAGE = 500; +/** More unreferenced than this share of a category waits for an admin's confirmation before anything goes. */ +const CONFIRM_SHARE = 0.2; +/** Keys per re-check right before a delete: one fresh, narrowed question to every source. */ +const RECHECK_BATCH = 100; +/** Deletes in flight at once. */ +const DELETE_CONCURRENCY = 20; /** Marks per statement: four bound values each, well under SQLite's limit. */ const MARK_BATCH = 100; -/** Deletes in flight at once. */ -const DELETE_BATCH = 20; +/** Listed objects between two looks at whether a restore began, so a restore waits seconds, not a whole listing. */ +const LIST_CHECK_EVERY = 500; const JOB_NAME = 'a storage reclaim'; - -interface Candidate { - key: string; - size: number; -} +const INSTALL_ID = /^[a-z0-9-]{8,64}$/; interface PassContext { now: number; + installId: string; + /** Deletes may go now; turned off for the rest of the pass once the switch reads off or a backup starts. */ deleting: boolean; sources: readonly StorageReferenceSource[]; shared: ReadonlySet; + confirmed: Partial>; generation: number; } @@ -61,33 +81,44 @@ interface ReclaimRunOptions { now?: Date; } -/** - * A restore began, or the database was swapped under the pass: nothing it - * learnt before holds any more. Reported as `database_reopened` either way. - */ +/** A restore began, or the database was swapped under the pass: nothing it learnt before holds any more. */ class ReclaimStopped extends Error {} /** * The reclaim pass (#2924): objects in upload storage that no row names any * more, found in two phases and, only when the admin switched deletion on, - * deleted. A pass lists each category the ledger allows, skips objects younger - * than a day (the bytes of an upload land before its row), asks every - * registered reference source which keys are still named, and marks the rest - * with the time it first saw them. An object goes only once its mark is a week - * old, after one more question to every source about that single key, through - * `StorageService.delete` (so replicas follow and a demo keeps its bytes). + * deleted. A pass lists each category the ledger allows at its root, skips + * objects younger than a day (the bytes of an upload land before its row), + * reads once per category what every registered reference source names, and + * marks the rest with the time it first saw them. An object goes only once its + * mark is a week old, after one more fresh question to every source about its + * batch, through `StorageService.delete` (so replicas follow and a demo keeps + * its bytes). What a user deletes or replaces is released at once instead + * (StorageReleaseService); the pass is the net for whatever that left behind. * - * Brakes, each failing closed: a category whose sources are not all - * registered, whose objects share a location with another category's, whose - * listing or lookups fail, or where more than a fifth of the objects look - * unreferenced is skipped and its marks start over. A pass never runs beside - * a backfill, a category migration or a restore (the one-job slot of - * StorageJobsService, which a restore waits on), never on a demo instance, - * and stops when a restore begins or the database is reopened under it. + * Safety, each failing closed: + * - a category whose sources are not all registered, whose location overlaps + * another category's, or where another installation's owner marker lies + * (or this one's cannot be written) is not judged; + * - a failed listing or lookup skips the category for the night; no skip + * touches the marks, so one bad night does not restart the week; + * - more than a fifth unreferenced deletes nothing until an admin confirmed + * that count (`confirm`), which no user action can do; + * - a delete that fails on the backend stops deleting in that category; + * - nothing is deleted while a backup reads the uploads, once the switch is + * turned off mid-pass, or on a demo instance; + * - a pass never runs beside a backfill, a category migration or a restore + * (the one-job slot of StorageJobsService, which a restore waits on), and + * stops when a restore begins or the database is reopened under it. + * + * Every restore turns deletion off and drops the marks, the confirmations and + * the reports: the restored rows may come from another installation, and this + * one keeps its own install id (written back over the restored one). */ @Injectable() -export class StorageReclaimService implements OnModuleInit { +export class StorageReclaimService implements OnModuleInit, OnApplicationBootstrap { private generation = 0; + private knownInstallId: string | null = null; constructor( private readonly storage: StorageService, @@ -96,38 +127,66 @@ export class StorageReclaimService implements OnModuleInit { private readonly jobs: StorageJobsService, private readonly lifecycle: DatabaseLifecycle, private readonly runtimeEnv: RuntimeEnvService, + private readonly audit: AuditService, + private readonly uow: UnitOfWork, + private readonly orm: MikroORM, @InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository, @InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository, ) {} onModuleInit(): void { - this.lifecycle.onReopened(async () => { - this.generation += 1; - }); + this.lifecycle.onReopened(() => this.afterReopen()); } + /** Makes the install id on the first boot, so a restore later has one of this install's own to keep. */ + async onApplicationBootstrap(): Promise { + try { + await withRequestContext(this.orm, () => this.installId()); + } catch (err) { + logWarn(`[storage] could not read or make the install id: ${messageOf(err)}`); + } + } + + /** Read only, so the admin state's GET never writes: an install id not made yet reads as deletion off. */ async state(): Promise { - return await readReclaimState(this.appSettings); + const installId = this.knownInstallId ?? (await this.appSettings.getValue(INSTALL_ID_KEY)); + return { + deleteEnabled: installId !== null && (await readReclaimDeleteEnabled(this.appSettings, installId)), + running: this.jobs.exclusiveHolder === JOB_NAME, + lastRun: await readReclaimReport(this.appSettings, RECLAIM_REPORT_KEY), + lastDeleteRun: await readReclaimReport(this.appSettings, RECLAIM_DELETE_REPORT_KEY), + }; } async setDeleteEnabled(enabled: boolean): Promise { - await this.appSettings.setValue(RECLAIM_DELETE_KEY, enabled ? 'true' : 'false'); + if (enabled) await this.appSettings.upsertOrReplace(RECLAIM_DELETE_KEY, await this.installId()); + else await this.appSettings.deleteValue(RECLAIM_DELETE_KEY); return await this.state(); } /** - * One pass over every category, its report persisted and returned. Refuses - * with a 409 on a demo instance and while another storage job holds the slot. + * The admin lets the pass delete in a category whose unreferenced share is + * over a fifth, up to the count the last report showed. 409 when the last + * report holds nothing that waits for it. + */ + async confirm(category: ServedCategory): Promise { + const row = (await readReclaimReport(this.appSettings, RECLAIM_REPORT_KEY))?.categories[category]; + if (!row?.confirmationRequired) { + throw refusal('reclaim_nothing_to_confirm', 'nothing in this category waits for a confirmation'); + } + const installId = await this.installId(); + const counts = await readReclaimConfirmations(this.appSettings, installId); + await writeReclaimConfirmations(this.appSettings, installId, { ...counts, [category]: row.unreferenced.objects }); + return await this.state(); + } + + /** + * One pass over every category in the caller, its report persisted and + * returned: the nightly job's. Refuses with a 409 on a demo instance and + * while another storage job holds the slot. */ async run(options: ReclaimRunOptions = {}): Promise { - if (this.runtimeEnv.isDemoMode()) { - throw new DomainError(409, 'storage reclaim does not run on a demo instance', { code: 'reclaim_demo' }); - } - if (!this.jobs.claimExclusive(JOB_NAME)) { - throw new DomainError(409, 'another storage job is running, one storage job at a time', { - code: 'reclaim_busy', - }); - } + this.claimSlot(); try { return await this.pass(options.reportOnly === true, options.now ?? new Date()); } finally { @@ -135,152 +194,269 @@ export class StorageReclaimService implements OnModuleInit { } } + /** + * A report-only pass in the background, for the admin's check: it can take + * longer than a proxy waits for an answer. The same refusals as `run`, at + * once; the report lands on the admin state, which says `running` meanwhile. + */ + start(): void { + this.claimSlot(); + void withRequestContext(this.orm, () => this.pass(true, new Date())) + .catch((err: unknown) => { + logError(`[storage] the reclaim check failed: ${messageOf(err)}`); + }) + .finally(() => { + this.jobs.releaseExclusive(JOB_NAME); + }); + } + + private claimSlot(): void { + if (this.runtimeEnv.isDemoMode()) throw refusal('reclaim_demo', 'storage reclaim does not run on a demo instance'); + if (!this.jobs.claimExclusive(JOB_NAME)) { + throw refusal('reclaim_busy', 'another storage job is running, one storage job at a time'); + } + } + + /** + * This install's id, made and stored on first use. + * + * @txStandalone an idempotent lazy provision: the row is made once and only + * read after, whatever the caller writes besides. + */ + private async installId(): Promise { + if (this.knownInstallId) return this.knownInstallId; + let id = await this.appSettings.getValue(INSTALL_ID_KEY); + if (id === null || !INSTALL_ID.test(id)) { + id = randomUUID(); + await this.appSettings.upsertOrReplace(INSTALL_ID_KEY, id); + } + this.knownInstallId = id; + return id; + } + + /** + * A restore or a reopen swapped the database: a pass in flight stops, and + * the restored rows get this install's id back, deletion off and no marks, + * confirmations or reports. Contained: a listener that throws fails the reopen. + */ + private async afterReopen(): Promise { + this.generation += 1; + try { + await withRequestContext(this.orm, () => + this.uow.transactional(async () => { + const wasOn = this.knownInstallId !== null && (await this.appSettings.getValue(RECLAIM_DELETE_KEY)) !== null; + if (this.knownInstallId) await this.appSettings.upsertOrReplace(INSTALL_ID_KEY, this.knownInstallId); + await RECLAIM_RESET_KEYS.reduce>(async (previous, key) => { + await previous; + await this.appSettings.deleteValue(key); + }, Promise.resolve()); + await this.marks.deleteAll(); + if (wasOn) + logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on'); + }), + ); + } catch (err) { + logError(`[storage] could not reset the reclaim state after the database was reopened: ${messageOf(err)}`); + } + } + /** * @txIndependent a sweep in passes: each category's marks and each delete * stand on their own, and a pass cut short leaves the work done so far valid. */ private async pass(reportOnly: boolean, now: Date): Promise { const wallStart = Date.now(); + const installId = await this.installId(); + const deleting = !reportOnly && (await readReclaimDeleteEnabled(this.appSettings, installId)); const ctx: PassContext = { now: now.getTime(), - deleting: !reportOnly && (await readReclaimDeleteEnabled(this.appSettings)), + installId, + deleting, sources: [...this.references.sources()], - shared: this.sharedLocations(), + shared: sharedLocationsOf(this.registry), + confirmed: await readReclaimConfirmations(this.appSettings, installId), generation: this.generation, }; const categories = {} as Record; let stopped: StorageReclaimReport['stopped'] = null; await SERVED_CATEGORIES.reduce>(async (previous, category) => { await previous; + const progress = newProgress(); if (stopped === null) { try { - categories[category] = await this.reclaimCategory(category, ctx); - return; + await this.reclaimCategory(category, ctx, progress); } catch (err) { - if (!(err instanceof ReclaimStopped)) throw err; - stopped = 'database_reopened'; + stopped = err instanceof ReclaimStopped ? 'database_reopened' : 'failed'; + if (stopped === 'failed') logError(`[storage] the reclaim pass failed in '${category}': ${messageOf(err)}`); + progress.skipped = 'interrupted'; } + } else { + progress.skipped = 'interrupted'; } - categories[category] = emptyReport(null, 0); + categories[category] = reportOf(progress); }, Promise.resolve()); const report: StorageReclaimReport = { startedAt: ctx.now, finishedAt: ctx.now + (Date.now() - wallStart), - mode: ctx.deleting ? 'delete' : 'report', + mode: deleting ? 'delete' : 'report', stopped, categories, }; - await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, JSON.stringify(report)); + await this.persist(report); return report; } - /** The categories whose resolved location overlaps another's, from the live registry. */ - private sharedLocations(): Set { - const assignments = new Map( - SERVED_CATEGORIES.map((category) => { - const { backendName, keyPrefix } = this.registry.resolve(category); - return [category, { backendName, keyPrefix }] as const; - }), - ); - return sharedLocationCategories(assignments, this.registry.snapshot().backends); + /** + * The report as the last pass's, and as the last deleting pass's when it ran + * with deletion on; an audit entry (the system as actor) when it deleted. + * + * @txIndependent each row stands on its own: the report, the delete report + * and the audit entry describe one finished pass, whichever of them lands. + */ + private async persist(report: StorageReclaimReport): Promise { + const text = JSON.stringify(report); + await this.appSettings.upsertOrReplace(RECLAIM_REPORT_KEY, text); + if (report.mode !== 'delete') return; + await this.appSettings.upsertOrReplace(RECLAIM_DELETE_REPORT_KEY, text); + const deleted = deletedTotal(report); + if (deleted.objects === 0) return; + await this.audit.writeAudit({ + userId: null, + action: 'storage.reclaim_deleted', + details: { + deleted: deleted.objects, + bytes: deleted.bytes, + ...(report.stopped ? { stopped: report.stopped } : {}), + categories: Object.fromEntries( + Object.entries(report.categories) + .filter(([, row]) => row.deleted.objects > 0) + .map(([category, row]) => [category, row.deleted.objects]), + ), + }, + }); } /** + * Judges one category into `progress`, which the caller reports even when + * this throws part way. + * * @txIndependent a sweep in passes: marks are written per batch and every * delete stands alone, so a failure part way keeps what came before. */ - private async reclaimCategory(category: ServedCategory, ctx: PassContext): Promise { + private async reclaimCategory(category: ServedCategory, ctx: PassContext, progress: CategoryProgress): Promise { const rule = RECLAIM_LEDGER[category]; - if (!rule.reclaim) return emptyReport('not_reclaimed', 0); + if (!rule.reclaim) { + progress.skipped = 'not_reclaimed'; + return; + } const sources = ctx.sources.filter((source) => claimsAny(source, category)); const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id)); - if (rule.requiredSources.some((id) => !answering.has(id))) return this.skip(category, 'sources_missing', 0); - if (ctx.shared.has(category)) return this.skip(category, 'shared_location', 0); - - let listed: { objects: number; old: Candidate[] }; - try { - listed = await this.listOld(category, ctx.now); - } catch (err) { - logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`); - return this.skip(category, 'list_failed', 0); + if (rule.requiredSources.some((id) => !answering.has(id))) { + progress.skipped = 'sources_missing'; + return; } - let unreferenced: Candidate[]; + if (ctx.shared.has(category)) { + progress.skipped = 'shared_location'; + return; + } + + let listed: { objects: number; old: ReclaimCandidate[]; owners: Set }; try { - unreferenced = await this.unreferencedOf(category, listed.old, sources); + listed = await this.listOld(category, ctx); + } catch (err) { + if (err instanceof ReclaimStopped) throw err; + logError(`[storage] reclaim could not list '${category}': ${messageOf(err)}`); + progress.skipped = 'list_failed'; + return; + } + progress.objects = listed.objects; + const owner = await this.ownerVerdict(category, listed.owners, ctx.installId); + if (owner !== null) { + progress.skipped = owner; + return; + } + + let named: Set; + try { + named = await this.namedIn(category, sources); } catch (err) { logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`); - return this.skip(category, 'source_failed', listed.objects); + progress.skipped = 'source_failed'; + return; } this.assertSameDatabase(ctx); - const unreferencedCount = countOf(unreferenced); - if (unreferenced.length > listed.objects * RECLAIM_BRAKE_SHARE) { - return { - ...(await this.skip(category, 'too_many_unreferenced', listed.objects)), - unreferenced: unreferencedCount, - }; - } + const unreferenced = listed.old.filter((candidate) => !named.has(candidate.key.toLowerCase())); + progress.unreferenced = unreferenced; const marks = new Map((await this.marks.marksOf(category)).map((mark) => [mark.key, mark])); await this.writeMarks(category, unreferenced, marks, ctx.now); const cutoff = ctx.now - RECLAIM_DELAY_MS; - const eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff)); - const deleted = ctx.deleting ? await this.deleteEligible(category, eligible, sources, ctx) : []; - return { - objects: listed.objects, - unreferenced: unreferencedCount, - eligible: countOf(eligible), - deleted: countOf(deleted), - skipped: null, - }; + progress.eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff)); + progress.confirmationRequired = + unreferenced.length > listed.objects * CONFIRM_SHARE && unreferenced.length > (ctx.confirmed[category] ?? 0); + if (ctx.deleting && !progress.confirmationRequired) await this.deleteEligible(category, sources, ctx, progress); } - /** A category this pass cannot judge: nothing is deleted and its marks start over. */ - private async skip( + /** The root-level objects of a category, of them the ones past the grace period, and the owner markers there. */ + private async listOld( category: ServedCategory, - reason: NonNullable, - objects: number, - ): Promise { - await this.marks.deleteCategory(category); - return emptyReport(reason, objects); - } - - /** The root-level objects of a category, and of them the ones past the grace period. */ - private async listOld(category: ServedCategory, now: number): Promise<{ objects: number; old: Candidate[] }> { + ctx: PassContext, + ): Promise<{ objects: number; old: ReclaimCandidate[]; owners: Set }> { let objects = 0; - const old: Candidate[] = []; - for await (const stat of this.storage.list(category)) { + let seen = 0; + const old: ReclaimCandidate[] = []; + const owners = new Set(); + for await (const stat of this.storage.list(category, '', { shallow: true })) { + seen += 1; + if (seen % LIST_CHECK_EVERY === 0) this.assertSameDatabase(ctx); if (!inReclaimScope(stat.key)) continue; + const owner = storageOwnerMarkerInstall(stat.key); + if (owner !== null) { + owners.add(owner); + continue; + } objects += 1; // A missing or zero modification time reads as infinitely old: such an object is kept. - if (!(stat.mtimeMs > 0) || now - stat.mtimeMs < RECLAIM_GRACE_MS) continue; + if (!(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS) continue; old.push({ key: stat.key, size: stat.size }); } - return { objects, old }; - } - - /** The candidates no source names, asked a page at a time. */ - private async unreferencedOf( - category: ServedCategory, - candidates: readonly Candidate[], - sources: readonly StorageReferenceSource[], - ): Promise { - return chunk(candidates, SOURCE_PAGE).reduce>(async (previous, page) => { - const done = await previous; - const referenced = await this.referencedIn( - category, - page.map((candidate) => candidate.key), - sources, - ); - done.push(...page.filter((candidate) => !referenced.has(candidate.key.toLowerCase()))); - return done; - }, Promise.resolve([])); + return { objects, old, owners }; } /** - * The keys some source still names, lowercased. Every key is claimed: the - * ledger's required sources answer for the whole category, and a category - * missing one never gets this far. + * Whether this installation may judge the place a category lives in: not + * where another one's owner marker lies, and not where its own could not be + * written, since then a second installation cannot see it either. */ + private async ownerVerdict( + category: ServedCategory, + owners: ReadonlySet, + installId: string, + ): Promise<'foreign_install' | 'owner_unknown' | null> { + const foreign = [...owners].some((owner) => owner !== installId); + if (!owners.has(installId)) { + try { + await this.storage.put(category, storageOwnerMarkerKey(installId), Readable.from([installId]), { + contentType: 'text/plain', + }); + } catch (err) { + logWarn(`[storage] reclaim could not write its owner marker into '${category}': ${messageOf(err)}`); + return foreign ? 'foreign_install' : 'owner_unknown'; + } + } + if (foreign) logWarn(`[storage] another TREK installation keeps its uploads where '${category}' does: not judged`); + return foreign ? 'foreign_install' : null; + } + + /** Every key of the category some source names, lowercased: one read per source for the whole category. */ + private async namedIn(category: ServedCategory, sources: readonly StorageReferenceSource[]): Promise> { + const answers = await Promise.all(sources.map((source) => source.namedKeys(category))); + const named = new Set(); + for (const answer of answers) for (const key of answer) named.add(key); + return named; + } + + /** The keys some source names right now, lowercased: the fresh question before a delete. */ private async referencedIn( category: ServedCategory, keys: readonly string[], @@ -303,7 +479,7 @@ export class StorageReclaimService implements OnModuleInit { */ private async writeMarks( category: ServedCategory, - unreferenced: readonly Candidate[], + unreferenced: readonly ReclaimCandidate[], marks: ReadonlyMap, now: number, ): Promise { @@ -322,48 +498,68 @@ export class StorageReclaimService implements OnModuleInit { } /** + * The eligible objects a batch at a time, while deleting may still go; a + * batch whose backend failed stops the category with `delete_failed`. + * * @txIndependent a batch at a time, each delete standing alone, so a failure * leaves the others' work standing. */ private async deleteEligible( category: ServedCategory, - eligible: readonly Candidate[], sources: readonly StorageReferenceSource[], ctx: PassContext, - ): Promise { - return chunk(eligible, DELETE_BATCH).reduce>(async (previous, batch) => { - const done = await previous; - done.push(...(await this.deleteBatch(category, batch, sources, ctx))); - return done; - }, Promise.resolve([])); + progress: CategoryProgress, + ): Promise { + await chunk(progress.eligible, RECHECK_BATCH).reduce>(async (previous, batch) => { + await previous; + if (progress.skipped !== null || !(await this.mayDelete(ctx))) return; + this.assertSameDatabase(ctx); + const failed = await this.deleteBatch(category, batch, sources, ctx, progress); + if (failed) progress.skipped = 'delete_failed'; + }, Promise.resolve()); + } + + /** Deleting goes on while the switch stays on and no backup reads the uploads; once off, off for the pass. */ + private async mayDelete(ctx: PassContext): Promise { + if (!ctx.deleting) return false; + if (this.jobs.backupRunning || !(await readReclaimDeleteEnabled(this.appSettings, ctx.installId))) { + ctx.deleting = false; + } + return ctx.deleting; } /** - * Checks each object is still the old one the pass judged, then asks every - * source about the batch again and deletes, at once, what none of them names. - * The question goes last so nothing slower than the delete itself sits - * between the answer and the delete, and once per batch so a pass reads each - * source's columns once per batch rather than once per object. + * Checks each object is still the old one the pass judged, asks every source + * about the batch again, fresh, and deletes what none of them names. The + * question goes last so nothing slower than the deletes sits between the + * answer and them. True when the backend failed, so the category stops. * - * @txIndependent every object goes on its own, see deleteOne. + * @txIndependent every object goes on its own; its mark goes after it. */ private async deleteBatch( category: ServedCategory, - batch: readonly Candidate[], + batch: readonly ReclaimCandidate[], sources: readonly StorageReferenceSource[], ctx: PassContext, - ): Promise { - const stats = await Promise.all(batch.map((candidate) => this.storage.stat(category, candidate.key))); + progress: CategoryProgress, + ): Promise { + let stats: Awaited>[]; + try { + stats = await Promise.all(batch.map((candidate) => this.storage.stat(category, candidate.key))); + } catch (err) { + logError(`[storage] reclaim stopped deleting in '${category}', the backend failed: ${messageOf(err)}`); + return true; + } // Gone, or written again since it was judged: the next pass looks at it afresh. const changed = batch.filter((_candidate, i) => { const stat = stats[i]; - return stat === null || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS; + return !stat || !(stat.mtimeMs > 0) || ctx.now - stat.mtimeMs < RECLAIM_GRACE_MS; }); if (changed.length > 0) await this.marks.deleteKeys(category, keysOf(changed)); const due = batch.filter( (candidate) => !changed.includes(candidate) && !this.storage.keepsOnDelete(category, candidate.key), ); - if (due.length === 0) return []; + if (due.length === 0) return false; let referenced: Set; try { referenced = await this.referencedIn(category, keysOf(due), sources); @@ -371,30 +567,27 @@ export class StorageReclaimService implements OnModuleInit { logError( `[storage] reclaim kept ${due.length} '${category}' object(s), their re-check failed: ${messageOf(err)}`, ); - return []; + return false; } const named = due.filter((candidate) => referenced.has(candidate.key.toLowerCase())); if (named.length > 0) await this.marks.deleteKeys(category, keysOf(named)); const unnamed = due.filter((candidate) => !referenced.has(candidate.key.toLowerCase())); this.assertSameDatabase(ctx); - const results = await Promise.all(unnamed.map((candidate) => this.deleteOne(category, candidate))); - return unnamed.filter((_candidate, i) => results[i]); - } - - /** - * Deletes one object the batch cleared, then its mark. False keeps it. - * - * @txIndependent the object goes first and the mark after, so a mark never - * outlives a delete that failed. - */ - private async deleteOne(category: ServedCategory, candidate: Candidate): Promise { - try { - await this.storage.delete(category, candidate.key); - } catch (err) { - logError(`[storage] reclaim could not delete a '${category}' object: ${messageOf(err)}`); - return false; - } - await this.marks.deleteKeys(category, [candidate.key]); + const deleted: ReclaimCandidate[] = []; + let failure: unknown = null; + await chunk(unnamed, DELETE_CONCURRENCY).reduce>(async (previous, group) => { + await previous; + if (failure !== null) return; + const results = await Promise.allSettled(group.map((candidate) => this.storage.delete(category, candidate.key))); + results.forEach((result, i) => { + if (result.status === 'fulfilled') deleted.push(group[i] as ReclaimCandidate); + else failure ??= result.reason; + }); + }, Promise.resolve()); + progress.deleted.push(...deleted); + if (deleted.length > 0) await this.marks.deleteKeys(category, keysOf(deleted)); + if (failure === null) return false; + logError(`[storage] reclaim stopped deleting in '${category}' after a failed delete: ${messageOf(failure)}`); return true; } @@ -405,13 +598,9 @@ export class StorageReclaimService implements OnModuleInit { } } -function emptyReport(skipped: StorageReclaimCategoryReport['skipped'], objects: number): StorageReclaimCategoryReport { - const none = { objects: 0, bytes: 0 }; - return { objects, unreferenced: none, eligible: none, deleted: none, skipped }; -} - -function countOf(candidates: readonly Candidate[]): { objects: number; bytes: number } { - return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) }; +/** A 409 whose body names the refusal in `code` next to the text, so the client can say it in the user's language. */ +function refusal(code: 'reclaim_busy' | 'reclaim_demo' | 'reclaim_nothing_to_confirm', message: string): DomainError { + return new DomainError(409, message, { code, details: { code } }); } function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean { @@ -433,7 +622,7 @@ function claimsKey(source: StorageReferenceSource, category: ServedCategory, key return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix)); } -function keysOf(candidates: readonly Candidate[]): string[] { +function keysOf(candidates: readonly ReclaimCandidate[]): string[] { return candidates.map((candidate) => candidate.key); } diff --git a/server/src/nest/storage/storage-reclaim.settings.ts b/server/src/nest/storage/storage-reclaim.settings.ts index af3d1c29a..1f8323084 100644 --- a/server/src/nest/storage/storage-reclaim.settings.ts +++ b/server/src/nest/storage/storage-reclaim.settings.ts @@ -1,32 +1,91 @@ import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; import { logWarn } from '../audit/audit-log.logger'; -import { storageReclaimReportSchema, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared'; +import type { ServedCategory } from './storage.types'; +import { storageReclaimReportSchema, type StorageReclaimReport } from '@trek/shared'; -/** The admin's switch: 'true' lets a pass delete. Absent, or anything else, means report only. */ +import { z } from 'zod'; + +/** + * This installation's random id, made on its first boot. A backup restore + * brings back the app_settings of whatever install made the backup, so the + * reclaim pass keeps the id it booted with and writes it back over the + * restored one (StorageReclaimService, on every database reopen). Outside the + * `storage.` prefix on purpose: the documented reset of the storage config + * (deleting every `storage.%` row) must not make this install a stranger to + * the owner markers it wrote. + */ +export const INSTALL_ID_KEY = 'install_id'; +/** + * The admin's switch, stored as the install id it was turned on for: any other + * value, a restored one from another install included, reads as off. + */ export const RECLAIM_DELETE_KEY = 'storage.reclaim_delete'; /** The last pass's report, one JSON row (the `storage.usage` pattern). */ export const RECLAIM_REPORT_KEY = 'storage.reclaim_report'; +/** The last report of a pass that ran with deletion on, which a check by hand never replaces. */ +export const RECLAIM_DELETE_REPORT_KEY = 'storage.reclaim_delete_report'; +/** + * Per category, the unreferenced count an admin confirmed when that was more + * than a fifth of the category, for the install it was confirmed on. + */ +const RECLAIM_CONFIRMED_KEY = 'storage.reclaim_confirmed'; -/** Fail closed: only the exact stored 'true' switches deletion on. */ -export async function readReclaimDeleteEnabled(appSettings: AppSettingsRepository): Promise { - return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === 'true'; +/** Every row a restore or a foreign database must not carry over into this install's reclaim. */ +export const RECLAIM_RESET_KEYS = [ + RECLAIM_DELETE_KEY, + RECLAIM_REPORT_KEY, + RECLAIM_DELETE_REPORT_KEY, + RECLAIM_CONFIRMED_KEY, +] as const; + +const confirmationsSchema = z.object({ + installId: z.string(), + counts: z.record(z.string(), z.number().int().nonnegative()), +}); + +/** Fail closed: only this install's own id switches deletion on. */ +export async function readReclaimDeleteEnabled( + appSettings: AppSettingsRepository, + installId: string, +): Promise { + return (await appSettings.getValue(RECLAIM_DELETE_KEY)) === installId; } -/** The stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */ -async function readReclaimReport(appSettings: AppSettingsRepository): Promise { - const value = await appSettings.getValue(RECLAIM_REPORT_KEY); +/** A stored report, parsed; null when there is none or it no longer parses (logged, never a 500). */ +export async function readReclaimReport( + appSettings: AppSettingsRepository, + key: typeof RECLAIM_REPORT_KEY | typeof RECLAIM_DELETE_REPORT_KEY, +): Promise { + const value = await appSettings.getValue(key); if (value == null) return null; try { return storageReclaimReportSchema.parse(JSON.parse(value)); } catch { - logWarn('[storage] the stored reclaim report is unparseable, treating it as never run'); + logWarn(`[storage] the stored '${key}' row is unparseable, treating it as never run`); return null; } } -export async function readReclaimState(appSettings: AppSettingsRepository): Promise { - return { - deleteEnabled: await readReclaimDeleteEnabled(appSettings), - lastRun: await readReclaimReport(appSettings), - }; +/** The confirmed counts of this install; none for a row another install wrote or that no longer parses. */ +export async function readReclaimConfirmations( + appSettings: AppSettingsRepository, + installId: string, +): Promise>> { + const value = await appSettings.getValue(RECLAIM_CONFIRMED_KEY); + if (value == null) return {}; + try { + const parsed = confirmationsSchema.parse(JSON.parse(value)); + return parsed.installId === installId ? (parsed.counts as Partial>) : {}; + } catch { + logWarn('[storage] the stored reclaim confirmations are unparseable, treating them as none'); + return {}; + } +} + +export async function writeReclaimConfirmations( + appSettings: AppSettingsRepository, + installId: string, + counts: Partial>, +): Promise { + await appSettings.upsertOrReplace(RECLAIM_CONFIRMED_KEY, JSON.stringify({ installId, counts })); } diff --git a/server/src/nest/storage/storage-reclaim.types.ts b/server/src/nest/storage/storage-reclaim.types.ts index 88effbdc4..bf71fb640 100644 --- a/server/src/nest/storage/storage-reclaim.types.ts +++ b/server/src/nest/storage/storage-reclaim.types.ts @@ -5,8 +5,7 @@ import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; * name stored objects (#2924). Storage cannot read those rows itself: it may * import no domain that imports it, and it may not inject another domain's * repositories. So each owner registers a source with - * `StorageReclaimRegistryService` and answers, for a page of keys, which of - * them its rows still name. + * `StorageReclaimRegistryService` and answers which keys its rows still name. * * The ids are a closed list: the ledger in storage names, per category, which * of them must have answered before a single object of that category may be @@ -53,11 +52,17 @@ export interface StorageReferenceSource { */ readonly claims: ReadonlyArray; /** - * Of these category-relative keys (as `StorageService.list` yields them, at - * most a page of 500), the ones a row still names. A row that is trashed, + * Every key of `category` the rows name, lowercased: one read of the whole + * column set, which a pass makes once per category. A row that is trashed, * pending or otherwise not gone counts. Throwing stops the pass for the * category; it never deletes on a failed answer. */ + namedKeys(category: ServedCategory): Promise>; + /** + * Of these category-relative keys (as `StorageService.list` yields them, a + * batch at a time), the ones a row names right now: the question a delete + * asks last, read fresh and narrowed to the rows that can name them. + */ referenced(category: ServedCategory, keys: readonly string[]): Promise>; } @@ -76,16 +81,18 @@ export interface StorageKeyOptions { } const UPLOADS_PATH = /(?:^|\/)uploads\/([a-z0-9][a-z0-9-]*)\//gi; -const KEY_END = /[?#"'<>\r\n]/; +/** Where a key inside a longer text ends at the latest: a quote, markup or a line break. */ +const TEXT_END = /["'<>\r\n]/; +const SUFFIX = /[?#]/; const SERVED = new Set(SERVED_CATEGORIES); /** * Every `/uploads//` in a value, wherever it stands: a path, an * absolute URL of this instance (`https://host/uploads/covers/x.jpg`), a leading - * `uploads/` without the slash, several in one text. A query or a fragment is - * dropped, the key is percent-decoded and trimmed. A key with a blank in it is - * reported both whole and cut at the blank, since a value may be a sentence: - * reporting one key too many only keeps an object, one too few would lose it. + * `uploads/` without the slash, several in one text. Each key is reported as it + * stands and in its other readings: cut at a query or fragment, percent-decoded, + * trimmed, cut at a blank. Reporting one key too many only keeps an object; one + * too few would lose it, so the raw spelling is always among them. */ export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] { if (typeof value !== 'string' || value === '') return []; @@ -94,7 +101,7 @@ export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] const start = match.index + match[0].length; const next = matches[i + 1]; const tail = value.slice(start, next ? next.index : value.length); - const end = tail.search(KEY_END); + const end = tail.search(TEXT_END); const category = match[1].toLowerCase(); return keyVariants(end === -1 ? tail : tail.slice(0, end)).map((key) => ({ category, key })); }); @@ -103,26 +110,28 @@ export function uploadKeysIn(value: string | null | undefined): StorageKeyRef[] /** * Every object a stored value can name: the upload paths above, the relative * `/` spelling (`journeys.cover_image` holds `covers/x`), and - * with `options` the bare or legacy-prefixed name of one category. A value - * that names nothing this way (an https URL elsewhere, a data: URL, garbage) - * answers an empty list: it references nothing a pass can find. + * with `options` the bare or legacy-prefixed name of one category. A bare name + * is the object's exact name (`uuid. #4711`, `uuid.pdf `), so it is reported + * untouched first and in its other readings after. A value that names nothing + * this way (an https URL elsewhere, a data: URL, garbage) answers an empty list: + * it references nothing a pass can find. */ export function storageKeysIn(value: string | null | undefined, options: StorageKeyOptions = {}): StorageKeyRef[] { - if (typeof value !== 'string') return []; - const trimmed = stripSuffix(value).trim(); - if (trimmed === '') return []; + if (typeof value !== 'string' || value === '') return []; const refs = uploadKeysIn(value); - const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/i.exec(trimmed); - if (relative && SERVED.has(relative[1].toLowerCase())) { - const category = relative[1].toLowerCase(); - for (const key of keyVariants(relative[2])) refs.push({ category, key }); + for (const form of new Set([value, value.trim(), stripSuffix(value).trim()])) { + const relative = /^\.?\/?([a-z0-9][a-z0-9-]*)\/(.+)$/is.exec(form); + if (relative && SERVED.has(relative[1].toLowerCase())) { + const category = relative[1].toLowerCase(); + for (const key of keyVariants(relative[2])) refs.push({ category, key }); + } } if (options.bare) { const bare = options.bare; - if (!/[/\\:]/.test(trimmed)) { - for (const key of keyVariants(trimmed)) refs.push({ category: bare, key }); + if (!/[/\\:]/.test(value)) { + for (const key of keyVariants(value)) refs.push({ category: bare, key }); } else if (options.basename) { - const last = trimmed.slice(Math.max(trimmed.lastIndexOf('/'), trimmed.lastIndexOf('\\')) + 1); + const last = value.slice(Math.max(value.lastIndexOf('/'), value.lastIndexOf('\\')) + 1); for (const key of keyVariants(last)) refs.push({ category: bare, key }); } } @@ -130,14 +139,13 @@ export function storageKeysIn(value: string | null | undefined, options: Storage } /** - * The candidates of `category` that any of `values` names, compared without - * regard to case: a local volume on Windows or macOS serves `A.jpg` for `a.jpg`, - * so an object is kept when a value names it in either spelling. The helper a - * source builds `referenced()` from, so no source parses with a regex of its own. + * The keys of `category` any of `values` names, lowercased: a local volume on + * Windows or macOS serves `A.jpg` for `a.jpg`, so an object is kept when a value + * names it in either spelling. The helper a source builds its answers from, so + * no source parses with a regex of its own. */ -export function referencedAmong( +export function keysNamedIn( category: ServedCategory, - candidates: readonly string[], values: Iterable, options: StorageKeyOptions = {}, ): Set { @@ -147,25 +155,57 @@ export function referencedAmong( if (ref.category === category) named.add(ref.key.toLowerCase()); } } - return new Set(candidates.filter((key) => named.has(key.toLowerCase()))); + return named; +} + +/** + * The longest run of plain characters in a key (`[a-z0-9-]`, the random part + * of every name TREK writes): a row that names the key in any spelling holds + * this run verbatim, so a `LIKE '%run%'` finds every row worth parsing. Null + * when the key has no run long enough to narrow anything. + */ +export function keySearchCore(key: string): string | null { + const runs = key.toLowerCase().match(/[a-z0-9-]+/g) ?? []; + const longest = runs.reduce((best, run) => (run.length > best.length ? run : best), ''); + return longest.length >= 8 ? longest : null; +} + +/** + * The marker object a pass keeps at the root of each category it judges, + * naming this installation (`trek-owner-`). A second TREK + * installation that keeps its uploads in the same place writes its own, and + * from then on neither deletes there. + */ +const OWNER_MARKER_PREFIX = 'trek-owner-'; +const OWNER_MARKER = /^trek-owner-([a-z0-9-]{8,64})$/; + +export function storageOwnerMarkerKey(installId: string): string { + return `${OWNER_MARKER_PREFIX}${installId}`; +} + +/** The install id an owner marker names, or null for any other key. Backups leave markers out. */ +export function storageOwnerMarkerInstall(key: string): string | null { + return OWNER_MARKER.exec(key)?.[1] ?? null; } function stripSuffix(value: string): string { - const cut = value.search(/[?#]/); + const cut = value.search(SUFFIX); return cut === -1 ? value : value.slice(0, cut); } -/** The raw key, decoded, and cut at a blank; never an empty key or one that climbs out of its category. */ +/** The raw key and its other readings; never an empty one. */ function keyVariants(raw: string): string[] { const forms = new Set(); const add = (key: string) => { - const clean = key.trim().replace(/^\/+/, ''); - if (clean === '' || clean.split('/').some((segment) => segment === '..' || segment === '.')) return; - forms.add(clean); + if (key.trim() === '') return; + forms.add(key); + const unslashed = key.replace(/^\/+/, ''); + if (unslashed.trim() !== '') forms.add(unslashed); }; - const whole = stripSuffix(raw); - for (const form of [whole, decoded(whole)]) { + const stripped = stripSuffix(raw); + for (const form of [raw, stripped, decoded(raw), decoded(stripped)]) { add(form); + add(form.trim()); add(form.trim().split(/\s/)[0] ?? ''); } return [...forms]; diff --git a/server/src/nest/storage/storage-usage-scan.job.ts b/server/src/nest/storage/storage-usage-scan.job.ts index f18838a0a..9a72de9d9 100644 --- a/server/src/nest/storage/storage-usage-scan.job.ts +++ b/server/src/nest/storage/storage-usage-scan.job.ts @@ -1,8 +1,13 @@ import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { StorageJobsService } from './storage-jobs.service'; import { StorageStatsService } from './storage-stats.service'; import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; -/** Nightly usage scan — 04:15, off the backup and thumb-sweep hours. */ +/** + * Nightly usage scan — 04:15, off the backup and thumb-sweep hours. A reclaim + * pass still running from 04:05 is waited for first: the two would list the + * same backends at once, and the figures should show what the pass freed. + */ @Injectable() export class StorageUsageScanJob implements OnApplicationBootstrap { private readonly logger = new Logger(StorageUsageScanJob.name); @@ -10,6 +15,7 @@ export class StorageUsageScanJob implements OnApplicationBootstrap { constructor( private readonly registrar: CronRegistrarService, private readonly stats: StorageStatsService, + private readonly jobs: StorageJobsService, ) {} onApplicationBootstrap(): void { @@ -21,6 +27,7 @@ export class StorageUsageScanJob implements OnApplicationBootstrap { async tick(): Promise { try { + await this.jobs.exclusiveIdle(); await this.stats.scan(); } catch (err) { this.logger.error(`usage scan failed: ${err instanceof Error ? err.message : String(err)}`); diff --git a/server/src/nest/storage/storage.service.ts b/server/src/nest/storage/storage.service.ts index cb890234b..11e265993 100644 --- a/server/src/nest/storage/storage.service.ts +++ b/server/src/nest/storage/storage.service.ts @@ -9,6 +9,7 @@ import { isClientAbortError, StorageNotFoundError, type ByteRange, + type ListOptions, type LocalTempFile, type ObjectStat, type PutOptions, @@ -168,9 +169,9 @@ export class StorageService { } /** Names in and out are category-relative — the key prefix stays a registry detail. */ - async *list(category: ServedCategory, subPrefix = ''): AsyncIterable { + async *list(category: ServedCategory, subPrefix = '', opts?: ListOptions): AsyncIterable { const { driver, keyPrefix } = this.registry.resolve(category); - for await (const stat of driver.list(keyPrefix + subPrefix)) { + for await (const stat of driver.list(keyPrefix + subPrefix, opts)) { yield { ...stat, key: stat.key.slice(keyPrefix.length) }; } } diff --git a/server/src/nest/storage/storage.types.ts b/server/src/nest/storage/storage.types.ts index dcd5c566a..4b8e04eb5 100644 --- a/server/src/nest/storage/storage.types.ts +++ b/server/src/nest/storage/storage.types.ts @@ -98,6 +98,11 @@ export function isClientAbortError(err: unknown): boolean { ); } +export interface ListOptions { + /** Only the objects directly under the prefix, none in a folder below it (the reclaim pass judges a category's root). */ + shallow?: boolean; +} + export interface StorageDriver { /** Backend instance name, e.g. 'uploads-local'. */ readonly id: string; @@ -107,7 +112,7 @@ export interface StorageDriver { stat(key: string): Promise; /** Idempotent; resolving on a missing key is success. */ delete(key: string): Promise; - list(prefix: string): AsyncIterable; + list(prefix: string, opts?: ListOptions): AsyncIterable; /** Fast-path for local drivers; `null`/absent is exactly the remote-driver branch every caller must handle. */ getLocalPath?(key: string): string | null; /** Same-filesystem spool dir (LocalDriver: `/.tmp`), or null for drivers with no local filesystem. */ diff --git a/server/src/nest/trips/trips-storage-refs.service.ts b/server/src/nest/trips/trips-storage-refs.service.ts index c1e9a5441..b5049a95c 100644 --- a/server/src/nest/trips/trips-storage-refs.service.ts +++ b/server/src/nest/trips/trips-storage-refs.service.ts @@ -1,5 +1,5 @@ import { Trips } from '../../db/entities/Trips.entity'; -import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { likeAny, StorageReclaimSource } from '../storage/storage-reclaim-source.service'; import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; import { Injectable } from '@nestjs/common'; @@ -15,8 +15,10 @@ export class TripsStorageRefsService extends StorageReclaimSource { readonly id = 'trips'; readonly claims = EVERY_RECLAIMABLE_CATEGORY; - protected async storedValues(): Promise> { - const rows = await this.em.getRepository(Trips).find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + protected async storedValues(containing: readonly string[] | null): Promise> { + const rows = await this.em + .getRepository(Trips) + .find(likeAny(['cover_image'], containing), { fields: ['cover_image'] }); return rows.map((row) => row.cover_image); } } diff --git a/server/tests/e2e/storage-reclaim.e2e.test.ts b/server/tests/e2e/storage-reclaim.e2e.test.ts index fd12b3604..3e32ce536 100644 --- a/server/tests/e2e/storage-reclaim.e2e.test.ts +++ b/server/tests/e2e/storage-reclaim.e2e.test.ts @@ -4,13 +4,15 @@ * AdminGuard and ManagedGuard, the Zod pipe on both bodies and the response * contracts the handlers declare all stand in front of it. Every owner * domain registers its reference source here, so a pass looks at every - * reclaimable category; the passes below only report. RCLE-001 to RCLE-007. + * reclaimable category; the checks below run in the background and only + * report. RCLE-001 to RCLE-007. */ import { buildApp } from '../../src/bootstrap'; import { AppSettings } from '../../src/db/entities/AppSettings.entity'; import { AuditLog } from '../../src/db/entities/AuditLog.entity'; import { StorageJobsService } from '../../src/nest/storage/storage-jobs.service'; -import { STORAGE_RECLAIMABLE_CATEGORIES } from '../../src/nest/storage/storage-reclaim.types'; +import { STORAGE_RECLAIMABLE_CATEGORIES, storageOwnerMarkerKey } from '../../src/nest/storage/storage-reclaim.types'; +import { StorageService } from '../../src/nest/storage/storage.service'; import { countRows, deleteRows, findRow } from '../helpers/factories/rows'; import { makeAdmin, makeUser } from '../helpers/factories/users'; import { sessionCookie } from './harness'; @@ -50,12 +52,33 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar }); afterAll(async () => { + // The uploads folder is the test run's shared one: take back the owner markers this app's checks wrote. + const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value; + if (installId) { + const storage = app.get(StorageService); + await Promise.all( + STORAGE_RECLAIMABLE_CATEGORIES.map((category) => + storage.delete(category, storageOwnerMarkerKey(installId)).catch(() => undefined), + ), + ); + } await app?.close(); }); - it('RCLE-001: 401 without a session on both routes', async () => { + /** The admin state's reclaim part once a check started by hand has finished. */ + async function settledState() { + await app.get(StorageJobsService).exclusiveIdle(); + const res = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); + expect(res.status).toBe(200); + return res.body.reclaim; + } + + it('RCLE-001: 401 without a session on every route', async () => { expect((await request(server).put('/api/admin/storage/reclaim').send({ deleteEnabled: true })).status).toBe(401); expect((await request(server).post('/api/admin/storage/reclaim/run').send({})).status).toBe(401); + expect((await request(server).post('/api/admin/storage/reclaim/confirm').send({ category: 'covers' })).status).toBe( + 401, + ); }); it('RCLE-002: 403 for a non-admin, and nothing is stored', async () => { @@ -68,63 +91,79 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar expect(await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' })).toBeFalsy(); }); - it('RCLE-003: PUT stores the switch, answers the state, audits, and GET /api/admin/storage carries it', async () => { + it('RCLE-003: PUT stores the switch for this install, answers the state, audits, and the admin state carries it', async () => { const res = await request(server) .put('/api/admin/storage/reclaim') .set('Cookie', adminCookie) .send({ deleteEnabled: true }); expect(res.status).toBe(200); - expect(res.body).toEqual({ deleteEnabled: true, lastRun: null }); + expect(res.body).toEqual({ deleteEnabled: true, running: false, lastRun: null, lastDeleteRun: null }); expect(await countRows(orm, AuditLog, { action: 'admin.storage_reclaim_update' })).toBe(1); - const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); - expect(state.status).toBe(200); - expect(state.body.reclaim).toEqual({ deleteEnabled: true, lastRun: null }); + // Stored as the install id it was turned on for, so a database restored elsewhere reads as off. + const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value; + expect(installId).toMatch(/^[0-9a-f-]{36}$/); + expect((await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' }))?.value).toBe(installId); + expect((await settledState()).deleteEnabled).toBe(true); + const off = await request(server) + .put('/api/admin/storage/reclaim') + .set('Cookie', adminCookie) + .send({ deleteEnabled: false }); + expect(off.body.deleteEnabled).toBe(false); }); - it('RCLE-004: PUT refuses a body outside the contract', async () => { + it('RCLE-004: PUT and confirm refuse a body outside the contract', async () => { const refused = await Promise.all( [{}, { deleteEnabled: 'yes' }, { deleteEnabled: true, extra: 1 }].map((body) => request(server).put('/api/admin/storage/reclaim').set('Cookie', adminCookie).send(body), ), ); expect(refused.map((res) => res.status)).toEqual([400, 400, 400]); - const run = await request(server) - .post('/api/admin/storage/reclaim/run') - .set('Cookie', adminCookie) - .send({ reportOnly: 'no' }); - expect(run.status).toBe(400); + const confirm = await Promise.all( + [{}, { category: 'nowhere' }, { category: 'covers', count: 3 }].map((body) => + request(server).post('/api/admin/storage/reclaim/confirm').set('Cookie', adminCookie).send(body), + ), + ); + expect(confirm.map((res) => res.status)).toEqual([400, 400, 400]); expect(await countRows(orm, AuditLog)).toBe(0); }); - it('RCLE-005: a report-only pass asks every registered source, deletes nothing, persists and audits', async () => { - const res = await request(server) - .post('/api/admin/storage/reclaim/run') - .set('Cookie', adminCookie) - .send({ reportOnly: true }); - expect(res.status).toBe(200); - expect(res.body.mode).toBe('report'); - for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { - expect(res.body.categories[category].skipped).not.toBe('sources_missing'); - expect(res.body.categories[category].deleted).toEqual({ objects: 0, bytes: 0 }); - } - expect(res.body.categories.backups.skipped).toBe('not_reclaimed'); + it('RCLE-005: POST run answers 202 at once, the check asks every source, deletes nothing and lands on the state', async () => { + const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({}); + expect(res.status).toBe(202); + expect(res.body).toEqual({ started: true }); const audit = await findRow(orm, AuditLog, { action: 'admin.storage_reclaim_run' }); - expect(JSON.parse(String(audit?.details))).toEqual({ mode: 'report', deleted: 0, bytes: 0 }); - const state = await request(server).get('/api/admin/storage').set('Cookie', adminCookie); - expect(state.body.reclaim.lastRun).toEqual(res.body); + expect(JSON.parse(String(audit?.details))).toEqual({ mode: 'report' }); + const reclaim = await settledState(); + expect(reclaim.running).toBe(false); + expect(reclaim.lastRun.mode).toBe('report'); + expect(reclaim.lastDeleteRun).toBeNull(); + for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { + expect(reclaim.lastRun.categories[category].skipped).not.toBe('sources_missing'); + expect(reclaim.lastRun.categories[category].deleted).toEqual({ objects: 0, bytes: 0 }); + } + expect(reclaim.lastRun.categories.backups.skipped).toBe('not_reclaimed'); }); - it('RCLE-006: 409 while another storage job holds the slot', async () => { + it('RCLE-006: 409 with a code while another storage job holds the slot, and while nothing waits for a confirmation', async () => { const jobs = app.get(StorageJobsService); expect(jobs.claimExclusive('a test job')).toBe(true); try { const res = await request(server).post('/api/admin/storage/reclaim/run').set('Cookie', adminCookie).send({}); expect(res.status).toBe(409); - expect(res.body).toEqual({ error: 'another storage job is running, one storage job at a time' }); + expect(res.body).toEqual({ + error: 'another storage job is running, one storage job at a time', + code: 'reclaim_busy', + }); expect(await countRows(orm, AuditLog)).toBe(0); } finally { jobs.releaseExclusive('a test job'); } + const confirm = await request(server) + .post('/api/admin/storage/reclaim/confirm') + .set('Cookie', adminCookie) + .send({ category: 'covers' }); + expect(confirm.status).toBe(409); + expect(confirm.body.code).toBe('reclaim_nothing_to_confirm'); }); it('RCLE-007: managed mode refuses the surface with the standard body', async () => { diff --git a/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts b/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts index 779523100..aad210f16 100644 --- a/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts +++ b/server/tests/unit/db/repositories/StorageReclaimMarks.repository.test.ts @@ -69,11 +69,11 @@ describe('StorageReclaimMarksRepository', () => { expect((await marks.marksOf('places')).map((m) => m.key)).toEqual(['a.jpg']); }); - it('RCLM-004: drops every mark of one category', async () => { + it('RCLM-004: drops every mark of every category, for a database a restore brought back', async () => { await marks.insertMissing('covers', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00'); await marks.insertMissing('places', [{ key: 'a.jpg', size: 1 }], '2026-01-01 00:00:00'); - expect(await marks.deleteCategory('covers')).toBe(1); + expect(await marks.deleteAll()).toBe(2); expect(await marks.marksOf('covers')).toEqual([]); - expect(await marks.marksOf('places')).toHaveLength(1); + expect(await marks.marksOf('places')).toEqual([]); }); }); diff --git a/server/tests/unit/nest/backup.controller.test.ts b/server/tests/unit/nest/backup.controller.test.ts index 82fd7ca06..6488eeee4 100644 --- a/server/tests/unit/nest/backup.controller.test.ts +++ b/server/tests/unit/nest/backup.controller.test.ts @@ -448,6 +448,20 @@ describe('BackupService (wrapper)', () => { expect(jobs.claimExclusive('a storage reclaim')).toBe(true); jobs.releaseExclusive('a storage reclaim'); }); + + it('BKS-BACKUP-001: a backup runs marked as reading the uploads, a failed one too, and the mark ends with it', async () => { + let runningInside = false; + vi.mocked(backupSvc.createBackup).mockImplementationOnce(async () => { + runningInside = jobs.backupRunning; + return { filename: 'svc.zip', size: 5 }; + }); + await wrapper.createBackup(); + expect(runningInside).toBe(true); + expect(jobs.backupRunning).toBe(false); + vi.mocked(backupSvc.createBackup).mockRejectedValueOnce(new Error('disk full')); + await expect(wrapper.createBackup()).rejects.toThrow('disk full'); + expect(jobs.backupRunning).toBe(false); + }); }); describe('BackupModule', () => { diff --git a/server/tests/unit/nest/backup.impl.test.ts b/server/tests/unit/nest/backup.impl.test.ts index 6ab971716..ddefb1562 100644 --- a/server/tests/unit/nest/backup.impl.test.ts +++ b/server/tests/unit/nest/backup.impl.test.ts @@ -714,6 +714,23 @@ describe('BACKUP-036 createBackup', () => { expect(storage.getLocalPathOrNull).toHaveBeenCalledTimes(1); }); + it('BACKUP-036i — leaves the storage reclaim owner marker out, so a restore elsewhere does not read as shared', async () => { + fsMock.existsSync.mockReturnValue(false); + setupArchiveSuccess(); + const storage = stubStorage({ + stat: statOf(1024), + list: listByCategory({ + covers: [{ key: 'trek-owner-0f8e2b7c-1d2a-4c3b-9e8f-7a6b5c4d3e2f' }, { key: 'cover.jpg' }], + }), + }); + + await createBackup(deps(storage)); + + const names = archiverInstanceMock.file.mock.calls.map((c) => c[1]?.name as string); + expect(names).toContain('uploads/covers/cover.jpg'); + expect(names.some((n) => n?.includes('trek-owner-'))).toBe(false); + }); + it('BACKUP-036h — only category prefixes are ever archived; backups/ and restore-* are structurally out (issue #1358)', async () => { // The uploads/** glob is gone: even when data and uploads map to the same // directory, enumeration only ever walks the six archived categories, so diff --git a/server/tests/unit/nest/storage-reference-sources.test.ts b/server/tests/unit/nest/storage-reference-sources.test.ts index 483779c86..67cc1dbb5 100644 --- a/server/tests/unit/nest/storage-reference-sources.test.ts +++ b/server/tests/unit/nest/storage-reference-sources.test.ts @@ -3,7 +3,11 @@ * (#2924), each against the migrated schema: which of a page of keys its rows * still name, in every spelling those columns hold. One object kept too many * costs a few bytes; one called unreferenced while a row names it is lost, so - * every case here is a spelling a real row can carry. RSRC-001 to RSRC-011. + * every case here is a spelling a real row can carry. A source answers twice: + * `namedKeys`, every key of a category its rows name (one read, for a pass), + * and `referenced`, a few keys asked fresh and narrowed to the rows holding + * their plain runs (the re-check before a delete, and a release). + * RSRC-001 to RSRC-014. */ import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity'; import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity'; @@ -197,4 +201,36 @@ describe('reference sources', () => { ); expect(answers).toEqual(SOURCES.map(() => [])); }); + + it('RSRC-012: a narrowed question still finds a uuid-named object in every spelling a row holds', async () => { + const id = '7b1f0c2e-5a4d-4e3f-8a9b-0c1d2e3f4a5b'; + await makeTrip(orm, userId, { + cover_image: `https://trip.example.test/uploads/covers/${id.toUpperCase()}.JPG?v=3`, + }); + await makePlace(orm, tripId, { image_url: `/uploads/places/${id}-p.jpg#frag` }); + const trips = t.get(TripsStorageRefsService); + expect(await asked(trips, 'covers', [`${id}.jpg`, `${id}-other.jpg`])).toEqual([`${id}.jpg`]); + expect(await asked(t.get(PlacesStorageRefsService), 'places', [`${id}-p.jpg`])).toEqual([`${id}-p.jpg`]); + }); + + it('RSRC-013: namedKeys answers every key of the category its rows name, lowercased, in one read', async () => { + await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Named-A.jpg' }); + await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-named-b.jpg' }); + const trips = t.get(TripsStorageRefsService); + const covers = await trips.namedKeys('covers'); + expect(covers.has('rsrc-named-a.jpg')).toBe(true); + expect(covers.has('rsrc-named-b.jpg')).toBe(false); + expect((await trips.namedKeys('places')).has('rsrc-named-b.jpg')).toBe(true); + expect(await t.get(FilesStorageRefsService).namedKeys('covers')).toEqual(new Set()); + }); + + it('RSRC-014: a trip file whose name holds a #, a ? or a trailing blank keeps its object both ways', async () => { + const id = '3c9d8e7f-6a5b-4c4d-9e3f-2a1b0c9d8e7f'; + const names = [`${id}. #4711`, `${id}.c#`, `${id}.v2?final`, `${id}.pdf `]; + await Promise.all(names.map((filename) => makeTripFile(orm, tripId, { filename }))); + const files = t.get(FilesStorageRefsService); + expect(await asked(files, 'files', names)).toEqual([...names].sort()); + const named = await files.namedKeys('files'); + for (const name of names) expect(named.has(name.toLowerCase())).toBe(true); + }); }); diff --git a/server/tests/unit/nest/storage/local.driver.test.ts b/server/tests/unit/nest/storage/local.driver.test.ts index 0aeb559fd..86f125cb3 100644 --- a/server/tests/unit/nest/storage/local.driver.test.ts +++ b/server/tests/unit/nest/storage/local.driver.test.ts @@ -136,4 +136,17 @@ describe('LocalDriver specifics', () => { for await (const stat of driver.list('nope/')) seen.push(stat); expect(seen).toEqual([]); }); + + it('a shallow list yields the objects at the prefix only, none of a folder below it', async () => { + const driver = new LocalDriver({ id: 'shallow', root: tmp() }); + driver.init(); + await driver.put('journey/top.jpg', Readable.from('a')); + await driver.put('journey/thumbs/deep.jpg', Readable.from('b')); + const shallow: string[] = []; + for await (const stat of driver.list('journey/', { shallow: true })) shallow.push(stat.key); + expect(shallow).toEqual(['journey/top.jpg']); + const deep: string[] = []; + for await (const stat of driver.list('journey/')) deep.push(stat.key); + expect(deep.sort()).toEqual(['journey/thumbs/deep.jpg', 'journey/top.jpg']); + }); }); diff --git a/server/tests/unit/nest/storage/mirror.driver.test.ts b/server/tests/unit/nest/storage/mirror.driver.test.ts index dfecbc40c..33a322c73 100644 --- a/server/tests/unit/nest/storage/mirror.driver.test.ts +++ b/server/tests/unit/nest/storage/mirror.driver.test.ts @@ -174,6 +174,20 @@ describe('MirrorDriver specifics', () => { expect(keys).toEqual(['backup-5.zip']); }); + it('hands a shallow list on to the primary, and to the replica it falls back to', async () => { + const fx = track(makeMirror()); + await fx.mirror.put('top.zip', Readable.from('1')); + const primaryList = vi.spyOn(fx.primary, 'list'); + for await (const stat of fx.mirror.list('', { shallow: true })) void stat; + expect(primaryList).toHaveBeenCalledWith('', { shallow: true }); + primaryList.mockImplementationOnce(() => { + throw new StorageBackendError('io'); + }); + const replicaList = vi.spyOn(fx.replica, 'list'); + for await (const stat of fx.mirror.list('', { shallow: true })) void stat; + expect(replicaList).toHaveBeenCalledWith('', { shallow: true }); + }); + it('fans delete out to all targets, tolerating a miss on either side', async () => { const fx = track(makeMirror()); await fx.mirror.put('backup-6.zip', Readable.from('bye')); diff --git a/server/tests/unit/nest/storage/s3.driver.test.ts b/server/tests/unit/nest/storage/s3.driver.test.ts index bed973b02..350262e3b 100644 --- a/server/tests/unit/nest/storage/s3.driver.test.ts +++ b/server/tests/unit/nest/storage/s3.driver.test.ts @@ -287,6 +287,18 @@ describe('S3Driver list', () => { for await (const stat of makeDriver(api, { keyPrefix: 'trek/prod' }).list('')) out.push(stat); expect(out.map((s) => s.key)).toEqual(['a/ok.bin']); }); + it('a shallow list asks S3 to fold every deeper key away with a delimiter', async () => { + const api = makeMockApi({ + ListObjectsV2: pages([{ Key: 'trek/prod/a/one.bin', Size: 2, LastModified: new Date(1000) }]), + }); + const out = []; + for await (const stat of makeDriver(api, { keyPrefix: 'trek/prod' }).list('a/', { shallow: true })) out.push(stat); + expect(out.map((s) => s.key)).toEqual(['a/one.bin']); + expect(api.ListObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ Prefix: 'trek/prod/a/', Delimiter: '/' })); + const deep = makeMockApi({ ListObjectsV2: pages([]) }); + for await (const stat of makeDriver(deep).list('a/')) void stat; + expect(deep.ListObjectsV2).toHaveBeenCalledWith(expect.not.objectContaining({ Delimiter: '/' })); + }); it('yields nothing for an empty result and validates the prefix first', async () => { const api = makeMockApi({ ListObjectsV2: pages([]) }); const driver = makeDriver(api); diff --git a/server/tests/unit/nest/storage/storage-admin.service.test.ts b/server/tests/unit/nest/storage/storage-admin.service.test.ts index f0e58b684..7eeaed198 100644 --- a/server/tests/unit/nest/storage/storage-admin.service.test.ts +++ b/server/tests/unit/nest/storage/storage-admin.service.test.ts @@ -4,6 +4,7 @@ import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; import { StorageEventsService } from '../../../../src/nest/storage/storage-events.service'; import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; +import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; import { StorageRegistryService, BACKENDS_KEY, @@ -74,7 +75,11 @@ async function makeService(opts: { uploadsRoot?: string } = {}) { const storage = new StorageService(registry); const jobs = new StorageJobsService(registry); const stats = new StorageStatsService(storage, appSettings); - const service = new StorageAdminService(appSettings, registry, storage, jobs, stats, uow); + // The reclaim state has its own suite (storage-reclaim.service.test.ts); here it only rides along. + const reclaim = { + state: async () => ({ deleteEnabled: false, running: false, lastRun: null, lastDeleteRun: null }), + } as unknown as StorageReclaimService; + const service = new StorageAdminService(appSettings, registry, storage, jobs, stats, uow, reclaim); return { service, registry, uploadsRoot, stats, jobs, uow }; } diff --git a/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts index 1fe8e076f..7db88202f 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts @@ -1,17 +1,20 @@ /** * StorageReclaimController over a stub service: what it forwards, what it * audits, and that a refusal of the service reaches the client unchanged. - * RCLC-001 to RCLC-004. + * RCLC-001 to RCLC-006. */ import type { AuditService } from '../../../../src/nest/audit/audit.service'; import { DomainError } from '../../../../src/nest/common/domain-error'; -import type { StorageReclaimRunDto, StorageReclaimSettingsDto } from '../../../../src/nest/storage/storage-admin.dto'; +import type { + StorageReclaimConfirmDto, + StorageReclaimSettingsDto, +} from '../../../../src/nest/storage/storage-admin.dto'; import { StorageReclaimController } from '../../../../src/nest/storage/storage-reclaim.controller'; import type { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; import { StorageModule } from '../../../../src/nest/storage/storage.module'; import type { User } from '../../../../src/types'; import { expectRegisteredController } from '../../../helpers/module-providers'; -import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared'; +import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport, type StorageReclaimState } from '@trek/shared'; import type { Request } from 'express'; import { describe, expect, it, vi } from 'vitest'; @@ -21,25 +24,36 @@ const req = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } as unknown a function report(): StorageReclaimReport { const none = { objects: 0, bytes: 0 }; - const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null }; + const row = { + objects: 0, + unreferenced: none, + eligible: none, + deleted: none, + confirmationRequired: false, + skipped: null, + }; return { startedAt: 1, finishedAt: 2, - mode: 'delete', + mode: 'report', stopped: null, categories: { ...(Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])) as StorageReclaimReport['categories']), - covers: { ...row, objects: 9, deleted: { objects: 2, bytes: 40 } }, - places: { ...row, objects: 3, deleted: { objects: 1, bytes: 2 } }, + covers: { ...row, objects: 9, unreferenced: { objects: 4, bytes: 40 }, confirmationRequired: true }, }, }; } +function state(over: Partial = {}): StorageReclaimState { + return { deleteEnabled: false, running: false, lastRun: null, lastDeleteRun: null, ...over }; +} + function makeController(over: Partial> = {}) { const writeAudit = vi.fn(); const service = { - setDeleteEnabled: vi.fn(async (enabled: boolean) => ({ deleteEnabled: enabled, lastRun: null })), - run: vi.fn(async () => report()), + setDeleteEnabled: vi.fn(async (enabled: boolean) => state({ deleteEnabled: enabled })), + start: vi.fn(), + confirm: vi.fn(async () => state({ lastRun: report() })), ...over, } as unknown as StorageReclaimService; const controller = new StorageReclaimController(service, { writeAudit } as unknown as AuditService); @@ -54,32 +68,56 @@ describe('StorageReclaimController', () => { it('RCLC-002: PUT stores the switch, answers the fresh state and audits the value', async () => { const { controller, service, writeAudit } = makeController(); const body = { deleteEnabled: true } as StorageReclaimSettingsDto; - expect(await controller.update(user, body, req)).toEqual({ deleteEnabled: true, lastRun: null }); + expect(await controller.update(user, body, req)).toEqual(state({ deleteEnabled: true })); expect(service.setDeleteEnabled).toHaveBeenCalledWith(true); expect(writeAudit).toHaveBeenCalledWith( expect.objectContaining({ userId: 7, action: 'admin.storage_reclaim_update', details: { deleteEnabled: true } }), ); }); - it('RCLC-003: POST run forwards reportOnly, answers the report and audits the totals', async () => { + it('RCLC-003: POST run starts the check in the background, answers at once and audits the start', async () => { const { controller, service, writeAudit } = makeController(); - const answer = await controller.run(user, { reportOnly: true } as StorageReclaimRunDto, req); - expect(answer.categories.covers.deleted.objects).toBe(2); - expect(service.run).toHaveBeenCalledWith({ reportOnly: true }); - await controller.run(user, {} as StorageReclaimRunDto, req); - expect(service.run).toHaveBeenLastCalledWith({ reportOnly: false }); + expect(await controller.run(user, req)).toEqual({ started: true }); + expect(service.start).toHaveBeenCalledTimes(1); + expect(writeAudit).toHaveBeenCalledWith( + expect.objectContaining({ userId: 7, action: 'admin.storage_reclaim_run', details: { mode: 'report' } }), + ); + }); + + it('RCLC-004: a refused start propagates as the DomainError and writes no audit row', async () => { + const refusal = new DomainError(409, 'another storage job is running, one storage job at a time', { + code: 'reclaim_busy', + }); + const { controller, writeAudit } = makeController({ + start: vi.fn(() => { + throw refusal; + }), + }); + await expect(controller.run(user, req)).rejects.toBe(refusal); + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('RCLC-005: POST confirm forwards the category, answers the state and audits the confirmed count', async () => { + const { controller, service, writeAudit } = makeController(); + const answer = await controller.confirm(user, { category: 'covers' } as StorageReclaimConfirmDto, req); + expect(answer.lastRun?.categories.covers.confirmationRequired).toBe(true); + expect(service.confirm).toHaveBeenCalledWith('covers'); expect(writeAudit).toHaveBeenCalledWith( expect.objectContaining({ - action: 'admin.storage_reclaim_run', - details: { mode: 'delete', deleted: 3, bytes: 42 }, + action: 'admin.storage_reclaim_confirm', + details: { category: 'covers', unreferenced: 4 }, }), ); }); - it('RCLC-004: a refused pass propagates as the DomainError and writes no audit row', async () => { - const refusal = new DomainError(409, 'another storage job is running, one storage job at a time'); - const { controller, writeAudit } = makeController({ run: vi.fn().mockRejectedValue(refusal) }); - await expect(controller.run(user, {} as StorageReclaimRunDto, req)).rejects.toBe(refusal); + it('RCLC-006: a confirmation with nothing waiting propagates its 409 and writes no audit row', async () => { + const refusal = new DomainError(409, 'nothing in this category waits for a confirmation', { + code: 'reclaim_nothing_to_confirm', + }); + const { controller, writeAudit } = makeController({ confirm: vi.fn().mockRejectedValue(refusal) }); + await expect(controller.confirm(user, { category: 'files' } as StorageReclaimConfirmDto, req)).rejects.toBe( + refusal, + ); expect(writeAudit).not.toHaveBeenCalled(); }); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.job.test.ts b/server/tests/unit/nest/storage/storage-reclaim.job.test.ts index f696ee34b..cb3697195 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.job.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.job.test.ts @@ -35,7 +35,14 @@ function report( stopped: StorageReclaimReport['stopped'] = null, ) { const none = { objects: 0, bytes: 0 }; - const row = { objects: 0, unreferenced: none, eligible: none, deleted: none, skipped: null }; + const row = { + objects: 0, + unreferenced: none, + eligible: none, + deleted: none, + confirmationRequired: false, + skipped: null, + }; return { startedAt: 1, finishedAt: 2, @@ -104,15 +111,18 @@ describe('StorageReclaimJob', () => { unreferenced: { objects: 2, bytes: 30 }, eligible: { objects: 1, bytes: 10 }, deleted: { objects: 1, bytes: 10 }, + confirmationRequired: false, skipped: null, }, - avatars: { ...report().categories.avatars, skipped: 'sources_missing' }, + files: { ...report().categories.files, confirmationRequired: true }, + avatars: { ...report().categories.avatars, skipped: 'interrupted' }, }, 'database_reopened', ); await makeJob(async () => busy).job.tick(); expect(logInfo).toHaveBeenCalledWith( - 'Storage reclaim: 2 unreferenced object(s); deleted 1 (10 bytes); skipped avatars (sources_missing); stopped: database_reopened', + 'Storage reclaim: 2 unreferenced object(s); deleted 1 (10 bytes); waiting for an admin to confirm files; ' + + 'skipped avatars (interrupted); stopped: database_reopened', ); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts index 8eb766a8e..5fd453179 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts @@ -1,6 +1,7 @@ /** * The reclaim ledger and the location check (#2924), plus the registry the - * reference sources land in. RCLL-001 to RCLL-011. + * reference sources land in. Local roots are resolved through their symlinks, + * so the layouts live in a temp folder. RCLL-001 to RCLL-013. */ import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service'; import { @@ -21,8 +22,10 @@ import type { BackendSnapshot } from '../../../../src/nest/storage/storage-regis import { CATEGORY_PREFIXES } from '../../../../src/nest/storage/storage-registry.service'; import { SERVED_CATEGORIES, type ServedCategory } from '../../../../src/nest/storage/storage.types'; +import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; -import { describe, expect, it, vi } from 'vitest'; +import { afterAll, describe, expect, it, vi } from 'vitest'; vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ ...(await importOriginal()), @@ -32,8 +35,20 @@ vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => logWarn: vi.fn(), })); -const UPLOADS = path.resolve('/srv/trek/uploads'); -const BACKUPS = path.resolve('/srv/trek/backups'); +const ROOT = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-ledger-'))); +/** A folder under the temp root, made if missing: a location is only resolved when its root exists. */ +function dir(...parts: string[]): string { + const target = path.join(ROOT, ...parts); + fs.mkdirSync(target, { recursive: true }); + return target; +} +const UPLOADS = dir('uploads'); +const BACKUPS = dir('backups'); +for (const category of SERVED_CATEGORIES) dir('uploads', CATEGORY_PREFIXES[category]); + +afterAll(() => { + fs.rmSync(ROOT, { recursive: true, force: true }); +}); function local(name: string, root: string): BackendSnapshot { return { name, type: 'local', source: 'settings', options: { root } }; @@ -90,7 +105,7 @@ describe('sharedLocationCategories', () => { }); it('RCLL-005: TREK_PLACE_PHOTO_DIR inside the uploads folder overlaps the category it lands in', () => { - const backends = [...BASE_BACKENDS, local('place-photos-local', path.join(UPLOADS, 'places'))]; + const backends = [...BASE_BACKENDS, local('place-photos-local', dir('uploads', 'places'))]; const shared = sharedLocationCategories( defaults({ 'photos-google': { backendName: 'place-photos-local', keyPrefix: '' } }), backends, @@ -108,8 +123,8 @@ describe('sharedLocationCategories', () => { expect(shared.has('covers')).toBe(false); }); - it('RCLL-007: two backends of other names on one folder overlap, compared without regard to case', () => { - const backends = [...BASE_BACKENDS, local('again', UPLOADS.toUpperCase() + path.sep)]; + it('RCLL-007: two backends of other names on one folder overlap, however the root is spelled', () => { + const backends = [...BASE_BACKENDS, local('again', `${path.join(UPLOADS, 'files', '..')}${path.sep}`)]; const shared = sharedLocationCategories( defaults({ avatars: { backendName: 'again', keyPrefix: 'covers/' } }), backends, @@ -145,8 +160,8 @@ describe('sharedLocationCategories', () => { it('RCLL-009: a mirror counts its primary and every replica; a replica inside another category excludes it', () => { const backends: BackendSnapshot[] = [ ...BASE_BACKENDS, - local('primary', path.resolve('/mnt/primary')), - local('replica', path.join(UPLOADS, 'journey')), + local('primary', dir('primary')), + local('replica', dir('uploads', 'journey')), { name: 'm', type: 'mirror', source: 'settings', options: { primary: 'primary', replicas: ['replica'] } }, ]; const shared = sharedLocationCategories(defaults({ covers: { backendName: 'm', keyPrefix: 'covers/' } }), backends); @@ -155,6 +170,29 @@ describe('sharedLocationCategories', () => { expect(shared.has('covers')).toBe(false); }); + it('RCLL-012: a place-photo folder reached through a symlink still overlaps the category it really is', () => { + // The image links /app/server/uploads to /app/uploads; an operator names the latter. + const linked = path.join(ROOT, 'linked-uploads'); + if (!fs.existsSync(linked)) fs.symlinkSync(UPLOADS, linked, 'junction'); + const backends = [...BASE_BACKENDS, local('place-photos-local', path.join(linked, 'places'))]; + const shared = sharedLocationCategories( + defaults({ 'photos-google': { backendName: 'place-photos-local', keyPrefix: '' } }), + backends, + ); + expect(shared.has('places')).toBe(true); + expect(shared.has('covers')).toBe(false); + }); + + it('RCLL-013: a local root that does not resolve reads as unknown, so shared', () => { + const backends = [...BASE_BACKENDS, local('gone', path.join(ROOT, 'never-made'))]; + const shared = sharedLocationCategories( + defaults({ covers: { backendName: 'gone', keyPrefix: 'covers/' } }), + backends, + ); + expect(shared.has('covers')).toBe(true); + expect(shared.has('files')).toBe(true); + }); + it('RCLL-010: a location that cannot be resolved fails closed, for itself and for every other category', () => { const backends: BackendSnapshot[] = [ ...BASE_BACKENDS, @@ -178,6 +216,7 @@ describe('StorageReclaimRegistryService', () => { const source = (id: StorageReferenceSource['id']): StorageReferenceSource => ({ id, claims: [{ category: 'covers', prefix: '' }], + namedKeys: async () => new Set(), referenced: async () => new Set(), }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts index ddef374d4..516b47619 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts @@ -3,11 +3,15 @@ * and LocalDriver on temp folders, the marks table in the snapshot db, and * fake reference sources standing in for the owner domains. The pass marks * first, deletes a week later only with the switch on and only after asking - * every source about the single key again; every brake skips a category and - * starts its marks over. RCLS-001 to RCLS-024. + * every source about its batch again. A skip keeps the marks; a share over a + * fifth waits for an admin's confirmation; an owner marker of another install + * keeps its hands off; a restore resets the switch, the marks and the reports + * but keeps this install's id. RCLS-001 to RCLS-036. */ import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; +import { AuditLog } from '../../../../src/db/entities/AuditLog.entity'; import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity'; +import { StorageReclaimMarksRepository } from '../../../../src/db/repositories/StorageReclaimMarks.repository'; import { dbNow } from '../../../../src/db/types'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; import { logError } from '../../../../src/nest/audit/audit-log.logger'; @@ -20,6 +24,7 @@ import { StorageReclaimService } from '../../../../src/nest/storage/storage-recl import { STORAGE_RECLAIMABLE_CATEGORIES, STORAGE_REFERENCE_SOURCE_IDS, + storageOwnerMarkerKey, type StorageReferenceClaim, type StorageReferenceSource, type StorageReferenceSourceId, @@ -28,7 +33,7 @@ import { StorageModule } from '../../../../src/nest/storage/storage.module'; import { StorageService } from '../../../../src/nest/storage/storage.service'; import type { ServedCategory } from '../../../../src/nest/storage/storage.types'; import { createSnapshotTestDb } from '../../../helpers/db-mock'; -import { deleteRows } from '../../../helpers/factories/rows'; +import { countRows, deleteRows, findRow } from '../../../helpers/factories/rows'; import { readAppSetting, setAppSetting } from '../../../helpers/factories/settings'; import { createTestModule, type TestModule } from '../../../helpers/test-module'; import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; @@ -55,12 +60,14 @@ let uploadsRoot: string; let backupsRoot: string; let reclaim: StorageReclaimService; let storage: StorageService; +let jobs: StorageJobsService; let references: StorageReclaimRegistryService; let reopen: () => Promise; +let installId: string; /** What the fake sources still name, per category. */ let named: Map>; -/** Called on every question a source is asked, before it answers. */ -let onAsk: ((category: ServedCategory, keys: readonly string[]) => void | Promise) | null; +/** Called on every question a source is asked, before it answers: `keys` is null for a whole-category read. */ +let onAsk: ((category: ServedCategory, keys: readonly string[] | null) => void | Promise) | null; /** The keys the fake sources name in `category`, created empty on first use. */ function namedIn(category: ServedCategory): Set { @@ -78,6 +85,10 @@ function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource { return { id, claims: categories.map((category): StorageReferenceClaim => ({ category, prefix: '' })), + async namedKeys(category) { + await onAsk?.(category, null); + return new Set([...(named.get(category) ?? [])].map((key) => key.toLowerCase())); + }, async referenced(category, keys) { await onAsk?.(category, keys); const set = named.get(category) ?? new Set(); @@ -102,7 +113,7 @@ function objectPath(category: string, key: string): string { return path.join(uploadsRoot, category, key); } -/** The orphans plus `kept` named covers, so the orphans stay under the brake's fifth. */ +/** The orphans plus `kept` named covers, so the orphans stay under a fifth. */ async function seedCovers(orphans: string[], kept = 4): Promise { const keptKeys = Array.from({ length: kept }, (_unused, i) => `kept-${i}.jpg`); await Promise.all([...orphans, ...keptKeys].map((key) => put('covers', key))); @@ -116,6 +127,24 @@ async function marksOf(category: string): Promise a.key.localeCompare(b.key)); } +/** What a synchronous call threw, or null. */ +function thrownBy(call: () => unknown): unknown { + try { + call(); + } catch (err) { + return err; + } + return null; +} + +async function markWeekOld(category: ServedCategory, keys: string[]): Promise { + await orm.repo(StorageReclaimMarks).insertMissing( + category, + keys.map((key) => ({ key, size: 5 })), + dbNow(new Date(0)), + ); +} + beforeAll(async () => { uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-uploads-')); backupsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-backups-')); @@ -138,8 +167,10 @@ beforeAll(async () => { }); reclaim = t.get(StorageReclaimService); storage = t.get(StorageService); + jobs = t.get(StorageJobsService); references = t.get(StorageReclaimRegistryService); for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id)); + installId = (await readAppSetting(orm, 'install_id')) ?? ''; }); beforeEach(async () => { @@ -150,6 +181,7 @@ beforeEach(async () => { } await deleteRows(orm, StorageReclaimMarks); await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } }); + await deleteRows(orm, AuditLog); named = new Map(); onAsk = null; }); @@ -165,8 +197,15 @@ afterAll(async () => { const T0 = new Date(Date.now() + 2 * DAY); const WEEK_LATER = new Date(T0.getTime() + 8 * DAY); +async function markedWeekOld(orphans: string[], kept = 4): Promise { + await seedCovers(orphans, kept); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: T0 }); +} + describe('StorageReclaimService: mark, then sweep', () => { - it('RCLS-001: a first pass only reports and marks; the report is persisted and the state carries it', async () => { + it('RCLS-001: a first pass reports and marks; the report is persisted and both states carry it', async () => { + expect(installId).toMatch(/^[0-9a-f-]{36}$/); await seedCovers(['orphan.jpg']); const report = await reclaim.run({ now: T0 }); expect(report.mode).toBe('report'); @@ -176,6 +215,7 @@ describe('StorageReclaimService: mark, then sweep', () => { unreferenced: { objects: 1, bytes: 5 }, eligible: { objects: 0, bytes: 0 }, deleted: { objects: 0, bytes: 0 }, + confirmationRequired: false, skipped: null, }); expect(report.categories.files).toMatchObject({ objects: 0, skipped: null }); @@ -183,41 +223,44 @@ describe('StorageReclaimService: mark, then sweep', () => { expect(report.categories[category].skipped).toBe('not_reclaimed'); } expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]); - expect(await reclaim.state()).toEqual({ deleteEnabled: false, lastRun: report }); - expect((await t.get(StorageAdminService).state()).reclaim).toEqual({ deleteEnabled: false, lastRun: report }); + const expected = { deleteEnabled: false, running: false, lastRun: report, lastDeleteRun: null }; + expect(await reclaim.state()).toEqual(expected); + expect((await t.get(StorageAdminService).state()).reclaim).toEqual(expected); }); - it('RCLS-002: with deletion on, a mark a week old goes through the facade and takes its mark along', async () => { - await seedCovers(['orphan.jpg']); - await reclaim.setDeleteEnabled(true); - const first = await reclaim.run({ now: T0 }); - expect(first.mode).toBe('delete'); - expect(first.categories.covers.deleted.objects).toBe(0); + it('RCLS-002: with deletion on, a mark a week old goes through the facade, takes its mark and is audited', async () => { + await markedWeekOld(['orphan.jpg']); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); const second = await reclaim.run({ now: WEEK_LATER }); + expect(second.mode).toBe('delete'); expect(second.categories.covers.eligible).toEqual({ objects: 1, bytes: 5 }); expect(second.categories.covers.deleted).toEqual({ objects: 1, bytes: 5 }); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(false); expect(fs.existsSync(objectPath('covers', 'kept-0.jpg'))).toBe(true); expect(await marksOf('covers')).toEqual([]); + expect((await reclaim.state()).lastDeleteRun).toEqual(second); + const audit = await findRow(orm, AuditLog, { action: 'storage.reclaim_deleted' }); + expect(audit?.user_id ?? null).toBeNull(); + expect(JSON.parse(String(audit?.details))).toEqual({ deleted: 1, bytes: 5, categories: { covers: 1 } }); }); - it('RCLS-003: a report-only pass deletes nothing even with deletion on', async () => { - await seedCovers(['orphan.jpg']); - await reclaim.setDeleteEnabled(true); - await reclaim.run({ now: T0 }); + it('RCLS-003: a report-only pass deletes nothing even with deletion on, and leaves the last deleting report', async () => { + await markedWeekOld(['orphan.jpg']); + const deleting = (await reclaim.state()).lastDeleteRun; const report = await reclaim.run({ now: WEEK_LATER, reportOnly: true }); expect(report.mode).toBe('report'); expect(report.categories.covers.eligible.objects).toBe(1); expect(report.categories.covers.deleted.objects).toBe(0); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + const state = await reclaim.state(); + expect(state.lastRun).toEqual(report); + expect(state.lastDeleteRun).toEqual(deleting); + expect(await countRows(orm, AuditLog, { action: 'storage.reclaim_deleted' })).toBe(0); }); it('RCLS-004: an object named again loses its mark, and its week starts over once it is unnamed', async () => { - await seedCovers(['orphan.jpg']); - await reclaim.setDeleteEnabled(true); - await reclaim.run({ now: T0 }); + await markedWeekOld(['orphan.jpg']); namedIn('covers').add('orphan.jpg'); const renamed = await reclaim.run({ now: new Date(T0.getTime() + DAY) }); expect(renamed.categories.covers.unreferenced.objects).toBe(0); @@ -230,13 +273,10 @@ describe('StorageReclaimService: mark, then sweep', () => { expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(WEEK_LATER) }]); }); - it('RCLS-005: the single-key re-check right before the delete keeps an object named in between', async () => { - await seedCovers(['orphan.jpg']); - await reclaim.setDeleteEnabled(true); - await reclaim.run({ now: T0 }); + it('RCLS-005: the fresh re-check right before the delete keeps an object named in between', async () => { + await markedWeekOld(['orphan.jpg']); onAsk = (category, keys) => { - const [only] = keys; - if (category === 'covers' && keys.length === 1 && only !== undefined) namedIn('covers').add(only); + if (category === 'covers' && keys !== null) namedIn('covers').add('orphan.jpg'); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.eligible.objects).toBe(1); @@ -250,43 +290,113 @@ describe('StorageReclaimService: mark, then sweep', () => { await put('covers', 'zero.jpg'); await put('files', 'assistant/attached.pdf'); await put('journey', 'thumbs/abcdef.jpg'); - const now = T0; - fs.utimesSync( - objectPath('covers', 'young.jpg'), - new Date(now.getTime() - 60_000), - new Date(now.getTime() - 60_000), - ); + fs.utimesSync(objectPath('covers', 'young.jpg'), new Date(T0.getTime() - 60_000), new Date(T0.getTime() - 60_000)); fs.utimesSync(objectPath('covers', 'zero.jpg'), 0, 0); - const report = await reclaim.run({ now }); + const listed = vi.spyOn(storage, 'list'); + const report = await reclaim.run({ now: T0 }); expect(report.categories.covers).toMatchObject({ objects: 2, unreferenced: { objects: 0, bytes: 0 } }); expect(report.categories.files.objects).toBe(0); expect(report.categories.journey.objects).toBe(0); expect(await marksOf('covers')).toEqual([]); + // Only a category's root is listed: the subtrees with sweeps of their own are never walked. + expect(listed).toHaveBeenCalledWith('journey', '', { shallow: true }); + }); + + it('RCLS-007: the owner marker is written into each category judged, never judged itself, and never counted', async () => { + await seedCovers(['orphan.jpg']); + await reclaim.run({ now: T0 }); + const marker = storageOwnerMarkerKey(installId); + for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { + expect(fs.readFileSync(objectPath(category, marker), 'utf8')).toBe(installId); + } + fs.utimesSync(objectPath('covers', marker), new Date(0), new Date(0)); + const again = await reclaim.run({ now: WEEK_LATER }); + expect(again.categories.covers.objects).toBe(5); + expect((await marksOf('covers')).map((mark) => mark.key)).toEqual(['orphan.jpg']); }); }); -describe('StorageReclaimService: brakes', () => { - it('RCLS-007: more than a fifth unreferenced skips the category, deletes nothing and clears its marks', async () => { +describe('StorageReclaimService: a large unreferenced share waits for the admin', () => { + it('RCLS-008: more than a fifth unreferenced marks and reports, deletes nothing, and keeps the marks', async () => { await seedCovers(['o1.jpg', 'o2.jpg'], 1); - await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'o1.jpg', size: 5 }], dbNow(new Date(0))); + await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); await reclaim.setDeleteEnabled(true); const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers).toEqual({ objects: 3, unreferenced: { objects: 2, bytes: 10 }, - eligible: { objects: 0, bytes: 0 }, + eligible: { objects: 2, bytes: 10 }, deleted: { objects: 0, bytes: 0 }, - skipped: 'too_many_unreferenced', + confirmationRequired: true, + skipped: null, }); expect(fs.existsSync(objectPath('covers', 'o1.jpg'))).toBe(true); - expect(await marksOf('covers')).toEqual([]); + expect(await marksOf('covers')).toEqual([ + { key: 'o1.jpg', first_seen_at: dbNow(new Date(0)) }, + { key: 'o2.jpg', first_seen_at: dbNow(new Date(0)) }, + ]); }); - it('RCLS-008: a category whose required source is missing, or answers for a part only, is skipped', async () => { + it('RCLS-009: an admin confirmation of the shown count lets the next pass delete; a larger share waits again', async () => { + await seedCovers(['o1.jpg', 'o2.jpg'], 1); + await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); + await reclaim.setDeleteEnabled(true); + await reclaim.run({ now: WEEK_LATER }); + const confirmed = await reclaim.confirm('covers'); + expect(confirmed.lastRun?.categories.covers.confirmationRequired).toBe(true); + expect(JSON.parse(String(await readAppSetting(orm, 'storage.reclaim_confirmed')))).toEqual({ + installId, + counts: { covers: 2 }, + }); + + await put('covers', 'o3.jpg'); + await put('covers', 'o4.jpg'); + await put('covers', 'o5.jpg'); + await markWeekOld('covers', ['o3.jpg', 'o4.jpg', 'o5.jpg']); + const bigger = await reclaim.run({ now: WEEK_LATER }); + expect(bigger.categories.covers.confirmationRequired).toBe(true); + expect(bigger.categories.covers.deleted.objects).toBe(0); + + await reclaim.confirm('covers'); + const after = await reclaim.run({ now: WEEK_LATER }); + expect(after.categories.covers).toMatchObject({ confirmationRequired: false, deleted: { objects: 5, bytes: 25 } }); + expect(fs.existsSync(objectPath('covers', 'kept-0.jpg'))).toBe(true); + }); + + it('RCLS-010: a confirmation is refused with a code when the last report holds nothing waiting', async () => { + await expect(reclaim.confirm('covers')).rejects.toMatchObject({ code: 'reclaim_nothing_to_confirm' }); await seedCovers(['orphan.jpg']); + await reclaim.run({ now: T0 }); + const refusal = await reclaim.confirm('covers').catch((err: unknown) => err); + expect(refusal).toBeInstanceOf(DomainError); + expect((refusal as DomainError).getStatus()).toBe(409); + expect((refusal as DomainError).getResponse()).toEqual({ + error: 'nothing in this category waits for a confirmation', + code: 'reclaim_nothing_to_confirm', + }); + }); + + it('RCLS-011: a confirmation another install wrote counts for nothing here', async () => { + await seedCovers(['o1.jpg', 'o2.jpg'], 1); + await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); + await setAppSetting( + orm, + 'storage.reclaim_confirmed', + JSON.stringify({ installId: '11111111-2222-4333-8444-555555555555', counts: { covers: 99 } }), + ); + await reclaim.setDeleteEnabled(true); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.confirmationRequired).toBe(true); + expect(report.categories.covers.deleted.objects).toBe(0); + }); +}); + +describe('StorageReclaimService: categories it may not judge', () => { + it('RCLS-012: a category whose required source is missing, or answers for a part only, is skipped with its marks', async () => { + await seedCovers(['orphan.jpg']); + await markWeekOld('covers', ['orphan.jpg']); const all = [...references.sources()]; - const withoutAuth = all.filter((source) => source.id !== 'auth'); - vi.spyOn(references, 'sources').mockReturnValue(withoutAuth); + vi.spyOn(references, 'sources').mockReturnValue(all.filter((source) => source.id !== 'auth')); let report = await reclaim.run({ now: T0 }); expect(report.categories.avatars.skipped).toBe('sources_missing'); expect(report.categories.covers.skipped).toBeNull(); @@ -294,28 +404,35 @@ describe('StorageReclaimService: brakes', () => { const partialTrips: StorageReferenceSource = { id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], + namedKeys: async () => new Set(), referenced: async () => new Set(), }; vi.spyOn(references, 'sources').mockReturnValue([...all.filter((source) => source.id !== 'trips'), partialTrips]); report = await reclaim.run({ now: T0 }); expect(report.categories.covers.skipped).toBe('sources_missing'); - expect(await marksOf('covers')).toEqual([]); + expect((await marksOf('covers')).map((mark) => mark.key)).toEqual(['orphan.jpg']); }); - it('RCLS-009: a source answering for a subtree only is not asked about root keys', async () => { - await seedCovers(['orphan.jpg']); + it('RCLS-013: a source answering for a subtree only is not asked about root keys before a delete', async () => { + await markedWeekOld(['orphan.jpg']); const subtree = vi.fn(async () => new Set()); vi.spyOn(references, 'sources').mockReturnValue([ ...references.sources(), - { id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], referenced: subtree }, + { + id: 'trips', + claims: [{ category: 'covers', prefix: 'sub/' }], + namedKeys: async () => new Set(), + referenced: subtree, + }, ]); - const report = await reclaim.run({ now: T0 }); - expect(report.categories.covers.unreferenced.objects).toBe(1); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.deleted.objects).toBe(1); expect(subtree).not.toHaveBeenCalled(); }); - it('RCLS-010: a category whose location holds another category is skipped', async () => { + it('RCLS-014: a category whose location holds another category is skipped with its marks', async () => { await seedCovers(['orphan.jpg']); + await markWeekOld('covers', ['orphan.jpg']); const nestedRoot = path.join(uploadsRoot, 'covers'); await setAppSetting( orm, @@ -328,6 +445,7 @@ describe('StorageReclaimService: brakes', () => { const report = await reclaim.run({ now: T0 }); expect(report.categories.covers.skipped).toBe('shared_location'); expect(report.categories.files.skipped).toBeNull(); + expect(await marksOf('covers')).toHaveLength(1); } finally { await setAppSetting(orm, 'storage.backends', backends()); await deleteRows(orm, AppSettings, { key: 'storage.categories' }); @@ -335,8 +453,31 @@ describe('StorageReclaimService: brakes', () => { } }); - it('RCLS-011: a listing that fails skips the category and clears its marks', async () => { - await orm.repo(StorageReclaimMarks).insertMissing('covers', [{ key: 'x.jpg', size: 1 }], dbNow(new Date(0))); + it('RCLS-015: another install marker in a category keeps this one off it, with its marks', async () => { + await markedWeekOld(['orphan.jpg']); + await put('covers', storageOwnerMarkerKey('99999999-8888-4777-8666-555555555555'), 'someone else'); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'foreign_install' }); + expect(report.categories.covers.deleted.objects).toBe(0); + expect(report.categories.places.skipped).toBeNull(); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + expect(await marksOf('covers')).toHaveLength(1); + }); + + it('RCLS-016: an owner marker that cannot be written leaves the category unjudged as owner_unknown', async () => { + await seedCovers(['orphan.jpg']); + const realPut = storage.put.bind(storage); + vi.spyOn(storage, 'put').mockImplementation(async (category, name, source, opts) => { + if (name.startsWith('trek-owner-')) throw new Error('read-only volume'); + return realPut(category, name, source, opts); + }); + const report = await reclaim.run({ now: T0 }); + expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'owner_unknown' }); + expect(await marksOf('covers')).toEqual([]); + }); + + it('RCLS-017: a listing that fails skips the category for the night and keeps its marks', async () => { + await markWeekOld('covers', ['x.jpg']); vi.spyOn(storage, 'list').mockImplementation(() => ({ [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(new Error('backend down')) }), })); @@ -344,11 +485,12 @@ describe('StorageReclaimService: brakes', () => { expect(report.categories.covers.skipped).toBe('list_failed'); expect(report.categories.files.skipped).toBe('list_failed'); expect(logError).toHaveBeenCalledWith(expect.stringContaining('backend down')); - expect(await marksOf('covers')).toEqual([]); + expect(await marksOf('covers')).toEqual([{ key: 'x.jpg', first_seen_at: dbNow(new Date(0)) }]); }); - it('RCLS-012: a source that fails skips the category, never reads it as unreferenced', async () => { + it('RCLS-018: a source that fails skips the category, never reads it as unreferenced, and keeps its marks', async () => { await seedCovers(['orphan.jpg']); + await markWeekOld('covers', ['orphan.jpg']); onAsk = (category) => { if (category === 'covers') throw new Error('trips table locked'); }; @@ -356,27 +498,32 @@ describe('StorageReclaimService: brakes', () => { expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'source_failed' }); expect(report.categories.places.skipped).toBeNull(); expect(logError).toHaveBeenCalledWith(expect.stringContaining('trips table locked')); + expect(await marksOf('covers')).toHaveLength(1); }); +}); - it('RCLS-013: a pass refuses on a demo instance and while another storage job holds the slot', async () => { +describe('StorageReclaimService: when a pass runs at all', () => { + it('RCLS-019: a pass refuses with a coded 409 on a demo instance and while another storage job holds the slot', async () => { const demo = vi.spyOn(t.get(RuntimeEnvService), 'isDemoMode').mockReturnValue(true); await expect(reclaim.run()).rejects.toMatchObject({ code: 'reclaim_demo' }); + expect(thrownBy(() => reclaim.start())).toMatchObject({ code: 'reclaim_demo' }); demo.mockRestore(); - const jobs = t.get(StorageJobsService); expect(jobs.claimExclusive('a test')).toBe(true); try { const refusal = await reclaim.run().catch((err: unknown) => err); expect(refusal).toBeInstanceOf(DomainError); - expect(refusal).toMatchObject({ code: 'reclaim_busy' }); expect((refusal as DomainError).getStatus()).toBe(409); + expect((refusal as DomainError).getResponse()).toEqual({ + error: 'another storage job is running, one storage job at a time', + code: 'reclaim_busy', + }); } finally { jobs.releaseExclusive('a test'); } }); - it('RCLS-014: while a pass holds the slot, a migration is refused with its name, and the slot is free after', async () => { - const jobs = t.get(StorageJobsService); + it('RCLS-020: while a pass holds the slot, a migration is refused with its name, and the slot is free after', async () => { let refusal: unknown = null; onAsk = async (category) => { if (category !== 'covers' || refusal) return; @@ -389,36 +536,41 @@ describe('StorageReclaimService: brakes', () => { jobs.releaseExclusive('after'); }); - it('RCLS-015: a database reopened mid-pass stops it before anything else is judged', async () => { - await seedCovers(['orphan.jpg']); - await put('places', 'p.jpg'); - let reopened = false; - onAsk = async (category) => { - if (category === 'covers' && !reopened) { - reopened = true; - await reopen(); - } - }; - const report = await reclaim.run({ now: T0 }); - expect(report.stopped).toBe('database_reopened'); - expect(report.categories.covers.unreferenced.objects).toBe(0); - expect(report.categories.places.objects).toBe(0); - expect(await marksOf('covers')).toEqual([]); - expect((await reclaim.state()).lastRun?.stopped).toBe('database_reopened'); + it('RCLS-021: start() runs a report-only pass in the background; the state says running until it lands', async () => { + await markedWeekOld(['orphan.jpg']); + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + onAsk = async () => gate; + reclaim.start(); + expect((await reclaim.state()).running).toBe(true); + expect(thrownBy(() => reclaim.start())).toMatchObject({ code: 'reclaim_busy' }); + release(); + await jobs.exclusiveIdle(); + const state = await reclaim.state(); + expect(state.running).toBe(false); + expect(state.lastRun?.mode).toBe('report'); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + }); + + it('RCLS-022: a pass refuses to start while a restore runs, and starts again once it ended', async () => { + await jobs.beginRestore(); + try { + await expect(reclaim.run({ now: T0 })).rejects.toMatchObject({ code: 'reclaim_busy' }); + } finally { + jobs.endRestore(); + } + expect(jobs.restoreRunning).toBe(false); + expect((await reclaim.run({ now: T0 })).stopped).toBeNull(); }); }); describe('StorageReclaimService: the delete itself', () => { - async function markedWeekOld(orphans: string[], kept = 4): Promise { - await seedCovers(orphans, kept); - await reclaim.setDeleteEnabled(true); - await reclaim.run({ now: T0 }); - } - - it('RCLS-016: a re-check that fails keeps the object', async () => { + it('RCLS-023: a re-check that fails keeps the object', async () => { await markedWeekOld(['orphan.jpg']); onAsk = (_category, keys) => { - if (keys.length === 1) throw new Error('lookup timed out'); + if (keys !== null) throw new Error('lookup timed out'); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(0); @@ -426,12 +578,11 @@ describe('StorageReclaimService: the delete itself', () => { expect(logError).toHaveBeenCalledWith(expect.stringContaining('lookup timed out')); }); - it('RCLS-017: an object gone or written again since it was judged is left and its mark dropped', async () => { + it('RCLS-024: an object gone or written again since it was judged is left and its mark dropped', async () => { await markedWeekOld(['gone.jpg', 'fresh.jpg'], 8); - // Changed between the listing and the delete: the batch looks at each object again first. const listed = storage.list.bind(storage); - vi.spyOn(storage, 'list').mockImplementation(async function* (category, prefix) { - yield* listed(category, prefix); + vi.spyOn(storage, 'list').mockImplementation(async function* (category, prefix, opts) { + yield* listed(category, prefix, opts); if (category !== 'covers') return; fs.rmSync(objectPath('covers', 'gone.jpg'), { force: true }); fs.utimesSync(objectPath('covers', 'fresh.jpg'), WEEK_LATER, WEEK_LATER); @@ -443,65 +594,123 @@ describe('StorageReclaimService: the delete itself', () => { expect(await marksOf('covers')).toEqual([]); }); - it('RCLS-024: the re-check before the delete asks about the whole batch at once, not once per object', async () => { + it('RCLS-025: the re-check before the delete asks about the whole batch at once, and the listing reads each source once', async () => { await markedWeekOld(['a.jpg', 'b.jpg', 'c.jpg'], 12); - const asked: string[][] = []; + const asked: Array = []; onAsk = (category, keys) => { - if (category === 'covers') asked.push([...keys].sort()); + if (category === 'covers') asked.push(keys === null ? null : [...keys].sort()); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(3); - // Per source: the listing's question, then one re-check for the three. - const rechecks = asked.filter((keys) => keys.length === 3 && keys.every((key) => !key.startsWith('kept-'))); - expect(rechecks.length).toBe(STORAGE_REFERENCE_SOURCE_IDS.filter((id) => id !== 'auth' && id !== 'files').length); - expect(asked.some((keys) => keys.length === 1)).toBe(false); + const freeForm = STORAGE_REFERENCE_SOURCE_IDS.filter((id) => id !== 'auth' && id !== 'files').length; + expect(asked.filter((keys) => keys === null)).toHaveLength(freeForm); + expect(asked.filter((keys) => keys !== null)).toEqual( + Array.from({ length: freeForm }, () => ['a.jpg', 'b.jpg', 'c.jpg']), + ); }); - it('RCLS-018: a delete that fails keeps the mark for the next pass', async () => { + it('RCLS-026: a delete that fails stops deleting in that category, keeps the mark and says why once', async () => { await markedWeekOld(['orphan.jpg']); vi.spyOn(storage, 'delete').mockRejectedValue(new Error('permission denied')); const report = await reclaim.run({ now: WEEK_LATER }); - expect(report.categories.covers.deleted.objects).toBe(0); + expect(report.categories.covers).toMatchObject({ skipped: 'delete_failed', deleted: { objects: 0, bytes: 0 } }); + expect(report.categories.places.skipped).toBeNull(); expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(T0) }]); - expect(logError).toHaveBeenCalledWith(expect.stringContaining('permission denied')); + expect(vi.mocked(logError).mock.calls.filter(([line]) => String(line).includes('permission denied'))).toHaveLength( + 1, + ); }); - it('RCLS-019: an object the facade keeps on delete is not counted as reclaimed', async () => { + it('RCLS-027: a stat that fails on the backend stops the category the same way', async () => { + await markedWeekOld(['orphan.jpg']); + vi.spyOn(storage, 'stat').mockRejectedValue(new Error('503 SlowDown')); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.skipped).toBe('delete_failed'); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + }); + + it('RCLS-028: an object the facade keeps on delete is not counted as reclaimed', async () => { await markedWeekOld(['orphan.jpg']); vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true); const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(0); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); }); + + it('RCLS-029: deletion switched off mid-pass stops every later delete', async () => { + await markedWeekOld(['orphan.jpg']); + await put('places', 'place-orphan.jpg'); + namedIn('places').add('p1.jpg'); + await Promise.all(['p1.jpg', 'p2.jpg', 'p3.jpg', 'p4.jpg', 'p5.jpg'].map((key) => put('places', key))); + ['p2.jpg', 'p3.jpg', 'p4.jpg', 'p5.jpg'].forEach((key) => namedIn('places').add(key)); + await markWeekOld('places', ['place-orphan.jpg']); + const realDelete = storage.delete.bind(storage); + vi.spyOn(storage, 'delete').mockImplementation(async (category, name) => { + await realDelete(category, name); + if (category === 'covers') await reclaim.setDeleteEnabled(false); + }); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.categories.covers.deleted.objects).toBe(1); + expect(report.categories.places).toMatchObject({ eligible: { objects: 1 }, deleted: { objects: 0 } }); + expect(fs.existsSync(objectPath('places', 'place-orphan.jpg'))).toBe(true); + }); + + it('RCLS-030: nothing is deleted while a backup reads the uploads', async () => { + await markedWeekOld(['orphan.jpg']); + jobs.beginBackup(); + try { + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.mode).toBe('delete'); + expect(report.categories.covers).toMatchObject({ eligible: { objects: 1 }, deleted: { objects: 0 } }); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + } finally { + jobs.endBackup(); + } + }); }); -describe('StorageReclaimService: the switch and the stored report', () => { - it('RCLS-020: the switch is stored as true or false and only the exact true deletes', async () => { - expect((await reclaim.setDeleteEnabled(true)).deleteEnabled).toBe(true); - expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBe('true'); - expect((await reclaim.setDeleteEnabled(false)).deleteEnabled).toBe(false); - await setAppSetting(orm, 'storage.reclaim_delete', 'yes'); - expect((await reclaim.state()).deleteEnabled).toBe(false); +describe('StorageReclaimService: a pass that stops, and a restore', () => { + it('RCLS-031: a pass stopped by a reopen keeps what it deleted and marks every unfinished category interrupted', async () => { + await markedWeekOld(['orphan.jpg']); + await put('places', 'p.jpg'); + const realDelete = storage.delete.bind(storage); + vi.spyOn(storage, 'delete').mockImplementation(async (category, name) => { + await realDelete(category, name); + if (category === 'covers') await reopen(); + }); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.stopped).toBe('database_reopened'); + expect(report.categories.covers).toMatchObject({ skipped: null, deleted: { objects: 1, bytes: 5 } }); + for (const category of ['avatars', 'places'] as const) { + expect(report.categories[category]).toMatchObject({ objects: 0, skipped: 'interrupted' }); + } + expect((await reclaim.state()).lastRun?.stopped).toBe('database_reopened'); + expect(await countRows(orm, AuditLog, { action: 'storage.reclaim_deleted' })).toBe(1); }); - it('RCLS-021: a stored report that no longer parses reads as never run', async () => { - await setAppSetting(orm, 'storage.reclaim_report', '{not json'); - expect((await reclaim.state()).lastRun).toBeNull(); - await setAppSetting(orm, 'storage.reclaim_report', JSON.stringify({ startedAt: 'yesterday' })); - expect((await reclaim.state()).lastRun).toBeNull(); + it('RCLS-032: an unexpected failure part way still persists the report with what was deleted', async () => { + await markedWeekOld(['orphan.jpg']); + const spy = vi + .spyOn(StorageReclaimMarksRepository.prototype, 'deleteKeys') + .mockRejectedValueOnce(new Error('disk I/O error')); + try { + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.stopped).toBe('failed'); + expect(report.categories.covers).toMatchObject({ skipped: 'interrupted', deleted: { objects: 1, bytes: 5 } }); + expect(report.categories.places.skipped).toBe('interrupted'); + expect((await reclaim.state()).lastDeleteRun?.stopped).toBe('failed'); + expect(logError).toHaveBeenCalledWith(expect.stringContaining('disk I/O error')); + } finally { + spy.mockRestore(); + } }); - it('RCLS-022: a restore that begins mid-pass stops it before the delete, and waits for it to let go', async () => { - await seedCovers(['orphan.jpg']); - await reclaim.setDeleteEnabled(true); - await reclaim.run({ now: T0 }); - const jobs = t.get(StorageJobsService); + it('RCLS-033: a restore that begins mid-pass stops it before the delete, and waits for it to let go', async () => { + await markedWeekOld(['orphan.jpg']); let restoreStarted: Promise | null = null; - let restoreAsked = false; let restoreWent = false; onAsk = (_category, keys) => { - if (keys.length !== 1 || restoreAsked) return; - restoreAsked = true; + if (keys === null || restoreStarted) return; restoreStarted = jobs.beginRestore().then(() => { restoreWent = true; }); @@ -509,8 +718,8 @@ describe('StorageReclaimService: the switch and the stored report', () => { try { const report = await reclaim.run({ now: WEEK_LATER }); expect(report.stopped).toBe('database_reopened'); + expect(report.categories.covers.skipped).toBe('interrupted'); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); - expect(restoreAsked).toBe(true); await restoreStarted; expect(restoreWent).toBe(true); } finally { @@ -518,15 +727,40 @@ describe('StorageReclaimService: the switch and the stored report', () => { } }); - it('RCLS-023: a pass refuses to start while a restore runs, and starts again once it ended', async () => { - const jobs = t.get(StorageJobsService); - await jobs.beginRestore(); - try { - await expect(reclaim.run({ now: T0 })).rejects.toMatchObject({ code: 'reclaim_busy' }); - } finally { - jobs.endRestore(); - } - expect(jobs.restoreRunning).toBe(false); - expect((await reclaim.run({ now: T0 })).stopped).toBeNull(); + it('RCLS-034: a reopen turns deletion off and drops marks, confirmations and reports, but keeps the install id', async () => { + await markedWeekOld(['orphan.jpg']); + await setAppSetting(orm, 'storage.reclaim_confirmed', JSON.stringify({ installId, counts: { covers: 1 } })); + // A restored database carries another install's id and switch. + await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + await setAppSetting(orm, 'storage.reclaim_delete', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + await reopen(); + expect(await readAppSetting(orm, 'install_id')).toBe(installId); + const reset = ['storage.reclaim_delete', 'storage.reclaim_confirmed', 'storage.reclaim_report']; + expect(await Promise.all(reset.map((key) => readAppSetting(orm, key)))).toEqual([null, null, null]); + expect(await marksOf('covers')).toEqual([]); + expect(await reclaim.state()).toEqual({ deleteEnabled: false, running: false, lastRun: null, lastDeleteRun: null }); + }); +}); + +describe('StorageReclaimService: the switch and the stored reports', () => { + it('RCLS-035: the switch is stored as this install id; any other value, another install id included, reads off', async () => { + expect((await reclaim.setDeleteEnabled(true)).deleteEnabled).toBe(true); + expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBe(installId); + expect((await reclaim.setDeleteEnabled(false)).deleteEnabled).toBe(false); + expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBeNull(); + const readsOff = async (value: string): Promise => { + await setAppSetting(orm, 'storage.reclaim_delete', value); + return (await reclaim.state()).deleteEnabled; + }; + expect(await readsOff('true')).toBe(false); + expect(await readsOff('yes')).toBe(false); + expect(await readsOff('12345678-aaaa-4bbb-8ccc-1234567890ab')).toBe(false); + }); + + it('RCLS-036: a stored report that no longer parses reads as never run', async () => { + await setAppSetting(orm, 'storage.reclaim_report', '{not json'); + expect((await reclaim.state()).lastRun).toBeNull(); + await setAppSetting(orm, 'storage.reclaim_delete_report', JSON.stringify({ startedAt: 'yesterday' })); + expect((await reclaim.state()).lastDeleteRun).toBeNull(); }); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.types.test.ts b/server/tests/unit/nest/storage/storage-reclaim.types.test.ts index e8573fcaf..8526ed64f 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.types.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.types.test.ts @@ -3,11 +3,15 @@ * stored value names, in every spelling the columns hold. A key reported too * many only keeps an object; one missed would delete it, so the cases lean on * the spellings the validators let through (absolute self-URLs, queries, - * relative journey covers, legacy file names). RCLP-001 to RCLP-012. + * relative journey covers, legacy file names) and on object names that hold + * the characters a URL treats as special. RCLP-001 to RCLP-016. */ import { - referencedAmong, + keySearchCore, + keysNamedIn, storageKeysIn, + storageOwnerMarkerInstall, + storageOwnerMarkerKey, uploadKeysIn, type StorageKeyOptions, type StorageKeyRef, @@ -15,44 +19,42 @@ import { import { describe, expect, it } from 'vitest'; -const ref = (category: string, key: string): StorageKeyRef => ({ category, key }); +/** The keys of one category among the refs, as a set: the order and the repeats do not matter to a source. */ +function keysIn(refs: StorageKeyRef[], category: string): Set { + return new Set(refs.filter((ref) => ref.category === category).map((ref) => ref.key)); +} describe('uploadKeysIn', () => { it('RCLP-001: finds the path in its plain, slash-less and absolute self-URL spellings', () => { - expect(uploadKeysIn('/uploads/covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]); - expect(uploadKeysIn('uploads/files/b.pdf')).toEqual([ref('files', 'b.pdf')]); - expect(uploadKeysIn('https://trip.example.com/uploads/places/c.png')).toEqual([ref('places', 'c.png')]); - expect(uploadKeysIn('HTTPS://HOST/Uploads/Covers/D.JPG')).toEqual([ref('covers', 'D.JPG')]); + expect(keysIn(uploadKeysIn('/uploads/covers/a.jpg'), 'covers')).toEqual(new Set(['a.jpg'])); + expect(keysIn(uploadKeysIn('uploads/files/b.pdf'), 'files')).toEqual(new Set(['b.pdf'])); + expect(keysIn(uploadKeysIn('https://trip.example.com/uploads/places/c.png'), 'places')).toEqual(new Set(['c.png'])); + expect(keysIn(uploadKeysIn('HTTPS://HOST/Uploads/Covers/D.JPG'), 'covers')).toEqual(new Set(['D.JPG'])); }); - it('RCLP-002: drops a query or a fragment, decodes and trims the key', () => { - expect(uploadKeysIn('/uploads/places/x.jpg?v=2')).toEqual([ref('places', 'x.jpg')]); - expect(uploadKeysIn('/uploads/places/x.jpg#top')).toEqual([ref('places', 'x.jpg')]); - expect(uploadKeysIn(' /uploads/covers/y.jpg ')).toEqual([ref('covers', 'y.jpg')]); - expect(uploadKeysIn('/uploads/photos/my%20photo.jpg')).toEqual([ - ref('photos', 'my%20photo.jpg'), - ref('photos', 'my photo.jpg'), - ref('photos', 'my'), - ]); + it('RCLP-002: reports the key as it stands and cut at a query or fragment, decoded and trimmed', () => { + expect(keysIn(uploadKeysIn('/uploads/places/x.jpg?v=2'), 'places')).toEqual(new Set(['x.jpg?v=2', 'x.jpg'])); + expect(keysIn(uploadKeysIn('/uploads/places/x.jpg#top'), 'places')).toEqual(new Set(['x.jpg#top', 'x.jpg'])); + expect([...keysIn(uploadKeysIn(' /uploads/covers/y.jpg '), 'covers')]).toContain('y.jpg'); + expect(keysIn(uploadKeysIn('/uploads/photos/my%20photo.jpg'), 'photos')).toEqual( + new Set(['my%20photo.jpg', 'my photo.jpg', 'my']), + ); }); it('RCLP-003: finds every path in a text, each cut where the next begins or the text ends', () => { - expect(uploadKeysIn('see /uploads/covers/a.jpg and "/uploads/places/b.jpg" too')).toEqual([ - ref('covers', 'a.jpg and'), - ref('covers', 'a.jpg'), - ref('places', 'b.jpg'), - ]); - expect(uploadKeysIn('/uploads/covers/a.jpg/uploads/places/b.jpg')).toEqual([ - ref('covers', 'a.jpg'), - ref('places', 'b.jpg'), - ]); + const refs = uploadKeysIn('see /uploads/covers/a.jpg and "/uploads/places/b.jpg" too'); + expect(keysIn(refs, 'covers')).toEqual(new Set(['a.jpg and ', 'a.jpg and', 'a.jpg'])); + expect(keysIn(refs, 'places')).toEqual(new Set(['b.jpg'])); + const joined = uploadKeysIn('/uploads/covers/a.jpg/uploads/places/b.jpg'); + expect(keysIn(joined, 'covers')).toEqual(new Set(['a.jpg'])); + expect(keysIn(joined, 'places')).toEqual(new Set(['b.jpg'])); }); - it('RCLP-004: keeps a nested key whole and refuses empty or climbing keys', () => { - expect(uploadKeysIn('/uploads/journey/thumbs/abc.jpg')).toEqual([ref('journey', 'thumbs/abc.jpg')]); + it('RCLP-004: keeps a nested key whole and reports nothing for an empty one', () => { + expect(keysIn(uploadKeysIn('/uploads/journey/thumbs/abc.jpg'), 'journey')).toEqual(new Set(['thumbs/abc.jpg'])); expect(uploadKeysIn('/uploads/covers/')).toEqual([]); - expect(uploadKeysIn('/uploads/covers/../avatars/a.jpg')).toEqual([]); - expect(uploadKeysIn('/uploads/covers/%2E%2E/x')).toEqual([ref('covers', '%2E%2E/x')]); + // A climbing key names no object a listing yields; reporting it only keeps. + expect([...keysIn(uploadKeysIn('/uploads/covers/%2E%2E/x'), 'covers')]).toContain('%2E%2E/x'); }); it('RCLP-005: names nothing for empty values, other hosts and other paths', () => { @@ -65,61 +67,87 @@ describe('uploadKeysIn', () => { }); it('RCLP-006: a broken percent sequence keeps the raw key', () => { - expect(uploadKeysIn('/uploads/covers/100%.jpg')).toEqual([ref('covers', '100%.jpg')]); + expect(keysIn(uploadKeysIn('/uploads/covers/100%.jpg'), 'covers')).toEqual(new Set(['100%.jpg'])); }); }); describe('storageKeysIn', () => { it('RCLP-007: reads the relative / spelling of journey covers', () => { - expect(storageKeysIn('covers/a.jpg')).toEqual([ref('covers', 'a.jpg')]); - expect(storageKeysIn('journey/b.jpg?x=1')).toEqual([ref('journey', 'b.jpg')]); - expect(storageKeysIn('./journey/thumbs/c.jpg')).toEqual([ref('journey', 'thumbs/c.jpg')]); + expect(keysIn(storageKeysIn('covers/a.jpg'), 'covers')).toEqual(new Set(['a.jpg'])); + expect(keysIn(storageKeysIn('journey/b.jpg?x=1'), 'journey')).toEqual(new Set(['b.jpg?x=1', 'b.jpg'])); + expect(keysIn(storageKeysIn('./journey/thumbs/c.jpg'), 'journey')).toEqual(new Set(['thumbs/c.jpg'])); expect(storageKeysIn('notacategory/a.jpg')).toEqual([]); }); it('RCLP-008: a bare name belongs to the column category, a URL or a path does not', () => { - expect(storageKeysIn('a1b2.jpg', { bare: 'avatars' })).toEqual([ref('avatars', 'a1b2.jpg')]); + expect(keysIn(storageKeysIn('a1b2.jpg', { bare: 'avatars' }), 'avatars')).toEqual(new Set(['a1b2.jpg'])); expect(storageKeysIn('https://idp.example.com/pic.jpg', { bare: 'avatars' })).toEqual([]); - expect(storageKeysIn('some/dir/x.pdf', { bare: 'files' })).toEqual([]); + expect(keysIn(storageKeysIn('some/dir/x.pdf', { bare: 'files' }), 'files')).toEqual(new Set()); expect(storageKeysIn('a1b2.jpg')).toEqual([]); }); it('RCLP-009: with basename, a legacy path still names its last segment', () => { - expect(storageKeysIn('files/x.pdf', { bare: 'files', basename: true })).toEqual([ - ref('files', 'x.pdf'), - ref('files', 'x.pdf'), - ]); - expect(storageKeysIn('/uploads/files/y.pdf', { bare: 'files', basename: true })).toEqual([ - ref('files', 'y.pdf'), - ref('files', 'y.pdf'), - ]); - expect(storageKeysIn('C:\\old\\uploads\\z.pdf', { bare: 'files', basename: true })).toEqual([ - ref('files', 'z.pdf'), - ]); + const options: StorageKeyOptions = { bare: 'files', basename: true }; + expect(keysIn(storageKeysIn('files/x.pdf', options), 'files')).toEqual(new Set(['x.pdf'])); + expect(keysIn(storageKeysIn('/uploads/files/y.pdf', options), 'files')).toEqual(new Set(['y.pdf'])); + expect(keysIn(storageKeysIn('C:\\old\\uploads\\z.pdf', options), 'files')).toEqual(new Set(['z.pdf'])); }); - it('RCLP-010: names nothing for a blank or a non-string', () => { + it('RCLP-010: names nothing for a blank or a non-string, and only the raw text for a bare query', () => { expect(storageKeysIn(' ', { bare: 'files' })).toEqual([]); - expect(storageKeysIn('?only=query', { bare: 'files' })).toEqual([]); + expect(keysIn(storageKeysIn('?only=query', { bare: 'files' }), 'files')).toEqual(new Set(['?only=query'])); expect(storageKeysIn(null)).toEqual([]); expect(storageKeysIn(42 as unknown as string)).toEqual([]); }); + + it('RCLP-013: a bare name holding a #, a ? or a trailing blank is reported untouched first', () => { + // An upload's key is , and extname takes whatever follows the last dot. + const options: StorageKeyOptions = { bare: 'files', basename: true }; + expect([...keysIn(storageKeysIn('0f8e2b7c. #4711', options), 'files')]).toContain('0f8e2b7c. #4711'); + expect([...keysIn(storageKeysIn('0f8e2b7c.c#', options), 'files')]).toContain('0f8e2b7c.c#'); + expect([...keysIn(storageKeysIn('0f8e2b7c.v2?final', options), 'files')]).toContain('0f8e2b7c.v2?final'); + expect([...keysIn(storageKeysIn('0f8e2b7c.pdf ', options), 'files')]).toContain('0f8e2b7c.pdf '); + expect([...keysIn(storageKeysIn('legacy/0f8e2b7c. #4711', options), 'files')]).toContain('0f8e2b7c. #4711'); + // The URL spellings keep the raw tail too, so a path naming such an object keeps it. + expect([...keysIn(storageKeysIn('/uploads/files/0f8e2b7c.c#'), 'files')]).toContain('0f8e2b7c.c#'); + }); }); -describe('referencedAmong', () => { - it('RCLP-011: answers the candidates of one category the values name, without regard to case', () => { +describe('keysNamedIn', () => { + it('RCLP-011: answers the keys of one category the values name, lowercased', () => { const values = ['/uploads/covers/A.JPG', 'covers/b.jpg', '/uploads/places/c.jpg', null, 'https://elsewhere/x']; - expect(referencedAmong('covers', ['a.jpg', 'b.jpg', 'c.jpg', 'd.jpg'], values)).toEqual( - new Set(['a.jpg', 'b.jpg']), - ); - expect(referencedAmong('places', ['c.jpg'], values)).toEqual(new Set(['c.jpg'])); + expect(keysNamedIn('covers', values)).toEqual(new Set(['a.jpg', 'b.jpg'])); + expect(keysNamedIn('places', values)).toEqual(new Set(['c.jpg'])); }); it('RCLP-012: passes the column options on, so a bare avatar name keeps its object', () => { const avatarColumn: StorageKeyOptions = { bare: 'avatars' }; - expect(referencedAmong('avatars', ['u1.png'], ['u1.png'], avatarColumn)).toEqual(new Set(['u1.png'])); - expect( - referencedAmong('avatars', ['u1.png', 'u2.png'], ['u1.png', 'https://idp/u2.png'], { bare: 'avatars' }), - ).toEqual(new Set(['u1.png'])); + expect(keysNamedIn('avatars', ['U1.png', 'https://idp/u2.png'], avatarColumn)).toEqual(new Set(['u1.png'])); + }); +}); + +describe('keySearchCore', () => { + it('RCLP-014: the longest plain run a row naming the key in any spelling holds verbatim', () => { + expect(keySearchCore('0F8E2B7C-1D2A-4C3B-9E8F-7A6B5C4D3E2F.jpg')).toBe('0f8e2b7c-1d2a-4c3b-9e8f-7a6b5c4d3e2f'); + expect(keySearchCore('0f8e2b7c-1d2a. #4711')).toBe('0f8e2b7c-1d2a'); + expect(keySearchCore('my photo.jpg')).toBeNull(); + expect(keySearchCore('a.jpg')).toBeNull(); + }); +}); + +describe('owner markers', () => { + it('RCLP-015: the marker key names the install, and only a marker key reads as one', () => { + const id = '0f8e2b7c-1d2a-4c3b-9e8f-7a6b5c4d3e2f'; + expect(storageOwnerMarkerKey(id)).toBe(`trek-owner-${id}`); + expect(storageOwnerMarkerInstall(storageOwnerMarkerKey(id))).toBe(id); + expect(storageOwnerMarkerInstall('0f8e2b7c.jpg')).toBeNull(); + expect(storageOwnerMarkerInstall('trek-owner-')).toBeNull(); + expect(storageOwnerMarkerInstall(`sub/trek-owner-${id}`)).toBeNull(); + }); + + it('RCLP-016: a marker key is a valid root-level key no upload is ever named', () => { + const key = storageOwnerMarkerKey('0f8e2b7c-1d2a-4c3b-9e8f-7a6b5c4d3e2f'); + expect(key.includes('/')).toBe(false); + expect(key.startsWith('.')).toBe(false); }); }); diff --git a/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts b/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts index 5fd8e688f..d4eb47663 100644 --- a/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts +++ b/server/tests/unit/nest/storage/storage-usage-scan.job.test.ts @@ -5,6 +5,7 @@ * the tick error-containment mirrors VersionCheckJob/DemoResetJob there. */ import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service'; +import type { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; import type { StorageStatsService } from '../../../../src/nest/storage/storage-stats.service'; import { StorageUsageScanJob } from '../../../../src/nest/storage/storage-usage-scan.job'; import { Logger } from '@nestjs/common'; @@ -30,19 +31,24 @@ function statsStub(): StorageStatsService { return { scan: vi.fn().mockResolvedValue(undefined) } as unknown as StorageStatsService; } +/** No exclusive pass holds the slot: the scan goes at once. */ +function jobsStub(idle: Promise = Promise.resolve()): StorageJobsService { + return { exclusiveIdle: vi.fn(() => idle) } as unknown as StorageJobsService; +} + beforeEach(() => vi.restoreAllMocks()); describe('StorageUsageScanJob', () => { it('SCAN-001 registers the nightly 04:15 cron when the registrar gate is open', async () => { const registrar = registrarStub(); - const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub()); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub(), jobsStub()); await job.onApplicationBootstrap(); expect(registrar.register).toHaveBeenCalledWith('storage-usage-scan', '15 4 * * *', expect.any(Function)); }); it('SCAN-002 does not register under the test gate (registrar.isEnabled() false)', async () => { const registrar = registrarStub(false); - const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub()); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, statsStub(), jobsStub()); await job.onApplicationBootstrap(); expect(registrar.register).not.toHaveBeenCalled(); }); @@ -50,7 +56,7 @@ describe('StorageUsageScanJob', () => { it('SCAN-003 the registered callback drives tick(), which calls stats.scan()', async () => { const registrar = registrarStub(); const stats = statsStub(); - const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, stats); + const job = new StorageUsageScanJob(registrar as unknown as CronRegistrarService, stats, jobsStub()); await job.onApplicationBootstrap(); const onTick = registrar.register.mock.calls[0]![2] as () => void; onTick(); @@ -60,7 +66,7 @@ describe('StorageUsageScanJob', () => { it('SCAN-004 tick() resolves and contains an Error rejection to the usage scan log line', async () => { const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); const stats = { scan: vi.fn().mockRejectedValue(new Error('disk full')) } as unknown as StorageStatsService; - const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats); + const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats, jobsStub()); await expect(job.tick()).resolves.toBeUndefined(); expect(errorSpy).toHaveBeenCalledWith('usage scan failed: disk full'); }); @@ -68,8 +74,23 @@ describe('StorageUsageScanJob', () => { it('SCAN-005 tick() stringifies a non-Error rejection rather than crashing', async () => { const errorSpy = vi.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); const stats = { scan: vi.fn().mockRejectedValue('backend unreachable') } as unknown as StorageStatsService; - const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats); + const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats, jobsStub()); await expect(job.tick()).resolves.toBeUndefined(); expect(errorSpy).toHaveBeenCalledWith('usage scan failed: backend unreachable'); }); + + it('SCAN-006 tick() waits for a reclaim pass that still holds the slot before it scans', async () => { + let release: () => void = () => undefined; + const idle = new Promise((resolve) => { + release = resolve; + }); + const stats = statsStub(); + const job = new StorageUsageScanJob(registrarStub() as unknown as CronRegistrarService, stats, jobsStub(idle)); + const ticked = job.tick(); + await Promise.resolve(); + expect(stats.scan).not.toHaveBeenCalled(); + release(); + await ticked; + expect(stats.scan).toHaveBeenCalledTimes(1); + }); }); diff --git a/shared/src/storage/storage.schema.spec.ts b/shared/src/storage/storage.schema.spec.ts index d34be2f91..3bc8cbf64 100644 --- a/shared/src/storage/storage.schema.spec.ts +++ b/shared/src/storage/storage.schema.spec.ts @@ -6,8 +6,10 @@ import { storageAdminStateSchema, storageBackfillStatusSchema, storageConfigSchema, + STORAGE_RECLAIM_SKIP_REASONS, + storageReclaimConfirmRequestSchema, storageReclaimReportSchema, - storageReclaimRunRequestSchema, + storageReclaimRunStartedSchema, storageReclaimSettingsRequestSchema, storageReclaimStateSchema, storageSecretFields, @@ -191,7 +193,14 @@ describe('backfill + usage wire shapes', () => { describe('the reclaim contract', () => { const counts = { objects: 0, bytes: 0 }; - const perCategory = { objects: 3, unreferenced: counts, eligible: counts, deleted: counts, skipped: null }; + const perCategory = { + objects: 3, + unreferenced: counts, + eligible: counts, + deleted: counts, + confirmationRequired: false, + skipped: null, + }; const report = { startedAt: 1_700_000_000_000, finishedAt: 1_700_000_001_000, @@ -205,25 +214,60 @@ describe('the reclaim contract', () => { expect(storageReclaimReportSchema.parse(report)).toEqual(report); }); - it('a report missing a category, or naming an unknown skip reason, is refused', () => { + it('a report missing a category, a confirmation flag, or naming an unknown skip reason, is refused', () => { const { photos: _photos, ...withoutPhotos } = report.categories; expect(storageReclaimReportSchema.safeParse({ ...report, categories: withoutPhotos }).success).toBe(false); const odd = { ...report, categories: { ...report.categories, covers: { ...perCategory, skipped: 'tired' } } }; expect(storageReclaimReportSchema.safeParse(odd).success).toBe(false); + const { confirmationRequired: _flag, ...unflagged } = perCategory; + const old = { ...report, categories: { ...report.categories, covers: unflagged } }; + expect(storageReclaimReportSchema.safeParse(old).success).toBe(false); }); - it('the admin state carries the switch and a nullable last run', () => { + it('names every skip reason a pass reports, the brake that used to skip a category gone', () => { + expect([...STORAGE_RECLAIM_SKIP_REASONS]).toEqual([ + 'not_reclaimed', + 'sources_missing', + 'shared_location', + 'foreign_install', + 'owner_unknown', + 'list_failed', + 'source_failed', + 'delete_failed', + 'interrupted', + ]); + for (const skipped of STORAGE_RECLAIM_SKIP_REASONS) { + const row = { ...report, categories: { ...report.categories, covers: { ...perCategory, skipped } } }; + expect(storageReclaimReportSchema.safeParse(row).success).toBe(true); + } + expect(storageReclaimReportSchema.safeParse({ ...report, stopped: 'failed' }).success).toBe(true); + expect(storageReclaimReportSchema.safeParse({ ...report, stopped: 'tired' }).success).toBe(false); + }); + + it('the admin state carries the switch, the running flag and both nullable reports', () => { const shape = storageAdminStateSchema.pick({ reclaim: true }); - expect(shape.parse({ reclaim: { deleteEnabled: false, lastRun: null } })).toEqual({ - reclaim: { deleteEnabled: false, lastRun: null }, - }); - expect(storageReclaimStateSchema.safeParse({ deleteEnabled: 'yes', lastRun: null }).success).toBe(false); + const idle = { deleteEnabled: false, running: false, lastRun: null, lastDeleteRun: null }; + expect(shape.parse({ reclaim: idle })).toEqual({ reclaim: idle }); + expect(shape.parse({ reclaim: null })).toEqual({ reclaim: null }); + expect(storageReclaimStateSchema.parse({ ...idle, running: true, lastRun: report, lastDeleteRun: report })).toEqual( + { + ...idle, + running: true, + lastRun: report, + lastDeleteRun: report, + }, + ); + expect(storageReclaimStateSchema.safeParse({ ...idle, deleteEnabled: 'yes' }).success).toBe(false); + expect(storageReclaimStateSchema.safeParse({ deleteEnabled: false, lastRun: null }).success).toBe(false); }); - it('the requests are closed objects', () => { + it('the requests are closed objects, and a started check answers started: true', () => { expect(storageReclaimSettingsRequestSchema.parse({ deleteEnabled: true })).toEqual({ deleteEnabled: true }); expect(storageReclaimSettingsRequestSchema.safeParse({ deleteEnabled: true, extra: 1 }).success).toBe(false); - expect(storageReclaimRunRequestSchema.parse({})).toEqual({}); - expect(storageReclaimRunRequestSchema.safeParse({ reportOnly: 'no' }).success).toBe(false); + expect(storageReclaimConfirmRequestSchema.parse({ category: 'covers' })).toEqual({ category: 'covers' }); + expect(storageReclaimConfirmRequestSchema.safeParse({ category: 'nowhere' }).success).toBe(false); + expect(storageReclaimConfirmRequestSchema.safeParse({ category: 'covers', count: 3 }).success).toBe(false); + expect(storageReclaimRunStartedSchema.parse({ started: true })).toEqual({ started: true }); + expect(storageReclaimRunStartedSchema.safeParse({ started: false }).success).toBe(false); }); }); diff --git a/shared/src/storage/storage.schema.ts b/shared/src/storage/storage.schema.ts index d7dda6fe0..d23c99c1d 100644 --- a/shared/src/storage/storage.schema.ts +++ b/shared/src/storage/storage.schema.ts @@ -283,20 +283,27 @@ export const STORAGE_RECLAIM_CATEGORIES = [...STORAGE_CATEGORIES, 'photos'] as c export const storageReclaimCategorySchema = z.enum(STORAGE_RECLAIM_CATEGORIES); /** - * Why a category deleted nothing on a pass, whatever the switch says: + * Why a category was not judged on a pass, or stopped short, whatever the switch says: * - `not_reclaimed`: the category is never reclaimed (a cache, the backups, the legacy photos folder); * - `sources_missing`: a domain whose rows can name its objects did not answer, so nothing can be proven unreferenced; * - `shared_location`: its objects sit where another category's do, or where they cannot be told apart; - * - `too_many_unreferenced`: more than a fifth of its objects look unreferenced, which points at a shared bucket; - * - `list_failed` / `source_failed`: the listing or a reference lookup failed, so the pass stopped there. + * - `foreign_install`: another TREK installation keeps its uploads in the same place (its owner marker is there); + * - `owner_unknown`: this installation's owner marker could not be written there, so a shared place cannot be ruled out; + * - `list_failed` / `source_failed`: the listing or a reference lookup failed, so the pass stopped there; + * - `delete_failed`: deleting failed on the backend, so the pass stopped deleting in this category; + * - `interrupted`: the pass stopped (`stopped` says why) before it finished this category; its figures are partial. + * A skip keeps the marks it found: only a pass that judges the category moves them. */ export const STORAGE_RECLAIM_SKIP_REASONS = [ 'not_reclaimed', 'sources_missing', 'shared_location', - 'too_many_unreferenced', + 'foreign_install', + 'owner_unknown', 'list_failed', 'source_failed', + 'delete_failed', + 'interrupted', ] as const; const storageObjectCountSchema = z.object({ objects: z.number().int().nonnegative(), bytes: z.number().nonnegative() }); @@ -309,6 +316,11 @@ export const storageReclaimCategoryReportSchema = z.object({ /** Unreferenced on every pass for at least a week: what a pass with deletion on removes. */ eligible: storageObjectCountSchema, deleted: storageObjectCountSchema, + /** + * More than a fifth of the category is unreferenced, and more than an admin + * last confirmed: deletion waits for a confirmation (POST .../reclaim/confirm). + */ + confirmationRequired: z.boolean(), skipped: z.enum(STORAGE_RECLAIM_SKIP_REASONS).nullable(), }); export type StorageReclaimCategoryReport = z.infer; @@ -316,27 +328,41 @@ export type StorageReclaimCategoryReport = z.infer; export const storageReclaimStateSchema = z.object({ - /** Off after install and upgrade: a pass then only reports. */ + /** Off after install, upgrade and every backup restore: a pass then only reports. */ deleteEnabled: z.boolean(), + /** A pass is running now (the nightly one or a check started by hand). */ + running: z.boolean(), + /** The last pass of any kind; null before the first. */ lastRun: storageReclaimReportSchema.nullable(), + /** The last pass that ran with deletion on, which a check started by hand never replaces; null before one. */ + lastDeleteRun: storageReclaimReportSchema.nullable(), }); export type StorageReclaimState = z.infer; export const storageReclaimSettingsRequestSchema = z.strictObject({ deleteEnabled: z.boolean() }); export type StorageReclaimSettingsRequest = z.infer; -/** `reportOnly` runs a pass that deletes nothing even with deletion switched on. */ -export const storageReclaimRunRequestSchema = z.strictObject({ reportOnly: z.boolean().optional() }); -export type StorageReclaimRunRequest = z.infer; +/** POST .../reclaim/run answers 202 at once; the pass reports through the admin state. */ +export const storageReclaimRunStartedSchema = z.object({ started: z.literal(true) }); + +/** + * Lets the pass delete in a category whose unreferenced share is over a + * fifth, up to the count the last report showed for it. + */ +export const storageReclaimConfirmRequestSchema = z.strictObject({ category: storageReclaimCategorySchema }); +export type StorageReclaimConfirmRequest = z.infer; export const storageAdminStateSchema = z.object({ backends: z.array( diff --git a/wiki/Admin-Storage.md b/wiki/Admin-Storage.md index 367197ea5..e7a2ffce5 100644 --- a/wiki/Admin-Storage.md +++ b/wiki/Admin-Storage.md @@ -127,33 +127,70 @@ includes the legacy photo library. ## Unreferenced files -Uploaded files can outlive what used them: deleting a trip, a user or a place -image, or replacing a cover, leaves the old file in storage. Every night -(04:05 server time) TREK checks the covers, place images, avatars, trip files -and journal photos for files that no row names any more, and the -**Unreferenced files** section shows what it found: +Uploaded files can outlive what used them. When you delete or replace +something that shows a file (a trip or list cover, a place image, a journal +photo, an avatar, a trip file), TREK deletes that file right away, unless +something else still shows it: a copied trip, a journey that took over the +trip cover, a saved place in a list. Whatever is left over after that (files of +deleted trips and users, files from before this check existed) is found by a +check every night at 04:05 server time. It looks at the covers, place images, +avatars, trip files and journal photos for files that nothing in TREK names +any more, and the **Unreferenced files** section shows what it found: -- **Can be freed**: unused for at least a week, so the nightly run deletes them - once deletion is on. +- **Can be freed**: unused on every check for at least a week, so the nightly + check deletes them once deletion is on. - **Unused for less than a week**: these get their week first. A file that is used again in the meantime (an undo, a copied trip) starts over. -- **Freed on the last check**: what the nightly run deleted. +- **Freed on the last check**: what the last nightly check with deletion on + deleted. A check you start by hand never deletes, so it never changes this + figure. **Delete unreferenced files** is off after install and upgrade, so the check -only reports. Switched on, the nightly run deletes a file once it has been -unused for a week and a last check right before the delete still finds -nothing that uses it. **Check now** runs a check that only reports, whatever -the switch says. Each run is recorded in the [Audit-Log](Audit-Log). +only reports. Switched on, the nightly check deletes a file once it has been +unused for a week and one more look right before the delete still finds +nothing that uses it. **Check now** starts a check in the background that only +reports, whatever the switch says; the section shows that it is running and +updates when it is done. Switching deletion on or off, starting a check, +confirming a category (below) and every nightly check that deleted files are +recorded in the [Audit-Log](Audit-Log); the nightly entry names how many files +went per category. -A category is skipped, with the reason shown next to it, when not every part -of TREK could be asked about it, when its files share a location with another -category, when a listing or a lookup fails, or when more than a fifth of its -files look unused. The last one usually means two installations share one -bucket: each would see the other's files as unused, so leave deletion off in -that setup. Caches (place photos), backups, assistant attachments and journal -thumbnails are cleaned up by their own routines and are never touched here. -No check runs on a demo instance, and none while a sync, a move or a restore -is running. +When more than a fifth of a category's files look unused, the check marks it +and deletes nothing there until you confirm it once with **Confirm** next to +the category. The confirmation covers the number of files shown at that +moment: if the number later grows past it, while still more than a fifth, the +category waits for you again. A long-running instance with many old leftovers +usually needs this once. + +A category is skipped for that night, with the reason shown next to it, when: + +- not every part of TREK could be asked about it; +- its files share a location with another category's; +- another TREK installation keeps its files in the same place (see below), or + TREK could not leave its own mark there; +- listing it or asking about it failed. What the earlier nights found stays, + so one failure does not restart anyone's week; +- deleting failed on the storage backend: the check then stops deleting in + that category for the night. + +**Several installations on one bucket or folder.** Each check leaves a small +file named `trek-owner-` at the top of every category it looks at, naming +this installation. When it finds such a file of another installation there, +it leaves that category alone, and so does the other installation once it +sees this one's file. Backups leave these files out. If you moved an +installation and a stale `trek-owner-` of its old self remains, delete +that one file to let the check work there again. + +**Restoring a backup** switches deletion off and forgets what earlier checks +found and what you confirmed, because the restored data may come from another +installation. Switch deletion on again once you have looked at the next +report. + +Caches (place photos), backups, assistant attachments and journal thumbnails +are cleaned up by their own routines and are never touched here. No check runs +on a demo instance, none while a sync, a move or a restore is running, and a +running check deletes nothing while a backup is being written or once you +switch deletion off. ## Secrets and encryption