diff --git a/server/scripts/repository-owners.json b/server/scripts/repository-owners.json index dd1b82c8a..4dc4058ea 100644 --- a/server/scripts/repository-owners.json +++ b/server/scripts/repository-owners.json @@ -120,6 +120,7 @@ "Settings": "settings", "ShareTokens": "share", "StorageReclaimMarks": "storage", + "StorageReleaseQueue": "storage", "Tags": "tags", "TodoCategoryAssignees": "todo", "TodoItems": "todo", diff --git a/server/src/app-config/README.md b/server/src/app-config/README.md index 7f9e2209e..35facaed2 100644 --- a/server/src/app-config/README.md +++ b/server/src/app-config/README.md @@ -59,7 +59,8 @@ tokens, `RuntimeEnvService`) and consumes the SAME derive functions. `resolveDataPaths()` is the only place the server anchors `data/` and `uploads/` (on `SERVER_ROOT`, the package directory, never on `process.cwd()`) and names what lives in them: backups, the scratch dir, the logs, the JWT -secret and encryption key files, and the database file (TREK_DB_FILE when set, +secret and encryption key files, the install id file the storage reclaim +tells a copied database apart by, and the database file (TREK_DB_FILE when set, else `data/travel.db`). Never derive one of these paths from `__dirname` in a new file: a moved file would quietly move the data. diff --git a/server/src/app-config/data-paths.ts b/server/src/app-config/data-paths.ts index 6193cb868..5fd091336 100644 --- a/server/src/app-config/data-paths.ts +++ b/server/src/app-config/data-paths.ts @@ -29,6 +29,12 @@ export interface DataPaths { readonly jwtSecretFile: string; /** `data/.encryption_key`: the at-rest key when ENCRYPTION_KEY does not provide it. */ readonly encryptionKeyFile: string; + /** + * `data/.trek-install-id`: this installation's random id, written on first + * start. It never enters a backup and no restore writes it, so a database + * restored or copied from another install is told apart by it (storage reclaim). + */ + readonly installIdFile: string; /** * The SQLite file a non-test process opens: TREK_DB_FILE when set, else * `data/travel.db`. Under NODE_ENV=test the connection is `:memory:` instead @@ -58,6 +64,7 @@ export function resolveDataPaths( logsDir: path.join(dataDir, 'logs'), jwtSecretFile: path.join(dataDir, '.jwt_secret'), encryptionKeyFile: path.join(dataDir, '.encryption_key'), + installIdFile: path.join(dataDir, '.trek-install-id'), dbFile: env.db.trekDbFile || path.join(dataDir, 'travel.db'), }; } diff --git a/server/src/db/entities/StorageReleaseQueue.entity.ts b/server/src/db/entities/StorageReleaseQueue.entity.ts new file mode 100644 index 000000000..d8fd9ccf7 --- /dev/null +++ b/server/src/db/entities/StorageReleaseQueue.entity.ts @@ -0,0 +1,21 @@ +import { EntityRepositoryType, PrimaryKeyProp, defineEntity, p } from '@mikro-orm/core'; +import { StorageReleaseQueueRepository } from '../repositories/StorageReleaseQueue.repository'; +import { DbTimestampType } from '../types'; + +export class StorageReleaseQueue { + [EntityRepositoryType]?: StorageReleaseQueueRepository; + [PrimaryKeyProp]?: ['category', 'key']; + category!: string; + key!: string; + due_at!: string; +} + +export const StorageReleaseQueueSchema = defineEntity({ + class: StorageReleaseQueue, + repository: () => StorageReleaseQueueRepository, + properties: { + category: p.text().primary(), + key: p.text().primary(), + due_at: p.type(DbTimestampType), + }, +}); diff --git a/server/src/db/entities/index.ts b/server/src/db/entities/index.ts index 5d77e026c..4b5112f54 100644 --- a/server/src/db/entities/index.ts +++ b/server/src/db/entities/index.ts @@ -119,6 +119,7 @@ import { SchoolHolidayRegionsSchema } from './SchoolHolidayRegions.entity'; import { SettingsSchema } from './Settings.entity'; import { ShareTokensSchema } from './ShareTokens.entity'; import { StorageReclaimMarksSchema } from './StorageReclaimMarks.entity'; +import { StorageReleaseQueueSchema } from './StorageReleaseQueue.entity'; import { TagsSchema } from './Tags.entity'; import { TodoCategoryAssigneesSchema } from './TodoCategoryAssignees.entity'; import { TodoItemsSchema } from './TodoItems.entity'; @@ -287,6 +288,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [ SettingsSchema, ShareTokensSchema, StorageReclaimMarksSchema, + StorageReleaseQueueSchema, TagsSchema, TodoCategoryAssigneesSchema, TodoItemsSchema, diff --git a/server/src/db/kysely/StorageReleaseQueue.table.ts b/server/src/db/kysely/StorageReleaseQueue.table.ts new file mode 100644 index 000000000..e9858158d --- /dev/null +++ b/server/src/db/kysely/StorageReleaseQueue.table.ts @@ -0,0 +1,8 @@ +// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand: +// change the migration, then run `npm run gen:db-types`. + +export interface StorageReleaseQueueTable { + category: string; + key: string; + due_at: string; +} diff --git a/server/src/db/kysely/db.ts b/server/src/db/kysely/db.ts index 575fbc16e..9738a7588 100644 --- a/server/src/db/kysely/db.ts +++ b/server/src/db/kysely/db.ts @@ -125,6 +125,7 @@ import type { SchoolHolidayRegionsTable } from './SchoolHolidayRegions.table'; import type { SettingsTable } from './Settings.table'; import type { ShareTokensTable } from './ShareTokens.table'; import type { StorageReclaimMarksTable } from './StorageReclaimMarks.table'; +import type { StorageReleaseQueueTable } from './StorageReleaseQueue.table'; import type { TagsTable } from './Tags.table'; import type { TodoCategoryAssigneesTable } from './TodoCategoryAssignees.table'; import type { TodoItemsTable } from './TodoItems.table'; @@ -299,6 +300,7 @@ export interface DB { settings: SettingsTable; share_tokens: ShareTokensTable; storage_reclaim_marks: StorageReclaimMarksTable; + storage_release_queue: StorageReleaseQueueTable; tags: TagsTable; todo_category_assignees: TodoCategoryAssigneesTable; todo_items: TodoItemsTable; diff --git a/server/src/db/migrations/Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes.ts b/server/src/db/migrations/Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes.ts new file mode 100644 index 000000000..c51b3125c --- /dev/null +++ b/server/src/db/migrations/Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes.ts @@ -0,0 +1,24 @@ +import { Migration } from '@mikro-orm/migrations'; + +/** + * What a user's delete or replace left unnamed waits here before it goes: one + * row per stored object, due once every undo that can write the old value back + * has run out. A job then asks every reference source again and deletes only + * what none of them names; the row goes either way. A row is all an owner's + * request writes, so a delete stays as cheap as it was before the storage + * reclaim, whatever the request named. + */ +export class Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes extends Migration { + override name = 'Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes'; + + override up(): void { + this.addSql(` + CREATE TABLE IF NOT EXISTS storage_release_queue ( + category TEXT NOT NULL, + key TEXT NOT NULL, + due_at DATETIME NOT NULL, + PRIMARY KEY (category, key) + ) + `); + } +} diff --git a/server/src/db/repositories/StorageReleaseQueue.repository.ts b/server/src/db/repositories/StorageReleaseQueue.repository.ts new file mode 100644 index 000000000..253daf209 --- /dev/null +++ b/server/src/db/repositories/StorageReleaseQueue.repository.ts @@ -0,0 +1,86 @@ +import type { StorageReleaseQueue } from '../entities/StorageReleaseQueue.entity'; +import type { DB } from '../kysely/db'; +import { TrekRepository } from './_shared/trek-repository'; + +type QueueKyselyDB = Pick; + +/** One object a user's change left unnamed, and when it may go. */ +export interface ReleaseQueueRow { + category: string; + key: string; + due_at: string; +} + +/** + * `storage_release_queue`: the objects a delete or a replace left unnamed, + * keyed by category and category-relative key, each with the time it may go. + * Owners only add rows; the release job reads the due ones of the categories + * it may judge, asks every reference source again and drops each row once it + * has decided, deleted or not. A key queued again waits from the later time. + */ +export class StorageReleaseQueueRepository extends TrekRepository { + private db() { + return this.kysely(); + } + + /** Queues these keys, due at `dueAt`; a key queued already waits until the later time. */ + async enqueue(rows: Array<{ category: string; key: string }>, dueAt: string): Promise { + if (rows.length === 0) return; + await this.db() + .insertInto('storage_release_queue') + .values(rows.map((row) => ({ category: row.category, key: row.key, due_at: dueAt }))) + .onConflict((oc) => oc.columns(['category', 'key']).doUpdateSet({ due_at: dueAt })) + .execute(); + } + + /** How many rows wait, due or not. */ + async size(): Promise { + return await this.count(); + } + + /** The categories holding a row due at `now`. */ + async dueCategories(now: string): Promise { + const rows = await this.db() + .selectFrom('storage_release_queue') + .select('category') + .distinct() + .where('due_at', '<=', now) + .execute(); + return rows.map((row) => row.category); + } + + /** The rows of these categories due at `now`, the longest waiting first, at most `limit`. */ + async dueIn(categories: string[], now: string, limit: number): Promise { + if (categories.length === 0 || limit <= 0) return []; + return await this.db() + .selectFrom('storage_release_queue') + .select(['category', 'key', 'due_at']) + .where('category', 'in', categories) + .where('due_at', '<=', now) + .orderBy('due_at') + .limit(limit) + .execute(); + } + + /** Drops the rows of these keys; answers how many went. */ + async deleteKeys(category: string, keys: string[]): Promise { + if (keys.length === 0) return 0; + const result = await this.db() + .deleteFrom('storage_release_queue') + .where('category', '=', category) + .where('key', 'in', keys) + .executeTakeFirst(); + return Number(result.numDeletedRows); + } + + /** Drops the rows due before `cutoff`: what waited that long is the nightly pass's. */ + async deleteDueBefore(cutoff: string): Promise { + const result = await this.db().deleteFrom('storage_release_queue').where('due_at', '<', cutoff).executeTakeFirst(); + return Number(result.numDeletedRows); + } + + /** Drops every row, for a database another installation's history came with. */ + async deleteAll(): Promise { + return await this.nativeDelete({}); + } +} diff --git a/server/src/nest/app-config/data-paths.service.ts b/server/src/nest/app-config/data-paths.service.ts index 8783b8377..5aa1d3a7c 100644 --- a/server/src/nest/app-config/data-paths.service.ts +++ b/server/src/nest/app-config/data-paths.service.ts @@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common'; /** * The data layout for Nest classes: the data directory, uploads, backups, - * scratch space, logs, the key files and the database file. + * scratch space, logs, the key files, the install id and the database file. * * Providers that read a data path take this one (AdminService, * StorageRegistryService). Code that runs before the container exists or is @@ -23,6 +23,7 @@ export class DataPathsService implements DataPaths { readonly logsDir: string; readonly jwtSecretFile: string; readonly encryptionKeyFile: string; + readonly installIdFile: string; readonly dbFile: string; constructor() { @@ -34,6 +35,7 @@ export class DataPathsService implements DataPaths { this.logsDir = paths.logsDir; this.jwtSecretFile = paths.jwtSecretFile; this.encryptionKeyFile = paths.encryptionKeyFile; + this.installIdFile = paths.installIdFile; this.dbFile = paths.dbFile; } } diff --git a/server/src/nest/journey/journey-settings.service.ts b/server/src/nest/journey/journey-settings.service.ts index 34b09ee64..b18a07722 100644 --- a/server/src/nest/journey/journey-settings.service.ts +++ b/server/src/nest/journey/journey-settings.service.ts @@ -54,7 +54,10 @@ export class JourneySettingsService { return previousSettings(before as unknown as Readonly> | undefined, patch); }); if (written) await this.journeys.broadcastJourneyEvent(journeyId, 'journey:updated', {}, sid); - if (typeof previous.cover_image === 'string') await this.release.releaseValues([previous.cover_image]); + // A replaced cover is queued, never deleted here: the assistant's undo can write it back. + if (typeof previous.cover_image === 'string' && previous.cover_image !== patch.cover_image) { + await this.release.releaseValues([previous.cover_image]); + } const journey = (await this.journeysRepo.findById(journeyId)) as Journey | undefined; return journey ? { journey, previous } : null; } diff --git a/server/src/nest/journey/journey.controller.ts b/server/src/nest/journey/journey.controller.ts index aa12eddff..0d137747b 100644 --- a/server/src/nest/journey/journey.controller.ts +++ b/server/src/nest/journey/journey.controller.ts @@ -26,7 +26,6 @@ import { RuntimeEnvService } from '../app-config/runtime-env.service'; import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write'; import { StorageService } from '../storage/storage.service'; import { StorageReleaseService } from '../storage/storage-release.service'; -import { journeyThumbName } from '../memories/thumbnail.service'; import { JourneyService } from './journey.service'; import { JourneyBookService } from './journey-book.service'; import { JourneyPhotoCaptureService, type ProviderPhotoBatch, type ProviderPhotoTarget } from './journey-photo-capture.service'; @@ -472,16 +471,13 @@ export class JourneyController { } /** - * After a photo's row went: its file and its poster go at once unless another - * row (another journey's photo, a journey cover) still names them, and the - * thumbnail derived from a file that went goes with it (#2924). + * After a photo's row went: its file and its poster are queued for release, + * and go once no other row (another journey's photo, a journey cover) names + * them (#2924). The thumbnail derived from the file is the nightly thumbnail + * sweep's, which drops every thumb no photo row accounts for. */ private async releasePhoto(photo: { file_path?: string | null; thumbnail_path?: string | null }): Promise { - const released = await this.release.releaseValues([photo.file_path, photo.thumbnail_path], { bare: 'journey' }); - const filePath = photo.file_path; - if (!filePath?.startsWith('journey/')) return; - if (!released.some((ref) => ref.category === 'journey' && `journey/${ref.key}` === filePath)) return; - await this.storage.delete('journey', journeyThumbName(filePath)).catch(() => { /* already gone */ }); + await this.release.releaseValues([photo.file_path, photo.thumbnail_path], { bare: 'journey' }); } // ── Gallery (prefix /:id/gallery — before /:id) ───────────────────────── diff --git a/server/src/nest/storage/storage-install.service.ts b/server/src/nest/storage/storage-install.service.ts new file mode 100644 index 000000000..80cd4a558 --- /dev/null +++ b/server/src/nest/storage/storage-install.service.ts @@ -0,0 +1,192 @@ +import { AppSettings } from '../../db/entities/AppSettings.entity'; +import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity'; +import { StorageReleaseQueue } from '../../db/entities/StorageReleaseQueue.entity'; +import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; +import type { StorageReclaimMarksRepository } from '../../db/repositories/StorageReclaimMarks.repository'; +import type { StorageReleaseQueueRepository } from '../../db/repositories/StorageReleaseQueue.repository'; +import { DataPathsService } from '../app-config/data-paths.service'; +import { logError, logInfo, logWarn } from '../audit/audit-log.logger'; +import { withRequestContext } from '../database/request-context'; +import { UnitOfWork } from '../database/unit-of-work'; +import { INSTALL_ID_KEY, RECLAIM_DELETE_KEY, RECLAIM_RESET_KEYS } from './storage-reclaim.settings'; +import { MikroORM } from '@mikro-orm/core'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; + +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +const INSTALL_ID = /^[a-z0-9-]{8,64}$/; + +/** + * Which installation this is, from a file in the data dir + * (`data/.trek-install-id`, made on the first start with a random id, mode + * 0600 like the at-rest key). No backup archive carries it and no restore + * writes it, so it stays with the data dir while a database can come from + * anywhere: a backup restore (in the app or at boot), a restart after a + * restore that failed half way, a copied `travel.db`. + * + * At boot, and after every reopen of the database, the database is reconciled + * with it: a database that names another id, or none, gets this one, and the + * reclaim state it brought (the deletion switch, the confirmations, both + * reports, the marks and the release queue) is dropped, since it describes + * another installation's storage. After a reopen that happens whatever the id + * says, as before: the rows a restore brings are an older history even when + * they are this installation's own. + * + * What it cannot tell apart is a data dir copied whole, the file with it: the + * wiki asks to give a clone its own data dir or delete the file. When the file + * can be neither read nor written, there is no id: nothing is deleted then. + */ +@Injectable() +export class StorageInstallService implements OnApplicationBootstrap { + /** Undefined until the file was read; null when it cannot be used. */ + private fileId: string | null | undefined; + /** Bumped on every reopen: a pass or a release tick that saw another epoch stops. */ + private currentEpoch = 0; + + constructor( + private readonly dataPaths: DataPathsService, + private readonly uow: UnitOfWork, + private readonly orm: MikroORM, + @InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository, + @InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository, + @InjectRepository(StorageReleaseQueue) private readonly queue: StorageReleaseQueueRepository, + ) {} + + async onApplicationBootstrap(): Promise { + try { + await withRequestContext(this.orm, () => this.reconcile()); + } catch (err) { + logError(`[storage] could not reconcile the database with this installation's id: ${messageOf(err)}`); + } + } + + /** How many reopens this process has seen. */ + get epoch(): number { + return this.currentEpoch; + } + + /** + * This installation's id, read from its file once and made there on the + * first start; null when the file holds no id or cannot be read or written, + * which keeps every delete off (fail closed). + */ + installId(): string | null { + if (this.fileId === undefined) this.fileId = this.readOrCreate(); + return this.fileId; + } + + /** + * Makes the database name this installation. One that names another, or + * none, loses its reclaim state first. Answers whether it reset something + * a database had brought along. + */ + async reconcile(): Promise { + const id = this.installId(); + if (id === null) return false; + const stored = await this.appSettings.getValue(INSTALL_ID_KEY); + if (stored === id) return false; + const wasOn = await this.reset(id); + if (stored !== null) { + logInfo( + `[storage] the database names another installation (restored, cloned or copied): its reclaim state was ` + + `dropped${wasOn ? ', and deletion is off until an admin turns it on' : ''}`, + ); + } + return stored !== null; + } + + /** + * The database was swapped (a restore, the demo reset): what ran on the old + * one stops, and the new one gets this installation's id and no reclaim + * state. Contained, since a listener that throws fails the reopen. + */ + async afterReopen(): Promise { + this.currentEpoch += 1; + try { + const id = this.installId(); + const wasOn = await withRequestContext(this.orm, () => this.reset(id)); + if (wasOn) + logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on'); + } catch (err) { + logError(`[storage] could not reset the reclaim state after the database was reopened: ${messageOf(err)}`); + } + } + + /** Drops the reclaim state and writes `id` (when there is one); answers whether deletion was on. */ + private async reset(id: string | null): Promise { + return await this.uow.transactional(async () => { + const wasOn = (await this.appSettings.getValue(RECLAIM_DELETE_KEY)) !== null; + if (id !== null) await this.appSettings.upsertOrReplace(INSTALL_ID_KEY, id); + await RECLAIM_RESET_KEYS.reduce>(async (previous, key) => { + await previous; + await this.appSettings.deleteValue(key); + }, Promise.resolve()); + await this.marks.deleteAll(); + await this.queue.deleteAll(); + return wasOn; + }); + } + + private readOrCreate(): string | null { + const file = this.dataPaths.installIdFile; + const read = readId(file); + if (read !== 'missing') return read; + const id = randomUUID(); + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + placeOnce(file, `${id}\n`); + return id; + } catch (err) { + if (codeOf(err) === 'EEXIST') { + const again = readId(file); + return again === 'missing' ? null : again; + } + logWarn(`[storage] could not write the install id file, nothing is reclaimed: ${messageOf(err)}`); + return null; + } + } +} + +/** + * Puts `text` at `file` unless a file is there already (EEXIST), whole: it is + * written next to it and linked into place, so two processes starting at once + * on one data dir end up with one id and neither reads a half-written file. A + * volume without hard links gets an exclusive create instead. + */ +function placeOnce(file: string, text: string): void { + const spool = `${file}.${process.pid}.${randomUUID()}.tmp`; + fs.writeFileSync(spool, text, { mode: 0o600 }); + try { + fs.linkSync(spool, file); + } catch (err) { + if (codeOf(err) === 'EEXIST') throw err; + fs.writeFileSync(file, text, { mode: 0o600, flag: 'wx' }); + } finally { + fs.rmSync(spool, { force: true }); + } +} + +/** The id the file holds, 'missing' when there is no file, null when it is unreadable or holds no id. */ +function readId(file: string): string | null | 'missing' { + try { + const text = fs.readFileSync(file, 'utf8').trim(); + if (INSTALL_ID.test(text)) return text; + logWarn(`[storage] '${file}' holds no install id, nothing is reclaimed until it is deleted or fixed`); + return null; + } catch (err) { + if (codeOf(err) === 'ENOENT') return 'missing'; + logWarn(`[storage] could not read the install id file, nothing is reclaimed: ${messageOf(err)}`); + return null; + } +} + +function codeOf(err: unknown): string | undefined { + return typeof err === 'object' && err !== null && 'code' in err ? String((err as { code: unknown }).code) : undefined; +} + +function messageOf(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/server/src/nest/storage/storage-reclaim-guard.service.ts b/server/src/nest/storage/storage-reclaim-guard.service.ts new file mode 100644 index 000000000..d2abe5a83 --- /dev/null +++ b/server/src/nest/storage/storage-reclaim-guard.service.ts @@ -0,0 +1,166 @@ +import { logWarn } from '../audit/audit-log.logger'; +import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; +import { RECLAIM_LEDGER, sharedLocationsOf } from './storage-reclaim.ledger'; +import { storageOwnerMarkerInstall, storageOwnerMarkerKey, type StorageReferenceSource } from './storage-reclaim.types'; +import { StorageRegistryService } from './storage-registry.service'; +import { StorageService } from './storage.service'; +import type { ServedCategory } from './storage.types'; +import { Injectable } from '@nestjs/common'; + +import { Readable } from 'node:stream'; + +/** Why a category may not be judged, before anything of it is listed. */ +type CategoryRefusal = 'not_reclaimed' | 'sources_missing' | 'shared_location'; +/** Why the place a category lives in may not be judged: another installation's, or one nobody can tell. */ +type OwnerRefusal = 'foreign_install' | 'owner_unknown'; + +/** A verdict the release queue reuses for this long; the nightly pass renews it every night. */ +const OWNER_VERDICT_TTL_MS = 26 * 60 * 60 * 1000; +/** A failed look at the owner markers is not retried before this. */ +const OWNER_FAILURE_TTL_MS = 60 * 60 * 1000; + +interface CachedVerdict { + verdict: OwnerRefusal | null; + /** Where the category lived when it was judged: a changed storage config judges again. */ + location: string; + until: number; +} + +/** + * The guards the reclaim pass and the release queue share, so a deferred + * release refuses exactly what the nightly pass refuses: + * - `judgeable`: the ledger reclaims the category, every source it requires + * answers for all of it, and its location overlaps no other category's; + * - `ownerVerdict`: no other installation's owner marker lies where the + * category's objects do, and this one's could be written there. + * + * The owner verdict is cached per category with the location it was judged + * at. The pass judges it from the listing it makes anyway; the release queue + * reuses that verdict for a day and lists the category's root itself only + * when there is none, so a release never deletes in a category the pass + * would refuse as another installation's. + */ +@Injectable() +export class StorageReclaimGuardService { + private readonly verdicts = new Map(); + + constructor( + private readonly storage: StorageService, + private readonly registry: StorageRegistryService, + private readonly references: StorageReclaimRegistryService, + ) {} + + /** + * The sources to ask about a category, or why it may not be judged. Pass + * `sources` and `shared` to judge several categories against one snapshot. + */ + judgeable( + category: ServedCategory, + sources: readonly StorageReferenceSource[] = this.references.sources(), + shared: ReadonlySet = sharedLocationsOf(this.registry), + ): { refused: CategoryRefusal } | { sources: StorageReferenceSource[] } { + const rule = RECLAIM_LEDGER[category]; + if (!rule.reclaim) return { refused: 'not_reclaimed' }; + const claiming = sources.filter((source) => source.claims.some((claim) => claim.category === category)); + const whole = new Set( + claiming + .filter((source) => source.claims.some((claim) => claim.category === category && claim.prefix === '')) + .map((source) => source.id), + ); + if (rule.requiredSources.some((id) => !whole.has(id))) return { refused: 'sources_missing' }; + if (shared.has(category)) return { refused: 'shared_location' }; + return { sources: claiming }; + } + + /** + * Whether this installation may judge where a category lives, from the + * owner markers a listing of its root found: not where another one's marker + * lies, and not where its own could not be written, since then a second + * installation cannot see it either. Writes this one's marker when it is + * missing, and caches the verdict. + */ + async ownerVerdict( + category: ServedCategory, + owners: ReadonlySet, + installId: string, + now: number = Date.now(), + ): Promise { + const location = this.locationOf(category); + const verdict = await this.judgeOwners(category, owners, installId); + this.verdicts.set(category, { verdict, location, until: now + OWNER_VERDICT_TTL_MS }); + return verdict; + } + + /** + * The owner verdict for a category, as the last judgement left it when that + * holds for where the category lives now; otherwise its root is listed for + * owner markers and judged afresh. A listing that fails reads as + * `owner_unknown` for an hour. + */ + async cachedOwnerVerdict( + category: ServedCategory, + installId: string, + now: number = Date.now(), + ): Promise { + const cached = this.verdicts.get(category); + if (cached && cached.until > now && cached.location === this.locationOf(category)) return cached.verdict; + let owners: Set; + try { + owners = await this.ownersAt(category); + } catch (err) { + logWarn(`[storage] could not look for owner markers in '${category}': ${messageOf(err)}`); + this.verdicts.set(category, { + verdict: 'owner_unknown', + location: this.locationOf(category), + until: now + OWNER_FAILURE_TTL_MS, + }); + return 'owner_unknown'; + } + return await this.ownerVerdict(category, owners, installId, now); + } + + /** Forgets every verdict: the database or the installation they were judged for changed. */ + forget(): void { + this.verdicts.clear(); + } + + private async judgeOwners( + category: ServedCategory, + owners: ReadonlySet, + installId: string, + ): Promise { + const foreign = [...owners].some((owner) => owner !== installId); + if (!owners.has(installId)) { + try { + await this.storage.put(category, storageOwnerMarkerKey(installId), Readable.from([installId]), { + contentType: 'text/plain', + }); + } catch (err) { + logWarn(`[storage] reclaim could not write its owner marker into '${category}': ${messageOf(err)}`); + return foreign ? 'foreign_install' : 'owner_unknown'; + } + } + if (foreign) logWarn(`[storage] another TREK installation keeps its uploads where '${category}' does: not judged`); + return foreign ? 'foreign_install' : null; + } + + /** The owner markers at a category's root. */ + private async ownersAt(category: ServedCategory): Promise> { + const owners = new Set(); + for await (const stat of this.storage.list(category, '', { shallow: true })) { + const owner = storageOwnerMarkerInstall(stat.key); + if (owner !== null) owners.add(owner); + } + return owners; + } + + /** Where a category lives now: its backend, prefix and the backends' settings. */ + private locationOf(category: ServedCategory): string { + const { backendName, keyPrefix } = this.registry.resolve(category); + return JSON.stringify([backendName, keyPrefix, this.registry.snapshot().backends]); + } +} + +function messageOf(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/server/src/nest/storage/storage-reclaim-source.service.ts b/server/src/nest/storage/storage-reclaim-source.service.ts index a80976f75..6e0855037 100644 --- a/server/src/nest/storage/storage-reclaim-source.service.ts +++ b/server/src/nest/storage/storage-reclaim-source.service.ts @@ -1,7 +1,9 @@ +import { logWarn } from '../audit/audit-log.logger'; import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; import { keySearchCore, keysNamedIn, + storageKeysIn, type StorageKeyOptions, type StorageReferenceClaim, type StorageReferenceSource, @@ -22,11 +24,18 @@ import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; * learns which tables those are. A column that takes any string (an absolute * URL of this instance, a path with a query, another category's object) * cannot be matched with `IN`, so the values are read and parsed here: - * - `namedKeys` reads every value once, for a pass that judges a whole category; + * - `namedKeysByCategory` reads every value once and parses it for every + * category asked, for a pass that judges them all; + * - `namedKeys` reads every value once for one category, fresh, for the + * question a pass asks right before it deletes; * - `referenced` reads only the rows whose value holds one of the keys' plain - * runs (`keySearchCore`, a `LIKE '%run%'` each), fresh, for the question a - * delete asks last and for a release right after a user's change. A key with - * no such run reads everything, as `namedKeys` does. + * runs (`keySearchCore`, a `LIKE '%run%'` each, at most MAX_LIKE_TERMS per + * query, as many queries as the keys need), for the release queue. A key + * with no such run is answered from one whole read; the release queue never + * asks about such a key, it leaves it to the nightly pass. + * + * The rows are read through the owner's TrekRepository, whose reads skip the + * identity map, so a pass over a large table keeps no entities alive. */ @Injectable() export abstract class StorageReclaimSource implements StorageReferenceSource, OnApplicationBootstrap { @@ -49,11 +58,32 @@ export abstract class StorageReclaimSource implements StorageReferenceSource, On return keysNamedIn(category, await this.storedValues(null), this.keyOptions); } + async namedKeysByCategory(categories: readonly ServedCategory[]): Promise>> { + const asked = new Set(categories.filter((category) => this.claimsCategory(category))); + const named = new Map>([...asked].map((category) => [category, new Set()])); + if (asked.size === 0) return named; + for (const value of await this.storedValues(null)) { + for (const ref of storageKeysIn(value, this.keyOptions)) { + named.get(ref.category as ServedCategory)?.add(ref.key.toLowerCase()); + } + } + return named; + } + async referenced(category: ServedCategory, keys: readonly string[]): Promise> { if (keys.length === 0 || !this.claimsCategory(category)) return new Set(); const cores = keys.map(keySearchCore); - const narrow = cores.every((core): core is string => core !== null) ? [...new Set(cores)] : null; - const named = keysNamedIn(category, await this.storedValues(narrow), this.keyOptions); + const named = new Set(); + if (cores.some((core) => core === null)) { + for (const key of keysNamedIn(category, await this.storedValues(null), this.keyOptions)) named.add(key); + } else { + const runs = [...new Set(cores.filter((core): core is string => core !== null))]; + await chunked(runs, RUNS_PER_QUERY).reduce>(async (previous, page) => { + await previous; + const values = await this.storedValues(page); + for (const key of keysNamedIn(category, values, this.keyOptions)) named.add(key); + }, Promise.resolve()); + } return new Set(keys.filter((key) => named.has(key.toLowerCase()))); } @@ -70,11 +100,21 @@ export abstract class StorageReclaimSource implements StorageReferenceSource, On } } +/** + * The most LIKE terms one narrowed read may hold. SQLite parses an OR chain + * as a left-deep tree and refuses one deeper than 1000, so a read is kept + * far below that whatever it is asked about. + */ +export const MAX_LIKE_TERMS = 50; +/** Every source reads at most two columns, so this many runs keep one read within MAX_LIKE_TERMS. */ +const RUNS_PER_QUERY = 25; + /** * The filter a source narrows its columns with: rows where one of `fields` * holds one of the runs, or every row where one of them is set when there are * none to narrow by. The runs are `[a-z0-9-]` only, so they need no LIKE - * escaping; SQLite's LIKE ignores ASCII case, as the parser does. + * escaping; SQLite's LIKE ignores ASCII case, as the parser does. A filter + * past MAX_LIKE_TERMS is refused rather than handed to SQLite. */ export function likeAny( fields: ReadonlyArray, @@ -84,5 +124,13 @@ export function likeAny( containing === null ? fields.map((field) => ({ [field]: { $ne: null } })) : fields.flatMap((field) => containing.map((run) => ({ [field]: { $like: `%${run}%` } }))); + if (terms.length > MAX_LIKE_TERMS) { + logWarn(`[storage] a reference lookup asked for ${terms.length} LIKE terms, more than ${MAX_LIKE_TERMS}`); + throw new Error(`a reference lookup may hold at most ${MAX_LIKE_TERMS} LIKE terms`); + } return { $or: terms } as unknown as FilterQuery; } + +function chunked(items: readonly T[], size: number): T[][] { + return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size)); +} diff --git a/server/src/nest/storage/storage-reclaim.controller.ts b/server/src/nest/storage/storage-reclaim.controller.ts index 6a92d4e6f..230d8cb9a 100644 --- a/server/src/nest/storage/storage-reclaim.controller.ts +++ b/server/src/nest/storage/storage-reclaim.controller.ts @@ -49,13 +49,14 @@ export class StorageReclaimController { /** * Starts a report-only pass in the background and answers 202 at once; the * admin state says `running` until its report lands. 409 on a demo or while - * another storage job, a reclaim pass included, runs. + * another storage job, a reclaim pass included, runs (a release tick, which + * takes seconds, is waited for). */ @Post('run') @HttpCode(202) @ResponseContract(storageReclaimRunStartedSchema) async run(@CurrentUser() user: User, @Req() req: Request): Promise<{ started: true }> { - this.reclaim.start(); + await this.reclaim.start(); await this.audit.writeAudit({ userId: user.id, action: 'admin.storage_reclaim_run', @@ -67,7 +68,9 @@ export class StorageReclaimController { /** * Lets the pass delete in a category whose unreferenced share is over a - * fifth, up to the count the last report showed. 409 when nothing waits. + * fifth, for the count the admin was shown (`unreferenced`). 409 + * `reclaim_nothing_to_confirm` when nothing waits, `reclaim_confirm_stale` + * when the last report's count is no longer the one shown. */ @Post('confirm') @HttpCode(200) @@ -77,15 +80,12 @@ export class StorageReclaimController { @Body() body: StorageReclaimConfirmDto, @Req() req: Request, ): Promise { - const state = await this.reclaim.confirm(body.category); + const state = await this.reclaim.confirm(body.category, body.unreferenced); await this.audit.writeAudit({ userId: user.id, action: 'admin.storage_reclaim_confirm', ip: getClientIp(req), - details: { - category: body.category, - unreferenced: state.lastRun?.categories[body.category].unreferenced.objects ?? 0, - }, + details: { category: body.category, unreferenced: body.unreferenced }, }); return state; } diff --git a/server/src/nest/storage/storage-reclaim.report.ts b/server/src/nest/storage/storage-reclaim.report.ts index e4f72f2d3..252cbbc1c 100644 --- a/server/src/nest/storage/storage-reclaim.report.ts +++ b/server/src/nest/storage/storage-reclaim.report.ts @@ -1,3 +1,5 @@ +import { confirmationCovers, type ReclaimConfirmations } from './storage-reclaim.settings'; +import type { ServedCategory } from './storage.types'; import type { StorageReclaimCategoryReport, StorageReclaimReport } from '@trek/shared'; /** One object a pass looks at. */ @@ -38,6 +40,28 @@ function countOf(candidates: readonly ReclaimCandidate[]): { objects: number; by return { objects: candidates.length, bytes: candidates.reduce((sum, candidate) => sum + candidate.size, 0) }; } +/** + * The report as the admin state shows it: a category whose live confirmation + * covers its unreferenced count no longer reads as waiting for one, so the + * prompt goes as soon as the admin confirmed, not only after the next pass. + */ +export function withConfirmations( + report: StorageReclaimReport, + confirmations: ReclaimConfirmations, + now: number, +): StorageReclaimReport { + const categories = Object.fromEntries( + Object.entries(report.categories).map(([category, row]) => [ + category, + row.confirmationRequired && + confirmationCovers(confirmations[category as ServedCategory], row.unreferenced.objects, now) + ? { ...row, confirmationRequired: false } + : row, + ]), + ) as StorageReclaimReport['categories']; + return { ...report, categories }; +} + /** What a report says the pass deleted, over every category. */ export function deletedTotal(report: StorageReclaimReport): { objects: number; bytes: number } { const rows = Object.values(report.categories); diff --git a/server/src/nest/storage/storage-reclaim.service.ts b/server/src/nest/storage/storage-reclaim.service.ts index 67c163b97..ac3796df6 100644 --- a/server/src/nest/storage/storage-reclaim.service.ts +++ b/server/src/nest/storage/storage-reclaim.service.ts @@ -7,72 +7,82 @@ import type { } from '../../db/repositories/StorageReclaimMarks.repository'; import { dbNow, parseDbTimestamp } from '../../db/types'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; -import { logError, logInfo, logWarn } from '../audit/audit-log.logger'; +import { logError, logInfo } from '../audit/audit-log.logger'; import { AuditService } from '../audit/audit.service'; import { DomainError } from '../common/domain-error'; import { DatabaseLifecycle } from '../database/database-lifecycle.service'; import { withRequestContext } from '../database/request-context'; -import { UnitOfWork } from '../database/unit-of-work'; +import { StorageInstallService } from './storage-install.service'; import { StorageJobsService } from './storage-jobs.service'; +import { StorageReclaimGuardService } from './storage-reclaim-guard.service'; import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; -import { inReclaimScope, RECLAIM_LEDGER, sharedLocationsOf } from './storage-reclaim.ledger'; +import { inReclaimScope, RECLAIM_LEDGER, reclaimableCategories, sharedLocationsOf } from './storage-reclaim.ledger'; import { deletedTotal, newProgress, reportOf, + withConfirmations, type CategoryProgress, type ReclaimCandidate, } from './storage-reclaim.report'; import { - INSTALL_ID_KEY, + confirmationCovers, + confirmationExpired, + confirmationStale, + overConfirmShare, RECLAIM_DELETE_KEY, RECLAIM_DELETE_REPORT_KEY, RECLAIM_REPORT_KEY, - RECLAIM_RESET_KEYS, readReclaimConfirmations, readReclaimDeleteEnabled, readReclaimReport, writeReclaimConfirmations, + type ReclaimConfirmations, } from './storage-reclaim.settings'; -import { storageOwnerMarkerInstall, storageOwnerMarkerKey, type StorageReferenceSource } from './storage-reclaim.types'; +import { storageOwnerMarkerInstall, type StorageReferenceSource } from './storage-reclaim.types'; import { StorageRegistryService } from './storage-registry.service'; +import { STORAGE_RELEASE_JOB } from './storage-release.service'; import { StorageService } from './storage.service'; import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; import { MikroORM } from '@mikro-orm/core'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { Injectable, type OnApplicationBootstrap, type OnModuleInit } from '@nestjs/common'; +import { Injectable, type OnModuleInit } from '@nestjs/common'; import type { StorageReclaimCategoryReport, StorageReclaimReport, StorageReclaimState } from '@trek/shared'; -import { randomUUID } from 'node:crypto'; -import { Readable } from 'node:stream'; - const HOUR_MS = 60 * 60 * 1000; /** An object younger than this is never judged: every writer puts the bytes before it inserts the row. */ const RECLAIM_GRACE_MS = 24 * HOUR_MS; /** How long an object must stay unreferenced, on every pass, before a pass may delete it. */ const RECLAIM_DELAY_MS = 7 * 24 * HOUR_MS; -/** More unreferenced than this share of a category waits for an admin's confirmation before anything goes. */ -const CONFIRM_SHARE = 0.2; -/** Keys per re-check right before a delete: one fresh, narrowed question to every source. */ -const RECHECK_BATCH = 100; -/** Deletes in flight at once. */ +/** Eligible objects per fresh read of every source right before their deletes. */ +const FRESH_BATCH = 1000; +/** Stats and deletes in flight at once. */ const DELETE_CONCURRENCY = 20; /** Marks per statement: four bound values each, well under SQLite's limit. */ const MARK_BATCH = 100; /** Listed objects between two looks at whether a restore began, so a restore waits seconds, not a whole listing. */ const LIST_CHECK_EVERY = 500; const JOB_NAME = 'a storage reclaim'; -const INSTALL_ID = /^[a-z0-9-]{8,64}$/; + +/** What each source names, per category, from the one read a pass makes of it; an Error when that read failed. */ +type NamedCache = Map> | Error>; interface PassContext { now: number; - installId: string; + /** Null when this installation has no id: nothing is judged then. */ + installId: string | null; /** Deletes may go now; turned off for the rest of the pass once the switch reads off or a backup starts. */ deleting: boolean; + /** A backup turned deleting off: every category with something due says so. */ + backupHeld: boolean; sources: readonly StorageReferenceSource[]; shared: ReadonlySet; - confirmed: Partial>; - generation: number; + confirmations: ReclaimConfirmations; + /** Confirmations this pass found used up or run out, by the time they were given. */ + spent: Map; + /** Read once, at the first category judged; each category's sets are dropped once it is judged. */ + named: NamedCache | null; + epoch: number; } interface ReclaimRunOptions { @@ -89,37 +99,37 @@ class ReclaimStopped extends Error {} * more, found in two phases and, only when the admin switched deletion on, * deleted. A pass lists each category the ledger allows at its root, skips * objects younger than a day (the bytes of an upload land before its row), - * reads once per category what every registered reference source names, and - * marks the rest with the time it first saw them. An object goes only once its - * mark is a week old, after one more fresh question to every source about its - * batch, through `StorageService.delete` (so replicas follow and a demo keeps - * its bytes). What a user deletes or replaces is released at once instead + * reads every registered reference source once for the whole pass, and marks + * the rest with the time it first saw them. An object goes only once its mark + * is a week old, after a fresh read of every source right before its batch, + * through `StorageService.delete` (so replicas follow and a demo keeps its + * bytes). What a user deletes or replaces is queued for release instead * (StorageReleaseService); the pass is the net for whatever that left behind. * * Safety, each failing closed: * - a category whose sources are not all registered, whose location overlaps * another category's, or where another installation's owner marker lies - * (or this one's cannot be written) is not judged; + * (or this one's cannot be written) is not judged (StorageReclaimGuardService); * - a failed listing or lookup skips the category for the night; no skip * touches the marks, so one bad night does not restart the week; * - more than a fifth unreferenced deletes nothing until an admin confirmed - * that count (`confirm`), which no user action can do; + * the count they were shown (`confirm`), which no user action can do; the + * confirmation covers half as much again for two weeks, and is used up once + * the share is back under a fifth; * - a delete that fails on the backend stops deleting in that category; - * - nothing is deleted while a backup reads the uploads, once the switch is - * turned off mid-pass, or on a demo instance; - * - a pass never runs beside a backfill, a category migration or a restore - * (the one-job slot of StorageJobsService, which a restore waits on), and - * stops when a restore begins or the database is reopened under it. + * - nothing is deleted while a backup reads the uploads (`backup_running`), + * once the switch is turned off mid-pass, or on a demo instance; + * - a pass never runs beside a backfill, a category migration, a release tick + * or a restore (the one-job slot of StorageJobsService, which a restore + * waits on), and stops when a restore begins or the database is reopened. * - * Every restore turns deletion off and drops the marks, the confirmations and - * the reports: the restored rows may come from another installation, and this - * one keeps its own install id (written back over the restored one). + * Which installation this is comes from a file in the data dir, not from the + * database (StorageInstallService): a database restored, cloned or copied from + * another one loses its switch, marks, confirmations and reports at boot, and + * every reopen does the same. */ @Injectable() -export class StorageReclaimService implements OnModuleInit, OnApplicationBootstrap { - private generation = 0; - private knownInstallId: string | null = null; - +export class StorageReclaimService implements OnModuleInit { constructor( private readonly storage: StorageService, private readonly registry: StorageRegistryService, @@ -128,64 +138,74 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr private readonly lifecycle: DatabaseLifecycle, private readonly runtimeEnv: RuntimeEnvService, private readonly audit: AuditService, - private readonly uow: UnitOfWork, private readonly orm: MikroORM, + private readonly install: StorageInstallService, + private readonly guard: StorageReclaimGuardService, @InjectRepository(StorageReclaimMarks) private readonly marks: StorageReclaimMarksRepository, @InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository, ) {} onModuleInit(): void { - this.lifecycle.onReopened(() => this.afterReopen()); + this.lifecycle.onReopened(async () => { + this.guard.forget(); + await this.install.afterReopen(); + }); } - /** Makes the install id on the first boot, so a restore later has one of this install's own to keep. */ - async onApplicationBootstrap(): Promise { - try { - await withRequestContext(this.orm, () => this.installId()); - } catch (err) { - logWarn(`[storage] could not read or make the install id: ${messageOf(err)}`); - } - } - - /** Read only, so the admin state's GET never writes: an install id not made yet reads as deletion off. */ + /** + * Read only, so the admin state's GET never writes (the install id file is + * made at boot). A category whose live confirmation covers its count reads + * as no longer waiting for one, before the next pass says so too. + */ async state(): Promise { - const installId = this.knownInstallId ?? (await this.appSettings.getValue(INSTALL_ID_KEY)); + const installId = this.install.installId(); + const lastRun = await readReclaimReport(this.appSettings, RECLAIM_REPORT_KEY); + const confirmations = installId === null ? {} : await readReclaimConfirmations(this.appSettings, installId); return { deleteEnabled: installId !== null && (await readReclaimDeleteEnabled(this.appSettings, installId)), running: this.jobs.exclusiveHolder === JOB_NAME, - lastRun: await readReclaimReport(this.appSettings, RECLAIM_REPORT_KEY), + lastRun: lastRun === null ? null : withConfirmations(lastRun, confirmations, Date.now()), lastDeleteRun: await readReclaimReport(this.appSettings, RECLAIM_DELETE_REPORT_KEY), }; } + /** 409 `reclaim_install_unknown` when this installation has no id to bind the switch to. */ async setDeleteEnabled(enabled: boolean): Promise { - if (enabled) await this.appSettings.upsertOrReplace(RECLAIM_DELETE_KEY, await this.installId()); + if (enabled) await this.appSettings.upsertOrReplace(RECLAIM_DELETE_KEY, this.requireInstallId()); else await this.appSettings.deleteValue(RECLAIM_DELETE_KEY); return await this.state(); } /** * The admin lets the pass delete in a category whose unreferenced share is - * over a fifth, up to the count the last report showed. 409 when the last - * report holds nothing that waits for it. + * over a fifth, for the count they were shown (`shown`). 409 + * `reclaim_nothing_to_confirm` when the last report holds nothing waiting + * there, `reclaim_confirm_stale` when its count is no longer the one shown. */ - async confirm(category: ServedCategory): Promise { + async confirm(category: ServedCategory, shown: number): Promise { + const installId = this.requireInstallId(); const row = (await readReclaimReport(this.appSettings, RECLAIM_REPORT_KEY))?.categories[category]; if (!row?.confirmationRequired) { throw refusal('reclaim_nothing_to_confirm', 'nothing in this category waits for a confirmation'); } - const installId = await this.installId(); - const counts = await readReclaimConfirmations(this.appSettings, installId); - await writeReclaimConfirmations(this.appSettings, installId, { ...counts, [category]: row.unreferenced.objects }); + if (confirmationStale(shown, row.unreferenced.objects)) { + throw refusal('reclaim_confirm_stale', 'the last check found another count since, reload and confirm again'); + } + const confirmations = await readReclaimConfirmations(this.appSettings, installId); + await writeReclaimConfirmations(this.appSettings, installId, { + ...confirmations, + [category]: { count: shown, confirmedAt: Date.now() }, + }); return await this.state(); } /** * One pass over every category in the caller, its report persisted and * returned: the nightly job's. Refuses with a 409 on a demo instance and - * while another storage job holds the slot. + * while another storage job holds the slot; a release tick is waited for. */ async run(options: ReclaimRunOptions = {}): Promise { + await this.releaseTickDone(); this.claimSlot(); try { return await this.pass(options.reportOnly === true, options.now ?? new Date()); @@ -199,7 +219,8 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr * longer than a proxy waits for an answer. The same refusals as `run`, at * once; the report lands on the admin state, which says `running` meanwhile. */ - start(): void { + async start(): Promise { + await this.releaseTickDone(); this.claimSlot(); void withRequestContext(this.orm, () => this.pass(true, new Date())) .catch((err: unknown) => { @@ -210,6 +231,11 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr }); } + /** A release tick holds the slot for seconds: a pass waits for it rather than losing its night. */ + private async releaseTickDone(): Promise { + if (this.jobs.exclusiveHolder === STORAGE_RELEASE_JOB) await this.jobs.exclusiveIdle(); + } + private claimSlot(): void { if (this.runtimeEnv.isDemoMode()) throw refusal('reclaim_demo', 'storage reclaim does not run on a demo instance'); if (!this.jobs.claimExclusive(JOB_NAME)) { @@ -217,47 +243,12 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr } } - /** - * This install's id, made and stored on first use. - * - * @txStandalone an idempotent lazy provision: the row is made once and only - * read after, whatever the caller writes besides. - */ - private async installId(): Promise { - if (this.knownInstallId) return this.knownInstallId; - let id = await this.appSettings.getValue(INSTALL_ID_KEY); - if (id === null || !INSTALL_ID.test(id)) { - id = randomUUID(); - await this.appSettings.upsertOrReplace(INSTALL_ID_KEY, id); - } - this.knownInstallId = id; - return id; - } - - /** - * A restore or a reopen swapped the database: a pass in flight stops, and - * the restored rows get this install's id back, deletion off and no marks, - * confirmations or reports. Contained: a listener that throws fails the reopen. - */ - private async afterReopen(): Promise { - this.generation += 1; - try { - await withRequestContext(this.orm, () => - this.uow.transactional(async () => { - const wasOn = this.knownInstallId !== null && (await this.appSettings.getValue(RECLAIM_DELETE_KEY)) !== null; - if (this.knownInstallId) await this.appSettings.upsertOrReplace(INSTALL_ID_KEY, this.knownInstallId); - await RECLAIM_RESET_KEYS.reduce>(async (previous, key) => { - await previous; - await this.appSettings.deleteValue(key); - }, Promise.resolve()); - await this.marks.deleteAll(); - if (wasOn) - logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on'); - }), - ); - } catch (err) { - logError(`[storage] could not reset the reclaim state after the database was reopened: ${messageOf(err)}`); + private requireInstallId(): string { + const installId = this.install.installId(); + if (installId === null) { + throw refusal('reclaim_install_unknown', 'this installation has no install id file, so nothing is reclaimed'); } + return installId; } /** @@ -266,23 +257,30 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr */ private async pass(reportOnly: boolean, now: Date): Promise { const wallStart = Date.now(); - const installId = await this.installId(); - const deleting = !reportOnly && (await readReclaimDeleteEnabled(this.appSettings, installId)); + const installId = this.install.installId(); + const deleting = !reportOnly && installId !== null && (await readReclaimDeleteEnabled(this.appSettings, installId)); const ctx: PassContext = { now: now.getTime(), installId, deleting, + backupHeld: false, sources: [...this.references.sources()], shared: sharedLocationsOf(this.registry), - confirmed: await readReclaimConfirmations(this.appSettings, installId), - generation: this.generation, + confirmations: installId === null ? {} : await readReclaimConfirmations(this.appSettings, installId), + spent: new Map(), + named: null, + epoch: this.install.epoch, }; const categories = {} as Record; let stopped: StorageReclaimReport['stopped'] = null; await SERVED_CATEGORIES.reduce>(async (previous, category) => { await previous; const progress = newProgress(); - if (stopped === null) { + if (!RECLAIM_LEDGER[category].reclaim) { + progress.skipped = 'not_reclaimed'; + } else if (stopped !== null) { + progress.skipped = 'interrupted'; + } else { try { await this.reclaimCategory(category, ctx, progress); } catch (err) { @@ -290,11 +288,10 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr if (stopped === 'failed') logError(`[storage] the reclaim pass failed in '${category}': ${messageOf(err)}`); progress.skipped = 'interrupted'; } - } else { - progress.skipped = 'interrupted'; } categories[category] = reportOf(progress); }, Promise.resolve()); + ctx.named = null; const report: StorageReclaimReport = { startedAt: ctx.now, finishedAt: ctx.now + (Date.now() - wallStart), @@ -302,10 +299,29 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr stopped, categories, }; + if (stopped !== 'database_reopened') await this.pruneConfirmations(ctx); await this.persist(report); return report; } + /** + * Drops the confirmations the pass found used up (the share is back under + * a fifth) or run out, unless an admin gave a new one meanwhile. + * + * @txStandalone one read and one write of the confirmations row, complete on its own. + */ + private async pruneConfirmations(ctx: PassContext): Promise { + if (ctx.spent.size === 0 || ctx.installId === null) return; + const current = await readReclaimConfirmations(this.appSettings, ctx.installId); + const kept: ReclaimConfirmations = {}; + for (const [category, confirmation] of Object.entries(current) as Array< + [ServedCategory, ReclaimConfirmations[ServedCategory]] + >) { + if (confirmation && ctx.spent.get(category) !== confirmation.confirmedAt) kept[category] = confirmation; + } + await writeReclaimConfirmations(this.appSettings, ctx.installId, kept); + } + /** * The report as the last pass's, and as the last deleting pass's when it ran * with deletion on; an audit entry (the system as actor) when it deleted. @@ -344,21 +360,12 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr * delete stands alone, so a failure part way keeps what came before. */ private async reclaimCategory(category: ServedCategory, ctx: PassContext, progress: CategoryProgress): Promise { - const rule = RECLAIM_LEDGER[category]; - if (!rule.reclaim) { - progress.skipped = 'not_reclaimed'; - return; - } - const sources = ctx.sources.filter((source) => claimsAny(source, category)); - const answering = new Set(sources.filter((source) => answersFor(source, category)).map((source) => source.id)); - if (rule.requiredSources.some((id) => !answering.has(id))) { - progress.skipped = 'sources_missing'; - return; - } - if (ctx.shared.has(category)) { - progress.skipped = 'shared_location'; + const judged = this.guard.judgeable(category, ctx.sources, ctx.shared); + if ('refused' in judged) { + progress.skipped = judged.refused; return; } + const { sources } = judged; let listed: { objects: number; old: ReclaimCandidate[]; owners: Set }; try { @@ -370,7 +377,10 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr return; } progress.objects = listed.objects; - const owner = await this.ownerVerdict(category, listed.owners, ctx.installId); + const owner = + ctx.installId === null + ? 'owner_unknown' + : await this.guard.ownerVerdict(category, listed.owners, ctx.installId, ctx.now); if (owner !== null) { progress.skipped = owner; return; @@ -378,7 +388,7 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr let named: Set; try { - named = await this.namedIn(category, sources); + named = await this.namedIn(category, sources, ctx); } catch (err) { logError(`[storage] reclaim could not check the references of '${category}': ${messageOf(err)}`); progress.skipped = 'source_failed'; @@ -392,9 +402,20 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr await this.writeMarks(category, unreferenced, marks, ctx.now); const cutoff = ctx.now - RECLAIM_DELAY_MS; progress.eligible = unreferenced.filter((candidate) => markedBefore(marks.get(candidate.key), cutoff)); + const over = overConfirmShare(unreferenced.length, listed.objects); + this.noteConfirmation(category, over, ctx); progress.confirmationRequired = - unreferenced.length > listed.objects * CONFIRM_SHARE && unreferenced.length > (ctx.confirmed[category] ?? 0); - if (ctx.deleting && !progress.confirmationRequired) await this.deleteEligible(category, sources, ctx, progress); + over && !confirmationCovers(ctx.confirmations[category], unreferenced.length, ctx.now); + if (progress.confirmationRequired || progress.eligible.length === 0) return; + if (ctx.deleting) await this.deleteEligible(category, sources, ctx, progress); + else if (ctx.backupHeld) progress.skipped = 'backup_running'; + } + + /** A confirmation is spent once the share is back under a fifth, and void once it ran out. */ + private noteConfirmation(category: ServedCategory, over: boolean, ctx: PassContext): void { + const confirmation = ctx.confirmations[category]; + if (!confirmation) return; + if (!over || confirmationExpired(confirmation, ctx.now)) ctx.spent.set(category, confirmation.confirmedAt); } /** The root-level objects of a category, of them the ones past the grace period, and the owner markers there. */ @@ -424,53 +445,49 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr } /** - * Whether this installation may judge the place a category lives in: not - * where another one's owner marker lies, and not where its own could not be - * written, since then a second installation cannot see it either. + * Every key of the category some source names, lowercased, from the one read + * of each source the pass makes at its first judged category; the category's + * sets are dropped once they are merged, so memory falls as the pass goes. */ - private async ownerVerdict( + private async namedIn( category: ServedCategory, - owners: ReadonlySet, - installId: string, - ): Promise<'foreign_install' | 'owner_unknown' | null> { - const foreign = [...owners].some((owner) => owner !== installId); - if (!owners.has(installId)) { - try { - await this.storage.put(category, storageOwnerMarkerKey(installId), Readable.from([installId]), { - contentType: 'text/plain', - }); - } catch (err) { - logWarn(`[storage] reclaim could not write its owner marker into '${category}': ${messageOf(err)}`); - return foreign ? 'foreign_install' : 'owner_unknown'; - } - } - if (foreign) logWarn(`[storage] another TREK installation keeps its uploads where '${category}' does: not judged`); - return foreign ? 'foreign_install' : null; - } - - /** Every key of the category some source names, lowercased: one read per source for the whole category. */ - private async namedIn(category: ServedCategory, sources: readonly StorageReferenceSource[]): Promise> { - const answers = await Promise.all(sources.map((source) => source.namedKeys(category))); + sources: readonly StorageReferenceSource[], + ctx: PassContext, + ): Promise> { + const cache = (ctx.named ??= await this.readNamed(ctx.sources)); const named = new Set(); - for (const answer of answers) for (const key of answer) named.add(key); + for (const source of sources) { + const answer = cache.get(source); + if (answer === undefined) throw new Error(`the source '${source.id}' was not read`); + if (answer instanceof Error) throw answer; + for (const key of answer.get(category) ?? []) named.add(key); + } + for (const answer of cache.values()) if (!(answer instanceof Error)) answer.delete(category); return named; } - /** The keys some source names right now, lowercased: the fresh question before a delete. */ - private async referencedIn( - category: ServedCategory, - keys: readonly string[], - sources: readonly StorageReferenceSource[], - ): Promise> { - const answers = await Promise.all( - sources.map(async (source) => { - const asked = keys.filter((key) => claimsKey(source, category, key)); - return asked.length === 0 ? new Set() : await source.referenced(category, asked); - }), - ); - const referenced = new Set(); - for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase()); - return referenced; + /** One read of each source for every reclaimable category, one source after another to bound memory. */ + private async readNamed(sources: readonly StorageReferenceSource[]): Promise { + const categories = reclaimableCategories(); + const cache: NamedCache = new Map(); + await sources.reduce>(async (previous, source) => { + await previous; + try { + cache.set(source, await source.namedKeysByCategory(categories)); + } catch (err) { + logError(`[storage] reclaim could not read the reference source '${source.id}': ${messageOf(err)}`); + cache.set(source, err instanceof Error ? err : new Error(String(err))); + } + }, Promise.resolve()); + return cache; + } + + /** Every key of the category some source names right now, lowercased: the fresh read before deletes. */ + private async namedNow(category: ServedCategory, sources: readonly StorageReferenceSource[]): Promise> { + const answers = await Promise.all(sources.map((source) => source.namedKeys(category))); + const named = new Set(); + for (const answer of answers) for (const key of answer) named.add(key.toLowerCase()); + return named; } /** @@ -499,7 +516,8 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr /** * The eligible objects a batch at a time, while deleting may still go; a - * batch whose backend failed stops the category with `delete_failed`. + * batch whose backend failed stops the category with `delete_failed`, a + * backup that started stops it with `backup_running`. * * @txIndependent a batch at a time, each delete standing alone, so a failure * leaves the others' work standing. @@ -510,9 +528,13 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr ctx: PassContext, progress: CategoryProgress, ): Promise { - await chunk(progress.eligible, RECHECK_BATCH).reduce>(async (previous, batch) => { + await chunk(progress.eligible, FRESH_BATCH).reduce>(async (previous, batch) => { await previous; - if (progress.skipped !== null || !(await this.mayDelete(ctx))) return; + if (progress.skipped !== null) return; + if (!(await this.mayDelete(ctx))) { + if (ctx.backupHeld) progress.skipped = 'backup_running'; + return; + } this.assertSameDatabase(ctx); const failed = await this.deleteBatch(category, batch, sources, ctx, progress); if (failed) progress.skipped = 'delete_failed'; @@ -521,18 +543,22 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr /** Deleting goes on while the switch stays on and no backup reads the uploads; once off, off for the pass. */ private async mayDelete(ctx: PassContext): Promise { - if (!ctx.deleting) return false; - if (this.jobs.backupRunning || !(await readReclaimDeleteEnabled(this.appSettings, ctx.installId))) { + if (!ctx.deleting || ctx.installId === null) return false; + if (this.jobs.backupRunning) { + ctx.deleting = false; + ctx.backupHeld = true; + logInfo('[storage] a backup is reading the uploads: this reclaim pass deletes nothing more, the next one will'); + } else if (!(await readReclaimDeleteEnabled(this.appSettings, ctx.installId))) { ctx.deleting = false; } return ctx.deleting; } /** - * Checks each object is still the old one the pass judged, asks every source - * about the batch again, fresh, and deletes what none of them names. The - * question goes last so nothing slower than the deletes sits between the - * answer and them. True when the backend failed, so the category stops. + * Checks each object is still the old one the pass judged, reads every + * source again, fresh and whole, and deletes what none of them names. The + * read goes last so nothing slower than the deletes sits between the answer + * and them. True when the backend failed, so the category stops. * * @txIndependent every object goes on its own; its mark goes after it. */ @@ -545,7 +571,7 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr ): Promise { let stats: Awaited>[]; try { - stats = await Promise.all(batch.map((candidate) => this.storage.stat(category, candidate.key))); + stats = await inGroups(batch, (candidate) => this.storage.stat(category, candidate.key)); } catch (err) { logError(`[storage] reclaim stopped deleting in '${category}', the backend failed: ${messageOf(err)}`); return true; @@ -562,7 +588,7 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr if (due.length === 0) return false; let referenced: Set; try { - referenced = await this.referencedIn(category, keysOf(due), sources); + referenced = await this.namedNow(category, sources); } catch (err) { logError( `[storage] reclaim kept ${due.length} '${category}' object(s), their re-check failed: ${messageOf(err)}`, @@ -594,12 +620,20 @@ export class StorageReclaimService implements OnModuleInit, OnApplicationBootstr /** A restore that began, or a reopen that happened, since the pass started ends it. */ private assertSameDatabase(ctx: PassContext): void { if (this.jobs.restoreRunning) throw new ReclaimStopped('a restore began during the pass'); - if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass'); + if (this.install.epoch !== ctx.epoch) throw new ReclaimStopped('the database was reopened during the pass'); } } /** A 409 whose body names the refusal in `code` next to the text, so the client can say it in the user's language. */ -function refusal(code: 'reclaim_busy' | 'reclaim_demo' | 'reclaim_nothing_to_confirm', message: string): DomainError { +function refusal( + code: + | 'reclaim_busy' + | 'reclaim_demo' + | 'reclaim_nothing_to_confirm' + | 'reclaim_confirm_stale' + | 'reclaim_install_unknown', + message: string, +): DomainError { return new DomainError(409, message, { code, details: { code } }); } @@ -608,20 +642,6 @@ function markedBefore(mark: ReclaimMarkRow | undefined, cutoff: number): boolean return firstSeen !== null && firstSeen.getTime() <= cutoff; } -/** The source answers for some part of the category. */ -function claimsAny(source: StorageReferenceSource, category: ServedCategory): boolean { - return source.claims.some((claim) => claim.category === category); -} - -/** The source answers for the whole category: what the ledger's required sources must do. */ -function answersFor(source: StorageReferenceSource, category: ServedCategory): boolean { - return source.claims.some((claim) => claim.category === category && claim.prefix === ''); -} - -function claimsKey(source: StorageReferenceSource, category: ServedCategory, key: string): boolean { - return source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix)); -} - function keysOf(candidates: readonly ReclaimCandidate[]): string[] { return candidates.map((candidate) => candidate.key); } @@ -630,6 +650,16 @@ function chunk(items: readonly T[], size: number): T[][] { return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size)); } +/** `work` over every item, DELETE_CONCURRENCY at a time, the answers in order; the first failure rejects. */ +async function inGroups(items: readonly T[], work: (item: T) => Promise): Promise { + const answers: R[] = []; + await chunk(items, DELETE_CONCURRENCY).reduce>(async (previous, group) => { + await previous; + answers.push(...(await Promise.all(group.map(work)))); + }, Promise.resolve()); + return answers; +} + function messageOf(err: unknown): string { return err instanceof Error ? err.message : String(err); } diff --git a/server/src/nest/storage/storage-reclaim.settings.ts b/server/src/nest/storage/storage-reclaim.settings.ts index 1f8323084..bf8d6fd91 100644 --- a/server/src/nest/storage/storage-reclaim.settings.ts +++ b/server/src/nest/storage/storage-reclaim.settings.ts @@ -6,13 +6,13 @@ import { storageReclaimReportSchema, type StorageReclaimReport } from '@trek/sha import { z } from 'zod'; /** - * This installation's random id, made on its first boot. A backup restore - * brings back the app_settings of whatever install made the backup, so the - * reclaim pass keeps the id it booted with and writes it back over the - * restored one (StorageReclaimService, on every database reopen). Outside the - * `storage.` prefix on purpose: the documented reset of the storage config - * (deleting every `storage.%` row) must not make this install a stranger to - * the owner markers it wrote. + * The install id the database was last reconciled with: a copy of the one in + * `data/.trek-install-id` (StorageInstallService), which no backup carries and + * no restore writes. A database whose row names another id came from another + * installation (a restore, a clone, a copied file), and its reclaim state is + * reset before anything reads it. Outside the `storage.` prefix on purpose: + * the documented reset of the storage config (deleting every `storage.%` row) + * must not read as a database from elsewhere. */ export const INSTALL_ID_KEY = 'install_id'; /** @@ -26,10 +26,15 @@ export const RECLAIM_REPORT_KEY = 'storage.reclaim_report'; export const RECLAIM_DELETE_REPORT_KEY = 'storage.reclaim_delete_report'; /** * Per category, the unreferenced count an admin confirmed when that was more - * than a fifth of the category, for the install it was confirmed on. + * than a fifth of the category, with when, for the install it was confirmed on. */ const RECLAIM_CONFIRMED_KEY = 'storage.reclaim_confirmed'; +/** More unreferenced than this share of a category waits for an admin's confirmation before anything goes. */ +const CONFIRM_SHARE = 0.2; +/** How long a confirmation lets a pass delete, unless the share fell under the line before. */ +const CONFIRM_TTL_MS = 14 * 24 * 60 * 60 * 1000; + /** Every row a restore or a foreign database must not carry over into this install's reclaim. */ export const RECLAIM_RESET_KEYS = [ RECLAIM_DELETE_KEY, @@ -40,9 +45,55 @@ export const RECLAIM_RESET_KEYS = [ const confirmationsSchema = z.object({ installId: z.string(), - counts: z.record(z.string(), z.number().int().nonnegative()), + categories: z.record( + z.string(), + z.object({ count: z.number().int().nonnegative(), confirmedAt: z.number().int().nonnegative() }), + ), }); +/** One admin confirmation: the unreferenced count the admin was shown, and when (epoch ms). */ +interface ReclaimConfirmation { + count: number; + confirmedAt: number; +} + +export type ReclaimConfirmations = Partial>; + +/** More of the category is unreferenced than a fifth of it. */ +export function overConfirmShare(unreferenced: number, objects: number): boolean { + return unreferenced > objects * CONFIRM_SHARE; +} + +/** The confirmation ran out: older than CONFIRM_TTL_MS. */ +export function confirmationExpired(confirmation: ReclaimConfirmation, now: number): boolean { + return now - confirmation.confirmedAt >= CONFIRM_TTL_MS; +} + +/** + * A live confirmation covers this many unreferenced objects: up to one and a + * half times the count the admin confirmed, and at least 100 more, so the + * orphans a busy instance adds before the backlog has drained do not undo it, + * while a surge well past it waits for a new one. + */ +export function confirmationCovers( + confirmation: ReclaimConfirmation | undefined, + unreferenced: number, + now: number, +): boolean { + if (!confirmation || confirmationExpired(confirmation, now)) return false; + return unreferenced <= Math.max(confirmation.count * 1.5, confirmation.count + 100); +} + +/** + * The count the admin was shown is not the one the last report holds: more + * than a tenth of it, or more than 50 objects, apart. Then a pass ran since + * the page loaded, and the admin would confirm a number they never saw. + */ +export function confirmationStale(shown: number, current: number): boolean { + const apart = Math.abs(current - shown); + return apart > shown * 0.1 || apart > 50; +} + /** Fail closed: only this install's own id switches deletion on. */ export async function readReclaimDeleteEnabled( appSettings: AppSettingsRepository, @@ -66,26 +117,31 @@ export async function readReclaimReport( } } -/** The confirmed counts of this install; none for a row another install wrote or that no longer parses. */ +/** The confirmations of this install; none for a row another install wrote or that no longer parses. */ export async function readReclaimConfirmations( appSettings: AppSettingsRepository, installId: string, -): Promise>> { +): Promise { const value = await appSettings.getValue(RECLAIM_CONFIRMED_KEY); if (value == null) return {}; try { const parsed = confirmationsSchema.parse(JSON.parse(value)); - return parsed.installId === installId ? (parsed.counts as Partial>) : {}; + return parsed.installId === installId ? (parsed.categories as ReclaimConfirmations) : {}; } catch { logWarn('[storage] the stored reclaim confirmations are unparseable, treating them as none'); return {}; } } +/** Stores this install's confirmations; none left drops the row. */ export async function writeReclaimConfirmations( appSettings: AppSettingsRepository, installId: string, - counts: Partial>, + categories: ReclaimConfirmations, ): Promise { - await appSettings.upsertOrReplace(RECLAIM_CONFIRMED_KEY, JSON.stringify({ installId, counts })); + if (Object.keys(categories).length === 0) { + await appSettings.deleteValue(RECLAIM_CONFIRMED_KEY); + return; + } + await appSettings.upsertOrReplace(RECLAIM_CONFIRMED_KEY, JSON.stringify({ installId, categories })); } diff --git a/server/src/nest/storage/storage-reclaim.types.ts b/server/src/nest/storage/storage-reclaim.types.ts index 23ae2ce1e..7d5d66efe 100644 --- a/server/src/nest/storage/storage-reclaim.types.ts +++ b/server/src/nest/storage/storage-reclaim.types.ts @@ -52,16 +52,21 @@ export interface StorageReferenceSource { */ readonly claims: ReadonlyArray; /** - * Every key of `category` the rows name, lowercased: one read of the whole - * column set, which a pass makes once per category. A row that is trashed, - * pending or otherwise not gone counts. Throwing stops the pass for the - * category; it never deletes on a failed answer. + * Every key of `category` the rows name, lowercased: one fresh read of the + * whole column set, the question a pass asks right before it deletes. A row + * that is trashed, pending or otherwise not gone counts. Throwing stops the + * pass for the category; it never deletes on a failed answer. */ namedKeys(category: ServedCategory): Promise>; /** - * Of these category-relative keys (as `StorageService.list` yields them, a - * batch at a time), the ones a row names right now: the question a delete - * asks last, read fresh and narrowed to the rows that can name them. + * The same for several categories from one read, each the source claims + * keyed in the answer: what a pass reads once and judges every category by. + */ + namedKeysByCategory(categories: readonly ServedCategory[]): Promise>>; + /** + * Of these category-relative keys (as `StorageService.list` yields them), the + * ones a row names right now, read narrowed to the rows that can name them: + * the question the release queue asks before it deletes. */ referenced(category: ServedCategory, keys: readonly string[]): Promise>; } diff --git a/server/src/nest/storage/storage-release.job.ts b/server/src/nest/storage/storage-release.job.ts new file mode 100644 index 000000000..b7feda744 --- /dev/null +++ b/server/src/nest/storage/storage-release.job.ts @@ -0,0 +1,39 @@ +import { logDebug, logError } from '../audit/audit-log.logger'; +import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { StorageReleaseService } from './storage-release.service'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; + +/** + * Every ten minutes, what users' deletes and replaces queued and what is due + * now (StorageReleaseService): each object asked about once more and deleted + * when no row names it. The registrar is handed the tick's own promise, so + * its lease covers the whole tick. + */ +@Injectable() +export class StorageReleaseJob implements OnApplicationBootstrap { + constructor( + private readonly registrar: CronRegistrarService, + private readonly release: StorageReleaseService, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + this.registrar.register('storage-release', '*/10 * * * *', () => this.tick()); + } + + /** + * @txIndependent every delete and every dropped row stands on its own + * (StorageReleaseService.processDue). + */ + async tick(now: Date = new Date()): Promise { + try { + const result = await this.release.processDue(now); + if (result.skipped !== null) logDebug(`Storage release skipped: ${result.skipped}`); + else if (result.deleted + result.kept > 0) { + logDebug(`Storage release: deleted ${result.deleted}, kept ${result.kept} still named`); + } + } catch (err: unknown) { + logError(`Storage release: ${err instanceof Error ? err.message : String(err)}`); + } + } +} diff --git a/server/src/nest/storage/storage-release.service.ts b/server/src/nest/storage/storage-release.service.ts index 8e19cbfa8..75c1025f8 100644 --- a/server/src/nest/storage/storage-release.service.ts +++ b/server/src/nest/storage/storage-release.service.ts @@ -1,38 +1,105 @@ +import { StorageReleaseQueue } from '../../db/entities/StorageReleaseQueue.entity'; +import type { + ReleaseQueueRow, + StorageReleaseQueueRepository, +} from '../../db/repositories/StorageReleaseQueue.repository'; +import { dbNow } from '../../db/types'; import { logDebug, logWarn } from '../audit/audit-log.logger'; +import { UnitOfWork } from '../database/unit-of-work'; +import { StorageInstallService } from './storage-install.service'; import { StorageJobsService } from './storage-jobs.service'; import { isValidKey } from './storage-keys'; +import { StorageReclaimGuardService } from './storage-reclaim-guard.service'; import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; -import { inReclaimScope, RECLAIM_LEDGER, sharedLocationsOf } from './storage-reclaim.ledger'; +import { inReclaimScope, RECLAIM_LEDGER, reclaimableCategories, sharedLocationsOf } from './storage-reclaim.ledger'; import { + keySearchCore, storageKeysIn, storageOwnerMarkerInstall, type StorageKeyOptions, - type StorageKeyRef, type StorageReferenceSource, } from './storage-reclaim.types'; import { StorageRegistryService } from './storage-registry.service'; import { StorageService } from './storage.service'; import { SERVED_CATEGORIES, type ServedCategory } from './storage.types'; +import { InjectRepository } from '@mikro-orm/nestjs'; import { Injectable } from '@nestjs/common'; +const HOUR_MS = 60 * 60 * 1000; +/** + * How long a released object waits before the queue may delete it: longer + * than every window in TREK that can write a value back that names it. + * - the assistant: a change card's undo lasts 15 minutes (UNDO_WINDOW_MS), and + * a proposed change may be approved for 24 hours (APPROVAL_TTL_MS), so a + * change proposed with an old cover path can still be applied a day later; + * - the offline mutation queue retries a write the server failed with a 5xx + * for about two hours (MAX_SERVER_ERROR_ATTEMPTS, doubling from 30 s); + * - the planner's own undo (usePlannerHistory) and a write queued while + * offline are bounded by nothing but the tab and the outage: those are + * unbounded, and the place undo does not offer to undo a delete that took an + * uploaded image (placeUndo.ts) for that reason. + * So the longest bounded window is a day; two hours on top cover the clock + * and the job's interval. What comes back later than that finds the object + * gone, as it did before the reclaim when every delete was immediate. + */ +export const RELEASE_DELAY_MS = 26 * HOUR_MS; +/** A queued row older than this past its time belongs to the nightly pass: its category stayed refused. */ +const QUEUE_MAX_AGE_MS = 30 * 24 * HOUR_MS; +/** Keys one stored value may queue: a value naming more than that is no upload path of ours. */ +const MAX_KEYS_PER_VALUE = 16; +/** Keys one call may queue; the rest is the nightly pass's. */ +const MAX_KEYS_PER_CALL = 1000; +/** No more rows than this wait at once; a release past it is the nightly pass's. */ +const QUEUE_LIMIT = 50_000; +/** Rows per insert: three bound values each. */ +const ENQUEUE_BATCH = 100; +/** Rows one tick looks at. */ +const TICK_LIMIT = 200; +/** Keys per question to the sources. */ +const ASK_BATCH = 50; +/** Deletes in flight at once. */ +const DELETE_CONCURRENCY = 10; + const SERVED = new Set(SERVED_CATEGORIES); +/** The name a release tick holds the one-job slot under. */ +export const STORAGE_RELEASE_JOB = 'a storage release'; + +export interface ReleaseTickResult { + /** Why the tick did nothing, or null when it ran. */ + skipped: 'busy' | 'no_install_id' | null; + deleted: number; + /** Rows dropped because a row still names the object, or the instance keeps it. */ + kept: number; +} + +/** A restore began, a backup started or the database was swapped mid-tick: the rest waits for the next one. */ +class ReleaseStopped extends Error {} + /** - * Deletes what a user's change left unnamed, right after it: a replaced or - * removed cover, place image or avatar, a deleted journey photo or trip file. - * The owner calls it once its transaction committed, with the values the rows - * held, and every registered reference source (the nightly pass's sources and - * parser) is asked whether anything still names each object. Only what no row - * names goes, through `StorageService.delete` (replicas follow, a demo keeps - * its bytes), so an object a trip copy, a journey or a saved place still shows - * stays, unlike before (#2924), while a deleted photo is gone at once as it - * always was. + * What a user's change left unnamed: a replaced or removed cover, place image + * or avatar, a deleted journey photo or trip file. The owner hands the values + * its rows held to `releaseValues` once its transaction committed, and every + * object they name is queued, which is all the request pays. Nothing goes + * before RELEASE_DELAY_MS, so an undo or a late write that brings the value + * back keeps its object. * - * The pass's own guards hold: only the root of a reclaimable category, never - * an owner marker, never while a restore runs, never where a required source - * is missing or the category's location overlaps another's. It never fails - * the caller's request: anything it leaves (an error, a guard) is the nightly - * pass's to find. Releases are logged at debug, not audited one by one. + * A job (StorageReleaseJob) then takes the due rows, asks every registered + * reference source again about them (the nightly pass's sources and parser) + * and deletes only what none of them names, through `StorageService.delete` + * (replicas follow, a demo keeps its bytes); the row goes either way. So an + * object a trip copy, a journey or a saved place still shows stays (#2924), + * while a deleted photo still goes as it did before, a day later. + * + * The queue refuses what the pass refuses (StorageReclaimGuardService): only + * the root of a reclaimable category, never an owner marker, never where a + * required source is missing, where the category's location overlaps another + * one's, or where another installation's owner marker lies (or this one's + * cannot be written). It does nothing while a restore or a backup runs or + * another storage job holds the slot, and without an install id. A refused + * category's rows wait, up to QUEUE_MAX_AGE_MS. Releases run whatever the + * nightly deletion switch says: they are what a delete always did. Queueing + * never fails the caller's request. */ @Injectable() export class StorageReleaseService { @@ -41,75 +108,186 @@ export class StorageReleaseService { private readonly registry: StorageRegistryService, private readonly references: StorageReclaimRegistryService, private readonly jobs: StorageJobsService, + private readonly install: StorageInstallService, + private readonly guard: StorageReclaimGuardService, + private readonly uow: UnitOfWork, + @InjectRepository(StorageReleaseQueue) private readonly queue: StorageReleaseQueueRepository, ) {} /** - * The objects these stored values name (any spelling a column holds), each - * deleted when no row names it any more. Answers the ones that went. + * Queues the objects these stored values name (any spelling a column holds) + * for a release once they are due; answers how many were queued. Never + * throws: what it cannot queue is the nightly pass's. + * + * @txStandalone a queue row stands on its own whatever the caller wrote: + * the job asks every source again before anything goes. */ async releaseValues( values: ReadonlyArray, options: StorageKeyOptions = {}, - ): Promise { - const byCategory = new Map>(); - for (const value of values) { - for (const ref of storageKeysIn(value, options)) { - if (!SERVED.has(ref.category)) continue; - const category = ref.category as ServedCategory; - const keys = byCategory.get(category) ?? new Set(); - keys.add(ref.key); - byCategory.set(category, keys); - } - } - const released = await Promise.all( - [...byCategory].map(async ([category, keys]) => - (await this.releaseIfUnreferenced(category, [...keys])).map((key) => ({ category, key })), - ), - ); - return released.flat(); - } - - /** These keys of one category, each deleted when it exists and no row names it. Answers the ones that went. */ - async releaseIfUnreferenced(category: ServedCategory, keys: readonly string[]): Promise { + ): Promise { try { - const candidates = [...new Set(keys)].filter( - (key) => inReclaimScope(key) && isValidKey(key) && storageOwnerMarkerInstall(key) === null, - ); - if (candidates.length === 0) return []; - const sources = this.sourcesFor(category); - if (sources === null) return []; - const referenced = await this.referencedIn(category, candidates, sources); - const unnamed = candidates.filter( - (key) => !referenced.has(key.toLowerCase()) && !this.storage.keepsOnDelete(category, key), - ); - const released = await Promise.all(unnamed.map((key) => this.deleteIfPresent(category, key))); - return unnamed.filter((_key, i) => released[i]); + const rows = releasable(values, options); + if (rows.length === 0) return 0; + if ((await this.queue.size()) + rows.length > QUEUE_LIMIT) { + logWarn(`[storage] the release queue is full, ${rows.length} object(s) are left to the nightly pass`); + return 0; + } + const dueAt = dbNow(new Date(Date.now() + RELEASE_DELAY_MS)); + await this.uow.transactional(async () => { + await chunk(rows, ENQUEUE_BATCH).reduce>(async (previous, page) => { + await previous; + await this.queue.enqueue(page, dueAt); + }, Promise.resolve()); + }); + return rows.length; } catch (err) { - logWarn(`[storage] could not release '${category}' objects, the nightly pass will: ${messageOf(err)}`); - return []; + logWarn(`[storage] could not queue released objects, the nightly pass will find them: ${messageOf(err)}`); + return 0; } } - /** The sources to ask, or null when the category may not be released from now. */ - private sourcesFor(category: ServedCategory): readonly StorageReferenceSource[] | null { - const rule = RECLAIM_LEDGER[category]; - if (!rule.reclaim || this.jobs.restoreRunning) return null; - const sources = this.references.sources().filter((source) => source.claims.some((c) => c.category === category)); - const whole = new Set( - sources - .filter((source) => source.claims.some((c) => c.category === category && c.prefix === '')) - .map((source) => source.id), - ); - if (rule.requiredSources.some((id) => !whole.has(id))) return null; - if (sharedLocationsOf(this.registry).has(category)) return null; - return sources; + /** + * One tick of the release job: the due rows of every category the guards + * let through, at most TICK_LIMIT, each object deleted when no source names + * it. Holds the one-job slot while it runs, so a restore waits for it and a + * backfill or a migration never runs beside it. + * + * @txIndependent every delete and every dropped row stands on its own: a + * tick cut short leaves the rest queued. + */ + async processDue(now: Date = new Date()): Promise { + const result: ReleaseTickResult = { skipped: null, deleted: 0, kept: 0 }; + const installId = this.install.installId(); + if (installId === null) return { ...result, skipped: 'no_install_id' }; + if (this.jobs.backupRunning || !this.jobs.claimExclusive(STORAGE_RELEASE_JOB)) + return { ...result, skipped: 'busy' }; + try { + await this.drain(installId, now.getTime(), this.install.epoch, result); + } finally { + this.jobs.releaseExclusive(STORAGE_RELEASE_JOB); + } + return result; } - private async referencedIn( + /** @txIndependent see processDue. */ + private async drain(installId: string, now: number, epoch: number, result: ReleaseTickResult): Promise { + await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS))); + const due = new Set(await this.queue.dueCategories(dbNow(new Date(now)))); + if (due.size === 0) return; + const allowed = await this.allowedCategories(due, installId, now); + const rows = await this.queue.dueIn([...allowed.keys()], dbNow(new Date(now)), TICK_LIMIT); + const byCategory = new Map(); + for (const row of rows) { + const category = row.category as ServedCategory; + byCategory.set(category, [...(byCategory.get(category) ?? []), row.key]); + } + try { + await [...byCategory].reduce>(async (previous, [category, keys]) => { + await previous; + await this.releaseCategory(category, keys, allowed.get(category) ?? [], epoch, result); + }, Promise.resolve()); + } catch (err) { + if (!(err instanceof ReleaseStopped)) throw err; + logDebug(`[storage] a release tick stopped: ${err.message}`); + } + } + + /** Of the categories with rows due, the ones the guards let a release judge now, each with the sources to ask. */ + private async allowedCategories( + due: ReadonlySet, + installId: string, + now: number, + ): Promise> { + const sources = this.references.sources(); + const shared = sharedLocationsOf(this.registry); + const allowed = new Map(); + const asked = reclaimableCategories().filter((category) => due.has(category)); + await asked.reduce>(async (previous, category) => { + await previous; + const judged = this.guard.judgeable(category, sources, shared); + if ('refused' in judged) return; + if ((await this.guard.cachedOwnerVerdict(category, installId, now)) !== null) return; + allowed.set(category, judged.sources); + }, Promise.resolve()); + return allowed; + } + + /** + * The due keys of one category, ASK_BATCH at a time: asked about, then + * deleted when unnamed. A failed question or delete stops the category for + * this tick and leaves its rows queued. + * + * @txIndependent every delete stands alone, its row goes after it. + */ + private async releaseCategory( category: ServedCategory, keys: readonly string[], sources: readonly StorageReferenceSource[], + epoch: number, + result: ReleaseTickResult, + ): Promise { + let failed = false; + await chunk(keys, ASK_BATCH).reduce>(async (previous, batch) => { + await previous; + if (failed) return; + this.assertUndisturbed(epoch); + failed = await this.releaseBatch(category, batch, sources, epoch, result); + }, Promise.resolve()); + } + + /** True when the batch failed part way, so the category stops for this tick. */ + private async releaseBatch( + category: ServedCategory, + batch: readonly string[], + sources: readonly StorageReferenceSource[], + epoch: number, + result: ReleaseTickResult, + ): Promise { + let referenced: Set; + try { + referenced = await this.referencedIn(category, batch, sources); + } catch (err) { + logWarn(`[storage] could not ask about released '${category}' objects, they wait: ${messageOf(err)}`); + return true; + } + const kept = batch.filter((key) => referenced.has(key.toLowerCase()) || this.storage.keepsOnDelete(category, key)); + const unnamed = batch.filter((key) => !kept.includes(key)); + this.assertUndisturbed(epoch); + const deleted: string[] = []; + let failure: unknown = null; + await chunk(unnamed, DELETE_CONCURRENCY).reduce>(async (previous, group) => { + await previous; + if (failure !== null) return; + const results = await Promise.allSettled(group.map((key) => this.storage.delete(category, key))); + results.forEach((settled, i) => { + if (settled.status === 'fulfilled') deleted.push(group[i] as string); + else failure ??= settled.reason; + }); + }, Promise.resolve()); + await this.queue.deleteKeys(category, [...kept, ...deleted]); + result.kept += kept.length; + result.deleted += deleted.length; + if (deleted.length > 0) logDebug(`[storage] released ${deleted.length} unreferenced '${category}' object(s)`); + if (failure === null) return false; + logWarn(`[storage] could not release a '${category}' object, it waits: ${messageOf(failure)}`); + return true; + } + + /** + * The keys some source names right now, lowercased. A key no source can + * narrow its read to (no plain run of 8 characters, which every name TREK + * writes has) counts as named without a question: asking about it would read + * whole tables, so it is the nightly pass's. + */ + private async referencedIn( + category: ServedCategory, + batch: readonly string[], + sources: readonly StorageReferenceSource[], ): Promise> { + const referenced = new Set(batch.filter((key) => keySearchCore(key) === null).map((key) => key.toLowerCase())); + const keys = batch.filter((key) => keySearchCore(key) !== null); + if (keys.length === 0) return referenced; const answers = await Promise.all( sources.map(async (source) => { const asked = keys.filter((key) => @@ -118,22 +296,50 @@ export class StorageReleaseService { return asked.length === 0 ? new Set() : await source.referenced(category, asked); }), ); - const referenced = new Set(); for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase()); return referenced; } - private async deleteIfPresent(category: ServedCategory, key: string): Promise { - try { - if ((await this.storage.stat(category, key)) === null) return false; - await this.storage.delete(category, key); - logDebug(`[storage] released an unreferenced '${category}' object`); - return true; - } catch (err) { - logWarn(`[storage] could not release a '${category}' object, the nightly pass will: ${messageOf(err)}`); - return false; + /** A restore or a backup that began, or a reopen that happened, since the tick started ends it. */ + private assertUndisturbed(epoch: number): void { + if (this.jobs.restoreRunning) throw new ReleaseStopped('a restore began'); + if (this.jobs.backupRunning) throw new ReleaseStopped('a backup began'); + if (this.install.epoch !== epoch) throw new ReleaseStopped('the database was reopened'); + } +} + +/** + * The rows these values queue: root-level keys of reclaimable categories, + * never an owner marker, at most MAX_KEYS_PER_VALUE per value and + * MAX_KEYS_PER_CALL in all. + */ +function releasable( + values: ReadonlyArray, + options: StorageKeyOptions, +): Array> { + const seen = new Set(); + const rows: Array> = []; + for (const value of values) { + const refs = storageKeysIn(value, options).filter( + (ref) => + SERVED.has(ref.category) && + RECLAIM_LEDGER[ref.category as ServedCategory].reclaim && + inReclaimScope(ref.key) && + isValidKey(ref.key) && + storageOwnerMarkerInstall(ref.key) === null, + ); + for (const ref of refs.slice(0, MAX_KEYS_PER_VALUE)) { + const id = `${ref.category}\n${ref.key}`; + if (seen.has(id) || rows.length >= MAX_KEYS_PER_CALL) continue; + seen.add(id); + rows.push({ category: ref.category, key: ref.key }); } } + return rows; +} + +function chunk(items: readonly T[], size: number): T[][] { + return Array.from({ length: Math.ceil(items.length / size) }, (_page, i) => items.slice(i * size, (i + 1) * size)); } function messageOf(err: unknown): string { diff --git a/server/src/nest/storage/storage.module.ts b/server/src/nest/storage/storage.module.ts index b04c228c8..7a5c5e9dd 100644 --- a/server/src/nest/storage/storage.module.ts +++ b/server/src/nest/storage/storage.module.ts @@ -1,5 +1,6 @@ import { AppSettings } from '../../db/entities/AppSettings.entity'; import { StorageReclaimMarks } from '../../db/entities/StorageReclaimMarks.entity'; +import { StorageReleaseQueue } from '../../db/entities/StorageReleaseQueue.entity'; import { AppConfigModule } from '../app-config/app-config.module'; import { AuditModule } from '../audit/audit.module'; import { DatabaseLifecycleModule } from '../database/database-lifecycle.module'; @@ -7,12 +8,15 @@ import { SchedulingModule } from '../scheduling/scheduling.module'; import { StorageAdminController } from './storage-admin.controller'; import { StorageAdminService } from './storage-admin.service'; import { StorageEventsService } from './storage-events.service'; +import { StorageInstallService } from './storage-install.service'; import { StorageJobsService } from './storage-jobs.service'; +import { StorageReclaimGuardService } from './storage-reclaim-guard.service'; import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; import { StorageReclaimController } from './storage-reclaim.controller'; import { StorageReclaimJob } from './storage-reclaim.job'; import { StorageReclaimService } from './storage-reclaim.service'; import { StorageRegistryService } from './storage-registry.service'; +import { StorageReleaseJob } from './storage-release.job'; import { StorageReleaseService } from './storage-release.service'; import { StorageStatsService } from './storage-stats.service'; import { StorageUsageScanJob } from './storage-usage-scan.job'; @@ -42,7 +46,10 @@ import { Module } from '@nestjs/common'; * running there and waits for a reclaim pass in flight to let go, a backup * marks itself so the pass deletes nothing it still reads. * StorageReleaseService is exported for the owners, which hand it the values a - * delete or a replace left unnamed right after their write. + * delete or a replace left unnamed right after their write; it queues them, + * and StorageReleaseJob deletes what is still unnamed once every undo window + * has passed. StorageInstallService keeps this installation's id in the data + * dir and resets the reclaim state of a database that came from elsewhere. */ @Module({ imports: [ @@ -50,7 +57,7 @@ import { Module } from '@nestjs/common'; AuditModule, DatabaseLifecycleModule, SchedulingModule, - MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks]), + MikroOrmModule.forFeature([AppSettings, StorageReclaimMarks, StorageReleaseQueue]), ], controllers: [StorageAdminController, StorageReclaimController], providers: [ @@ -62,9 +69,12 @@ import { Module } from '@nestjs/common'; StorageStatsService, StorageUsageScanJob, StorageReclaimRegistryService, + StorageInstallService, + StorageReclaimGuardService, StorageReclaimService, StorageReclaimJob, StorageReleaseService, + StorageReleaseJob, ], exports: [ StorageService, diff --git a/server/src/nest/trips/trips.service.ts b/server/src/nest/trips/trips.service.ts index b01ce0e94..2422ca3e4 100644 --- a/server/src/nest/trips/trips.service.ts +++ b/server/src/nest/trips/trips.service.ts @@ -537,7 +537,7 @@ export class TripsService { } }); if (vacayId !== undefined) await this.vacay.notifyPlanUsers(vacayId, undefined); - if (cover_image !== undefined) await this.release.releaseValues([trip.cover_image]); // kept while any row names it + if (cover_image !== undefined && cover_image !== trip.cover_image) await this.release.releaseValues([trip.cover_image]); const changes: Record = {}; if (title && title !== trip.title) changes.title = title; diff --git a/server/tests/e2e/storage-reclaim.e2e.test.ts b/server/tests/e2e/storage-reclaim.e2e.test.ts index 3e32ce536..7d899bc3c 100644 --- a/server/tests/e2e/storage-reclaim.e2e.test.ts +++ b/server/tests/e2e/storage-reclaim.e2e.test.ts @@ -5,19 +5,22 @@ * contracts the handlers declare all stand in front of it. Every owner * domain registers its reference source here, so a pass looks at every * reclaimable category; the checks below run in the background and only - * report. RCLE-001 to RCLE-007. + * report. RCLE-001 to RCLE-008. */ import { buildApp } from '../../src/bootstrap'; import { AppSettings } from '../../src/db/entities/AppSettings.entity'; import { AuditLog } from '../../src/db/entities/AuditLog.entity'; +import { StorageInstallService } from '../../src/nest/storage/storage-install.service'; import { StorageJobsService } from '../../src/nest/storage/storage-jobs.service'; import { STORAGE_RECLAIMABLE_CATEGORIES, storageOwnerMarkerKey } from '../../src/nest/storage/storage-reclaim.types'; import { StorageService } from '../../src/nest/storage/storage.service'; import { countRows, deleteRows, findRow } from '../helpers/factories/rows'; +import { setAppSetting } from '../helpers/factories/settings'; import { makeAdmin, makeUser } from '../helpers/factories/users'; import { sessionCookie } from './harness'; import { MikroORM } from '@mikro-orm/core'; import type { INestApplication } from '@nestjs/common'; +import { STORAGE_RECLAIM_CATEGORIES, type StorageReclaimReport } from '@trek/shared'; import type { Server } from 'http'; import request from 'supertest'; @@ -53,7 +56,7 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar afterAll(async () => { // The uploads folder is the test run's shared one: take back the owner markers this app's checks wrote. - const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value; + const installId = app.get(StorageInstallService).installId(); if (installId) { const storage = app.get(StorageService); await Promise.all( @@ -99,9 +102,11 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar expect(res.status).toBe(200); expect(res.body).toEqual({ deleteEnabled: true, running: false, lastRun: null, lastDeleteRun: null }); expect(await countRows(orm, AuditLog, { action: 'admin.storage_reclaim_update' })).toBe(1); - // Stored as the install id it was turned on for, so a database restored elsewhere reads as off. + // Stored as the install id it was turned on for, so a database restored elsewhere reads as off. The id + // comes from the data dir's file, and the database carries a copy it is reconciled with at boot. const installId = (await findRow(orm, AppSettings, { key: 'install_id' }))?.value; expect(installId).toMatch(/^[0-9a-f-]{36}$/); + expect(installId).toBe(app.get(StorageInstallService).installId()); expect((await findRow(orm, AppSettings, { key: 'storage.reclaim_delete' }))?.value).toBe(installId); expect((await settledState()).deleteEnabled).toBe(true); const off = await request(server) @@ -119,11 +124,15 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar ); expect(refused.map((res) => res.status)).toEqual([400, 400, 400]); const confirm = await Promise.all( - [{}, { category: 'nowhere' }, { category: 'covers', count: 3 }].map((body) => - request(server).post('/api/admin/storage/reclaim/confirm').set('Cookie', adminCookie).send(body), - ), + [ + {}, + { category: 'nowhere', unreferenced: 3 }, + { category: 'covers', count: 3 }, + { category: 'covers' }, + { category: 'covers', unreferenced: -1 }, + ].map((body) => request(server).post('/api/admin/storage/reclaim/confirm').set('Cookie', adminCookie).send(body)), ); - expect(confirm.map((res) => res.status)).toEqual([400, 400, 400]); + expect(confirm.map((res) => res.status)).toEqual([400, 400, 400, 400, 400]); expect(await countRows(orm, AuditLog)).toBe(0); }); @@ -161,11 +170,49 @@ describe('Storage reclaim e2e (whole app: real auth + admin guard + managed guar const confirm = await request(server) .post('/api/admin/storage/reclaim/confirm') .set('Cookie', adminCookie) - .send({ category: 'covers' }); + .send({ category: 'covers', unreferenced: 3 }); expect(confirm.status).toBe(409); expect(confirm.body.code).toBe('reclaim_nothing_to_confirm'); }); + it('RCLE-008: confirm takes the count shown, refuses a stale one with a code, and the prompt goes at once', async () => { + const none = { objects: 0, bytes: 0 }; + const row = { + objects: 0, + unreferenced: none, + eligible: none, + deleted: none, + confirmationRequired: false, + skipped: null, + }; + const report: StorageReclaimReport = { + startedAt: Date.now(), + finishedAt: Date.now(), + mode: 'report', + stopped: null, + categories: { + ...(Object.fromEntries(STORAGE_RECLAIM_CATEGORIES.map((c) => [c, row])) as StorageReclaimReport['categories']), + covers: { ...row, objects: 1000, unreferenced: { objects: 300, bytes: 3000 }, confirmationRequired: true }, + }, + }; + await setAppSetting(orm, 'storage.reclaim_report', JSON.stringify(report)); + const confirm = (unreferenced: number) => + request(server) + .post('/api/admin/storage/reclaim/confirm') + .set('Cookie', adminCookie) + .send({ category: 'covers', unreferenced }); + const stale = await confirm(5000); + expect(stale.status).toBe(409); + expect(stale.body.code).toBe('reclaim_confirm_stale'); + expect(await countRows(orm, AuditLog)).toBe(0); + const ok = await confirm(300); + expect(ok.status).toBe(200); + expect(ok.body.lastRun.categories.covers.confirmationRequired).toBe(false); + const audit = await findRow(orm, AuditLog, { action: 'admin.storage_reclaim_confirm' }); + expect(JSON.parse(String(audit?.details))).toEqual({ category: 'covers', unreferenced: 300 }); + expect((await settledState()).lastRun.categories.covers.confirmationRequired).toBe(false); + }); + it('RCLE-007: managed mode refuses the surface with the standard body', async () => { process.env.TREK_MANAGED = 'true'; try { diff --git a/server/tests/helpers/storage-release.ts b/server/tests/helpers/storage-release.ts index 324f96953..b4e87ad57 100644 --- a/server/tests/helpers/storage-release.ts +++ b/server/tests/helpers/storage-release.ts @@ -1,19 +1,16 @@ /** * Stand-ins for StorageReleaseService in suites that hand-build an owner - * service. The release itself (asking every reference source, the ledger's - * guards) has its own suites: storage-release.service.test.ts and the - * integration suite over the full app. + * service. The release itself (the queue, the job asking every reference + * source, the guards) has its own suites: storage-release.service.test.ts, + * storage-release.job.test.ts and the integration suite over the full app. * - * - `releaseNothing()` releases nothing and records the calls; - * - `releaseDirect(storage)` deletes every root-level object the values name, - * as a suite with no second row naming them would see the real one do. + * - `releaseNothing()` queues nothing and records the calls; + * - `releaseDirect(storage)` deletes every root-level object the values name + * at once, as the queue would once the objects are due in a suite with no + * second row naming them. */ import { inReclaimScope } from '../../src/nest/storage/storage-reclaim.ledger'; -import { - storageKeysIn, - type StorageKeyOptions, - type StorageKeyRef, -} from '../../src/nest/storage/storage-reclaim.types'; +import { storageKeysIn, type StorageKeyOptions } from '../../src/nest/storage/storage-reclaim.types'; import type { StorageReleaseService } from '../../src/nest/storage/storage-release.service'; import type { StorageService } from '../../src/nest/storage/storage.service'; import type { ServedCategory } from '../../src/nest/storage/storage.types'; @@ -21,39 +18,25 @@ import type { ServedCategory } from '../../src/nest/storage/storage.types'; import { vi } from 'vitest'; export function releaseNothing() { - const releaseValues = vi.fn(async (): Promise => []); - const releaseIfUnreferenced = vi.fn(async (): Promise => []); - return { releaseValues, releaseIfUnreferenced } as unknown as StorageReleaseService & { - releaseValues: typeof releaseValues; - releaseIfUnreferenced: typeof releaseIfUnreferenced; - }; + const releaseValues = vi.fn(async (): Promise => 0); + return { releaseValues } as unknown as StorageReleaseService & { releaseValues: typeof releaseValues }; } export function releaseDirect(storage: StorageService): StorageReleaseService { - const releaseIfUnreferenced = async (category: ServedCategory, keys: readonly string[]): Promise => { - const present = await Promise.all( - [...new Set(keys)].map(async (key) => - inReclaimScope(key) && (await storage.stat(category, key).catch(() => null)) !== null ? key : null, - ), - ); - const gone = present.filter((key): key is string => key !== null); - await Promise.all(gone.map((key) => storage.delete(category, key))); - return gone; - }; const releaseValues = async ( values: ReadonlyArray, options: StorageKeyOptions = {}, - ): Promise => { - const byCategory = new Map>(); - for (const ref of values.flatMap((value) => storageKeysIn(value, options))) { - byCategory.set(ref.category, (byCategory.get(ref.category) ?? new Set()).add(ref.key)); - } - const released = await Promise.all( - [...byCategory].map(async ([category, keys]) => - (await releaseIfUnreferenced(category as ServedCategory, [...keys])).map((key) => ({ category, key })), + ): Promise => { + const refs = values.flatMap((value) => storageKeysIn(value, options)).filter((ref) => inReclaimScope(ref.key)); + const unique = [...new Map(refs.map((ref) => [`${ref.category}\n${ref.key}`, ref])).values()]; + const present = await Promise.all( + unique.map(async (ref) => + (await storage.stat(ref.category as ServedCategory, ref.key).catch(() => null)) !== null ? ref : null, ), ); - return released.flat(); + const gone = present.filter((ref): ref is NonNullable => ref !== null); + await Promise.all(gone.map((ref) => storage.delete(ref.category as ServedCategory, ref.key))); + return gone.length; }; - return { releaseValues, releaseIfUnreferenced } as unknown as StorageReleaseService; + return { releaseValues } as unknown as StorageReleaseService; } diff --git a/server/tests/integration/storage-release.test.ts b/server/tests/integration/storage-release.test.ts index 09f85a8fe..0ef9d9def 100644 --- a/server/tests/integration/storage-release.test.ts +++ b/server/tests/integration/storage-release.test.ts @@ -1,19 +1,28 @@ /** - * The immediate release over the full app (#2924): what a user's replace or - * delete leaves unnamed is gone right after the request, through every owner's - * real reference source, while an object another row still names (a trip - * copy, a journey, a saved place) stays. The objects are written straight into - * the test run's uploads folder under random names and taken back afterwards. - * RELI-001 to RELI-006. + * The deferred release over the full app (#2924): what a user's replace or + * delete leaves unnamed is queued by the request and left in place, then + * deleted by the release job once it is due, through every owner's real + * reference source, while an object another row still names (a trip copy, a + * journey, a saved place, a value an undo wrote back) stays. The objects are + * written straight into the test run's uploads folder under random names and + * taken back afterwards, the owner markers the job wrote with them. + * RELI-001 to RELI-009. */ import { buildApp } from '../../src/bootstrap'; +import { Addons } from '../../src/db/entities/Addons.entity'; +import { StorageReleaseQueue } from '../../src/db/entities/StorageReleaseQueue.entity'; import { Users } from '../../src/db/entities/Users.entity'; +import { withRequestContext } from '../../src/nest/database/request-context'; +import { StorageInstallService } from '../../src/nest/storage/storage-install.service'; +import { STORAGE_RECLAIMABLE_CATEGORIES, storageOwnerMarkerKey } from '../../src/nest/storage/storage-reclaim.types'; +import { RELEASE_DELAY_MS, StorageReleaseService } from '../../src/nest/storage/storage-release.service'; +import { StorageService } from '../../src/nest/storage/storage.service'; import { authCookie } from '../helpers/auth'; import { makeCollection, makeCollectionPlace } from '../helpers/factories/collections'; import type { FactoryOrm } from '../helpers/factories/context'; import { makeJourney } from '../helpers/factories/journeys'; import { makePlace } from '../helpers/factories/places'; -import { updateRows } from '../helpers/factories/rows'; +import { countRows, updateRows, upsertRow } from '../helpers/factories/rows'; import { makeTrip } from '../helpers/factories/trips'; import { makeUser } from '../helpers/factories/users'; import { MikroORM } from '@mikro-orm/core'; @@ -47,6 +56,15 @@ let nestApp: INestApplication; let app: Application; let orm: FactoryOrm; +/** One tick of the release job at `at` (by default once everything queued so far is due). */ +async function tick(at: Date = new Date(Date.now() + RELEASE_DELAY_MS + 60_000)) { + return await withRequestContext(nestApp.get(MikroORM), () => nestApp.get(StorageReleaseService).processDue(at)); +} + +async function queued(category: string, key: string): Promise { + return await countRows(orm, StorageReleaseQueue, { category, key }); +} + beforeAll(async () => { nestApp = await buildApp(); app = nestApp.getHttpAdapter().getInstance(); @@ -54,11 +72,21 @@ beforeAll(async () => { }); afterAll(async () => { for (const file of written) fs.rmSync(file, { force: true }); + // The uploads folder is the test run's shared one: take back the owner markers the job wrote. + const installId = nestApp.get(StorageInstallService).installId(); + if (installId) { + const storage = nestApp.get(StorageService); + await Promise.all( + STORAGE_RECLAIMABLE_CATEGORIES.map((category) => + storage.delete(category, storageOwnerMarkerKey(installId)).catch(() => undefined), + ), + ); + } await nestApp.close(); }); -describe('the immediate release of what a change left unnamed', () => { - it('RELI-001: a replaced trip cover goes at once, but not while a trip copy still shows it', async () => { +describe('the deferred release of what a change left unnamed', () => { + it('RELI-001: a replaced trip cover waits in the queue, then goes, but not while a trip copy still shows it', async () => { const { user } = await makeUser(orm); const old = stored('covers'); const trip = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${old}` }); @@ -70,18 +98,24 @@ describe('the immediate release of what a change left unnamed', () => { .set('Cookie', authCookie(user.id)) .send({ cover_image: `/uploads/covers/${next}` }); expect(first.status).toBe(200); + expect(await queued('covers', old)).toBe(1); + await tick(); expect(exists('covers', old)).toBe(true); + expect(await queued('covers', old)).toBe(0); const second = await request(app) .put(`/api/trips/${copy.id}`) .set('Cookie', authCookie(user.id)) .send({ cover_image: `/uploads/covers/${next}` }); expect(second.status).toBe(200); + // The request itself deletes nothing. + expect(exists('covers', old)).toBe(true); + await tick(); expect(exists('covers', old)).toBe(false); expect(exists('covers', next)).toBe(true); }); - it('RELI-002: a deleted trip takes its cover along unless a journey inherited it', async () => { + it('RELI-002: a deleted trip takes its cover along a day later, unless a journey inherited it', async () => { const { user } = await makeUser(orm); const alone = stored('covers'); const inherited = stored('covers'); @@ -92,11 +126,13 @@ describe('the immediate release of what a change left unnamed', () => { const deleteTrip = (id: number) => request(app).delete(`/api/trips/${id}`).set('Cookie', authCookie(user.id)); expect((await deleteTrip(tripA.id)).status).toBe(200); expect((await deleteTrip(tripB.id)).status).toBe(200); + expect(exists('covers', alone)).toBe(true); + await tick(); expect(exists('covers', alone)).toBe(false); expect(exists('covers', inherited)).toBe(true); }); - it('RELI-003: a replaced place image goes, one a saved place still names stays', async () => { + it('RELI-003: a replaced place image goes once due, one a saved place still names stays', async () => { const { user } = await makeUser(orm); const trip = await makeTrip(orm, user.id); const own = stored('places'); @@ -115,11 +151,12 @@ describe('the immediate release of what a change left unnamed', () => { .send({ image_url: null }); expect((await clearImage(placeA.id)).status).toBe(200); expect((await clearImage(placeB.id)).status).toBe(200); + await tick(); expect(exists('places', own)).toBe(false); expect(exists('places', saved)).toBe(true); }); - it('RELI-004: a deleted place takes its image along', async () => { + it('RELI-004: a deleted place takes its image along once due, not before', async () => { const { user } = await makeUser(orm); const trip = await makeTrip(orm, user.id); const image = stored('places'); @@ -128,10 +165,15 @@ describe('the immediate release of what a change left unnamed', () => { .delete(`/api/trips/${trip.id}/places/${place.id}`) .set('Cookie', authCookie(user.id)); expect(res.status).toBe(200); + await tick(new Date()); + expect(exists('places', image)).toBe(true); + expect(await queued('places', image)).toBe(1); + await tick(); expect(exists('places', image)).toBe(false); + expect(await queued('places', image)).toBe(0); }); - it('RELI-005: a deleted avatar goes; an avatar another account still names stays', async () => { + it('RELI-005: a deleted avatar goes once due; an avatar another account still names stays', async () => { const { user } = await makeUser(orm); const { user: other } = await makeUser(orm); const own = stored('avatars', '.png'); @@ -140,10 +182,12 @@ describe('the immediate release of what a change left unnamed', () => { await updateRows(orm, Users, { id: other.id }, { avatar: shared }); const res = await request(app).delete('/api/auth/avatar').set('Cookie', authCookie(user.id)); expect(res.status).toBe(200); + await tick(); expect(exists('avatars', own)).toBe(false); await updateRows(orm, Users, { id: user.id }, { avatar: shared }); await request(app).delete('/api/auth/avatar').set('Cookie', authCookie(user.id)); + await tick(); expect(exists('avatars', shared)).toBe(true); }); @@ -155,5 +199,63 @@ describe('the immediate release of what a change left unnamed', () => { .set('Cookie', authCookie(user.id)) .send({ cover_image: 'https://images.example.test/cover.jpg' }); expect(res.status).toBe(200); + const tooLong = await request(app) + .put(`/api/trips/${trip.id}`) + .set('Cookie', authCookie(user.id)) + .send({ cover_image: `/uploads/covers/${'a'.repeat(2048)}` }); + expect(tooLong.status).toBe(400); + }); + + it('RELI-007: a cover an undo writes back before it is due stays, and its row goes', async () => { + const { user } = await makeUser(orm); + const old = stored('covers'); + const trip = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${old}` }); + const put = (cover: string | null) => + request(app).put(`/api/trips/${trip.id}`).set('Cookie', authCookie(user.id)).send({ cover_image: cover }); + expect((await put(null)).status).toBe(200); + expect(await queued('covers', old)).toBe(1); + expect((await put(`/uploads/covers/${old}`)).status).toBe(200); + await tick(); + expect(exists('covers', old)).toBe(true); + expect(await queued('covers', old)).toBe(0); + }); + + it('RELI-008: a PUT that sends the cover a trip already has queues nothing', async () => { + const { user } = await makeUser(orm); + const same = stored('covers'); + const trip = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${same}` }); + const res = await request(app) + .put(`/api/trips/${trip.id}`) + .set('Cookie', authCookie(user.id)) + .send({ cover_image: `/uploads/covers/${same}` }); + expect(res.status).toBe(200); + expect(await queued('covers', same)).toBe(0); + }); + + it('RELI-009: a replaced journey cover is queued, never deleted by the request; an unchanged one queues nothing', async () => { + await upsertRow(orm, Addons, { + id: 'journey', + name: 'Journey', + description: 'Travel journal', + type: 'global', + icon: 'Compass', + enabled: true, + sort_order: 35, + }); + const { user } = await makeUser(orm); + const old = stored('covers'); + const journey = await makeJourney(orm, user.id, { cover_image: `covers/${old}` }); + const patch = (cover: string) => + request(app).patch(`/api/journeys/${journey.id}`).set('Cookie', authCookie(user.id)).send({ cover_image: cover }); + expect((await patch(`covers/${old}`)).status).toBe(200); + expect(await queued('covers', old)).toBe(0); + const next = stored('covers'); + expect((await patch(`covers/${next}`)).status).toBe(200); + expect(await queued('covers', old)).toBe(1); + // What the assistant's undo writes back within the window keeps its object. + expect((await patch(`covers/${old}`)).status).toBe(200); + await tick(); + expect(exists('covers', old)).toBe(true); + expect(exists('covers', next)).toBe(false); }); }); diff --git a/server/tests/unit/app-config/data-paths.test.ts b/server/tests/unit/app-config/data-paths.test.ts index e545878f4..9ebdd44dc 100644 --- a/server/tests/unit/app-config/data-paths.test.ts +++ b/server/tests/unit/app-config/data-paths.test.ts @@ -41,6 +41,7 @@ describe('resolveDataPaths', () => { logsDir: path.join('/srv/trek', 'data', 'logs'), jwtSecretFile: path.join('/srv/trek', 'data', '.jwt_secret'), encryptionKeyFile: path.join('/srv/trek', 'data', '.encryption_key'), + installIdFile: path.join('/srv/trek', 'data', '.trek-install-id'), dbFile: path.join('/srv/trek', 'data', 'travel.db'), }); }); diff --git a/server/tests/unit/db/entities-index.test.ts b/server/tests/unit/db/entities-index.test.ts index c63c27a61..7536689be 100644 --- a/server/tests/unit/db/entities-index.test.ts +++ b/server/tests/unit/db/entities-index.test.ts @@ -63,8 +63,8 @@ describe('ALL_ENTITIES', () => { * `entity-schema-parity.test.ts` can't see — it iterates the metadata map, * which de-duplicates by class name). */ - it('ENT-003: exactly 166 entities, every class name unique', () => { - expect(ALL_ENTITIES.length).toBe(166); + it('ENT-003: exactly 167 entities, every class name unique', () => { + expect(ALL_ENTITIES.length).toBe(167); const names = ALL_ENTITIES.map((s) => s.meta.className); expect(new Set(names).size).toBe(names.length); }); diff --git a/server/tests/unit/db/entity-class-fields.test.ts b/server/tests/unit/db/entity-class-fields.test.ts index c81e6d633..8f4a790ea 100644 --- a/server/tests/unit/db/entity-class-fields.test.ts +++ b/server/tests/unit/db/entity-class-fields.test.ts @@ -60,7 +60,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; // + assistant_attachments: id, thread_id, user_id, trip_id, name, mime, size, sha256, storage_key, text_status // (an attached file is only ever written whole, by the turn it was attached in). // + storage_reclaim_marks: category, key, first_seen_at, size (a mark is only ever written whole, by the reclaim pass). -const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 608; +// + storage_release_queue: category, key, due_at (a queued release is only ever written whole, by the release queue). +const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 611; describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => { it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => { diff --git a/server/tests/unit/db/legacy-baseline.test.ts b/server/tests/unit/db/legacy-baseline.test.ts index c18b3ce87..8463f5d9f 100644 --- a/server/tests/unit/db/legacy-baseline.test.ts +++ b/server/tests/unit/db/legacy-baseline.test.ts @@ -101,6 +101,7 @@ const UNNUMBERED = [ 'Migration20200101046310_an_admin_can_run_a_plugin_outside_its_trek_range', 'Migration20200101046320_a_place_knows_where_its_details_came_from', 'Migration20200101046330_an_object_no_row_names_is_marked_before_it_goes', + 'Migration20200101046340_a_released_object_outlasts_every_undo_before_it_goes', ]; // The last step the positional runner ever had; planLegacyBaseline refuses a // schema_version past it. diff --git a/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts b/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts new file mode 100644 index 000000000..ef68e326a --- /dev/null +++ b/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts @@ -0,0 +1,91 @@ +/** + * StorageReleaseQueueRepository: the objects a delete or a replace left + * unnamed, each with the time it may go. RELQ-001 to RELQ-005. + */ +import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity'; +import type { StorageReleaseQueueRepository } from '../../../../src/db/repositories/StorageReleaseQueue.repository'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { deleteRows } from '../../../helpers/factories/rows'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const testDb = createSnapshotTestDb(); +let t: TestOrm; +let queue: StorageReleaseQueueRepository; + +beforeAll(async () => { + t = await createTestOrm(testDb); + queue = t.repo(StorageReleaseQueue); +}); +beforeEach(async () => { + await deleteRows(t, StorageReleaseQueue); + t.clear(); +}); +afterAll(async () => { + await t.close(); + testDb.close(); +}); + +describe('StorageReleaseQueueRepository', () => { + it('RELQ-001: queues keys per category and answers the due ones, the longest waiting first, up to a limit', async () => { + await queue.enqueue([{ category: 'covers', key: 'b.jpg' }], '2026-01-02 00:00:00'); + await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00'); + await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-01-01 00:00:00'); + await queue.enqueue([{ category: 'covers', key: 'later.jpg' }], '2026-02-01 00:00:00'); + expect(await queue.size()).toBe(4); + expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 10)).toEqual([ + { category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' }, + { category: 'covers', key: 'b.jpg', due_at: '2026-01-02 00:00:00' }, + ]); + expect(await queue.dueIn(['covers', 'places'], '2026-01-15 00:00:00', 1)).toHaveLength(1); + expect(await queue.dueIn([], '2026-01-15 00:00:00', 10)).toEqual([]); + expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 0)).toEqual([]); + }); + + it('RELQ-002: a key queued again waits from the later time, and an empty batch writes nothing', async () => { + await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00'); + await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-03-01 00:00:00'); + await queue.enqueue([], '2026-04-01 00:00:00'); + expect(await queue.size()).toBe(1); + expect(await queue.dueIn(['covers'], '2026-02-01 00:00:00', 10)).toEqual([]); + expect(await queue.dueIn(['covers'], '2026-03-01 00:00:00', 10)).toHaveLength(1); + }); + + it('RELQ-003: answers the categories holding a due row, each once', async () => { + await queue.enqueue( + [ + { category: 'covers', key: 'a.jpg' }, + { category: 'covers', key: 'b.jpg' }, + ], + '2026-01-01 00:00:00', + ); + await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-05-01 00:00:00'); + expect(await queue.dueCategories('2026-02-01 00:00:00')).toEqual(['covers']); + expect((await queue.dueCategories('2026-06-01 00:00:00')).sort()).toEqual(['covers', 'places']); + }); + + it('RELQ-004: drops the rows of the keys named, in their category only, and the rows due before a time', async () => { + await queue.enqueue( + [ + { category: 'covers', key: 'a.jpg' }, + { category: 'covers', key: 'b.jpg' }, + ], + '2026-01-01 00:00:00', + ); + await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-03-01 00:00:00'); + expect(await queue.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1); + expect(await queue.deleteKeys('covers', [])).toBe(0); + expect(await queue.deleteDueBefore('2026-02-01 00:00:00')).toBe(1); + expect(await queue.dueIn(['places'], '2026-03-01 00:00:00', 10)).toEqual([ + { category: 'places', key: 'a.jpg', due_at: '2026-03-01 00:00:00' }, + ]); + }); + + it('RELQ-005: drops every row, for a database another installation brought', async () => { + await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00'); + await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-01-01 00:00:00'); + expect(await queue.deleteAll()).toBe(2); + expect(await queue.size()).toBe(0); + }); +}); diff --git a/server/tests/unit/nest/backup.impl.test.ts b/server/tests/unit/nest/backup.impl.test.ts index ddefb1562..e808eeb5f 100644 --- a/server/tests/unit/nest/backup.impl.test.ts +++ b/server/tests/unit/nest/backup.impl.test.ts @@ -731,6 +731,18 @@ describe('BACKUP-036 createBackup', () => { expect(names.some((n) => n?.includes('trek-owner-'))).toBe(false); }); + it('BACKUP-036k — never bundles the install id file, so a restore cannot make another install look like this one', async () => { + // The key files both exist here: the archive still takes only the ones it names. + fsMock.existsSync.mockImplementation((p: string) => /\.(encryption_key|trek-install-id)$/.test(String(p))); + setupArchiveSuccess(); + await createBackup(deps(stubStorage({ stat: statOf(1024) }))); + const files = archiverInstanceMock.file.mock.calls.map((c) => [String(c[0]), c[1]?.name as string]); + expect(files.some(([from, name]) => from.includes('.trek-install-id') || name?.includes('.trek-install-id'))).toBe( + false, + ); + expect(archiverInstanceMock.glob).not.toHaveBeenCalled(); + }); + it('BACKUP-036h — only category prefixes are ever archived; backups/ and restore-* are structurally out (issue #1358)', async () => { // The uploads/** glob is gone: even when data and uploads map to the same // directory, enumeration only ever walks the six archived categories, so diff --git a/server/tests/unit/nest/journey.controller.test.ts b/server/tests/unit/nest/journey.controller.test.ts index bcf81fe4d..4949fa9a7 100644 --- a/server/tests/unit/nest/journey.controller.test.ts +++ b/server/tests/unit/nest/journey.controller.test.ts @@ -8,8 +8,6 @@ import type { JourneyPhotoLinksService } from '../../../src/nest/journey/journey import { JourneyPublicController } from '../../../src/nest/journey/journey-public.controller'; import { JourneyController } from '../../../src/nest/journey/journey.controller'; import type { JourneyService } from '../../../src/nest/journey/journey.service'; -import { journeyThumbName } from '../../../src/nest/memories/thumbnail.service'; -import type { StorageKeyRef } from '../../../src/nest/storage/storage-reclaim.types'; import type { StorageReleaseService } from '../../../src/nest/storage/storage-release.service'; import type { StorageService } from '../../../src/nest/storage/storage.service'; import type { User } from '../../../src/types'; @@ -47,14 +45,12 @@ const captureBackfillStub = { } as unknown as JourneyPhotoCaptureService; /** - * What a deleted photo hands on. By default every `journey/` it names - * goes, as the real release answers when no other row names them; a case - * about a shared object answers what it keeps. + * What a deleted photo hands on: the release queues every `journey/` it + * names, and answers how many. */ -const releaseValues = vi.fn(async (values: ReadonlyArray): Promise => - values - .filter((value): value is string => typeof value === 'string' && value.startsWith('journey/')) - .map((value) => ({ category: 'journey', key: value.slice('journey/'.length) })), +const releaseValues = vi.fn( + async (values: ReadonlyArray): Promise => + values.filter((value) => typeof value === 'string' && value.startsWith('journey/')).length, ); const releaseStub = { releaseValues } as unknown as StorageReleaseService; @@ -721,7 +717,7 @@ describe('JourneyController', () => { expect(updatePhoto).toHaveBeenCalledWith(7, 1, { caption: 'x' }, 'sock-1'); }); - it('DELETE photo releases its file and poster once the row is gone, and the thumb of a file that went', async () => { + it('DELETE photo queues its file and poster for release once the row is gone, and deletes nothing itself', async () => { (storageStub.delete as ReturnType).mockClear(); releaseValues.mockClear(); const deletePhoto = vi.fn().mockReturnValue({ id: 7, file_path: 'journey/a.jpg', thumbnail_path: 'journey/p.jpg' }); @@ -730,16 +726,17 @@ describe('JourneyController', () => { }); expect(deletePhoto).toHaveBeenCalledWith(7, 1); expect(releaseValues).toHaveBeenCalledWith(['journey/a.jpg', 'journey/p.jpg'], { bare: 'journey' }); - expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/a.jpg')); + // Neither the file nor its thumb goes inside the request: the queue waits out every undo, and the + // nightly thumbnail sweep drops a thumb no photo row accounts for. + expect(storageStub.delete).not.toHaveBeenCalled(); }); - it('DELETE photo keeps the thumb of a file another row still names, and of a path outside journey/', async () => { + it('DELETE photo outside journey/ hands the path on as it is and deletes nothing', async () => { (storageStub.delete as ReturnType).mockClear(); - releaseValues.mockResolvedValueOnce([]); - const shared = vi.fn().mockReturnValue({ id: 8, file_path: 'journey/shared.jpg', thumbnail_path: null }); - await ctl(svc({ deletePhoto: shared } as Partial)).deletePhoto(user, '8'); + releaseValues.mockClear(); const outside = vi.fn().mockReturnValue({ id: 9, file_path: 'elsewhere/x.jpg' }); await ctl(svc({ deletePhoto: outside } as Partial)).deletePhoto(user, '9'); + expect(releaseValues).toHaveBeenCalledWith(['elsewhere/x.jpg', undefined], { bare: 'journey' }); expect(storageStub.delete).not.toHaveBeenCalled(); }); @@ -770,7 +767,7 @@ describe('JourneyController', () => { expect(await ctl(svc()).galleryProviderPhotos(user, '9', { provider: 'immich', asset_id: 'a' })).toEqual({ id: 3 }); }); - it('DELETE gallery photo: 404, then releases the video, its poster and its thumb', async () => { + it('DELETE gallery photo: 404, then queues the video and its poster for release', async () => { expect( await thrownAsync(() => ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue(null) } as Partial)).deleteGalleryPhoto( @@ -789,12 +786,7 @@ describe('JourneyController', () => { ).toBeUndefined(); expect(deleteGalleryPhoto).toHaveBeenCalledWith(10, 1); expect(releaseValues).toHaveBeenCalledWith(['journey/clip.mp4', 'journey/poster.jpg'], { bare: 'journey' }); - expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/clip.mp4')); - // A thumb delete that finds nothing is swallowed. - (storageStub.delete as ReturnType).mockRejectedValueOnce(new Error('ENOENT')); - await expect( - ctl(svc({ deleteGalleryPhoto } as Partial)).deleteGalleryPhoto(user, '10'), - ).resolves.toBeUndefined(); + expect(storageStub.delete).not.toHaveBeenCalled(); }); it('PATCH /:id returns the updated journey on success', async () => { diff --git a/server/tests/unit/nest/storage-reference-sources.test.ts b/server/tests/unit/nest/storage-reference-sources.test.ts index 67cc1dbb5..1411536aa 100644 --- a/server/tests/unit/nest/storage-reference-sources.test.ts +++ b/server/tests/unit/nest/storage-reference-sources.test.ts @@ -7,7 +7,7 @@ * `namedKeys`, every key of a category its rows name (one read, for a pass), * and `referenced`, a few keys asked fresh and narrowed to the rows holding * their plain runs (the re-check before a delete, and a release). - * RSRC-001 to RSRC-014. + * RSRC-001 to RSRC-017. */ import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity'; import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity'; @@ -20,6 +20,7 @@ import { PhotosStorageRefsService } from '../../../src/nest/photos/photos-storag import { PlaceContribStorageRefsService } from '../../../src/nest/place-contrib/place-contrib-storage-refs.service'; import { PlacesStorageRefsService } from '../../../src/nest/places/places-storage-refs.service'; import { StorageReclaimRegistryService } from '../../../src/nest/storage/storage-reclaim-registry.service'; +import { likeAny, MAX_LIKE_TERMS } from '../../../src/nest/storage/storage-reclaim-source.service'; import { STORAGE_RECLAIMABLE_CATEGORIES, STORAGE_REFERENCE_SOURCE_IDS, @@ -233,4 +234,31 @@ describe('reference sources', () => { const named = await files.namedKeys('files'); for (const name of names) expect(named.has(name.toLowerCase())).toBe(true); }); + + it('RSRC-015: a question about many keys is split into reads of at most MAX_LIKE_TERMS terms each', async () => { + const ids = Array.from({ length: 60 }, (_unused, i) => `9d8c7b6a-5f4e-4d3c-8b2a-${String(i).padStart(12, '0')}`); + await Promise.all(ids.slice(0, 30).map((id) => makePlace(orm, tripId, { image_url: `/uploads/places/${id}.jpg` }))); + const keys = ids.map((id) => `${id}.jpg`); + expect(await asked(t.get(PlacesStorageRefsService), 'places', keys)).toEqual(keys.slice(0, 30).sort()); + // The photos source reads two columns, still within the bound. + expect(await asked(t.get(PhotosStorageRefsService), 'journey', keys)).toEqual([]); + }); + + it('RSRC-016: a filter past MAX_LIKE_TERMS terms is refused before it reaches SQLite', () => { + const runs = Array.from({ length: MAX_LIKE_TERMS + 1 }, (_unused, i) => `run-${String(i).padStart(8, '0')}`); + expect(() => likeAny<{ a: string }>(['a'], runs)).toThrow(/at most/); + expect(() => likeAny<{ a: string; b: string }>(['a', 'b'], runs.slice(0, 25))).not.toThrow(); + expect(() => likeAny<{ a: string; b: string }>(['a', 'b'], runs.slice(0, 26))).toThrow(/at most/); + }); + + it('RSRC-017: namedKeysByCategory reads once and answers every category the source claims, none it does not', async () => { + await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-once-a.jpg' }); + await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-once-b.jpg' }); + const answer = await t.get(TripsStorageRefsService).namedKeysByCategory(['covers', 'places', 'photos']); + expect(answer.get('covers')?.has('rsrc-once-a.jpg')).toBe(true); + expect(answer.get('covers')?.has('rsrc-once-b.jpg')).toBe(false); + expect(answer.get('places')?.has('rsrc-once-b.jpg')).toBe(true); + expect(answer.has('photos')).toBe(false); + expect(await t.get(FilesStorageRefsService).namedKeysByCategory(['covers'])).toEqual(new Map()); + }); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts index 7db88202f..6400d6e20 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.controller.test.ts @@ -1,7 +1,7 @@ /** * StorageReclaimController over a stub service: what it forwards, what it * audits, and that a refusal of the service reaches the client unchanged. - * RCLC-001 to RCLC-006. + * RCLC-001 to RCLC-007. */ import type { AuditService } from '../../../../src/nest/audit/audit.service'; import { DomainError } from '../../../../src/nest/common/domain-error'; @@ -52,7 +52,7 @@ function makeController(over: Partial state({ deleteEnabled: enabled })), - start: vi.fn(), + start: vi.fn(async () => undefined), confirm: vi.fn(async () => state({ lastRun: report() })), ...over, } as unknown as StorageReclaimService; @@ -89,19 +89,18 @@ describe('StorageReclaimController', () => { code: 'reclaim_busy', }); const { controller, writeAudit } = makeController({ - start: vi.fn(() => { - throw refusal; - }), + start: vi.fn().mockRejectedValue(refusal), }); await expect(controller.run(user, req)).rejects.toBe(refusal); expect(writeAudit).not.toHaveBeenCalled(); }); - it('RCLC-005: POST confirm forwards the category, answers the state and audits the confirmed count', async () => { + it('RCLC-005: POST confirm forwards the category and the count shown, answers the state and audits that count', async () => { const { controller, service, writeAudit } = makeController(); - const answer = await controller.confirm(user, { category: 'covers' } as StorageReclaimConfirmDto, req); + const body = { category: 'covers', unreferenced: 4 } as StorageReclaimConfirmDto; + const answer = await controller.confirm(user, body, req); expect(answer.lastRun?.categories.covers.confirmationRequired).toBe(true); - expect(service.confirm).toHaveBeenCalledWith('covers'); + expect(service.confirm).toHaveBeenCalledWith('covers', 4); expect(writeAudit).toHaveBeenCalledWith( expect.objectContaining({ action: 'admin.storage_reclaim_confirm', @@ -115,9 +114,18 @@ describe('StorageReclaimController', () => { code: 'reclaim_nothing_to_confirm', }); const { controller, writeAudit } = makeController({ confirm: vi.fn().mockRejectedValue(refusal) }); - await expect(controller.confirm(user, { category: 'files' } as StorageReclaimConfirmDto, req)).rejects.toBe( - refusal, - ); + const body = { category: 'files', unreferenced: 3 } as StorageReclaimConfirmDto; + await expect(controller.confirm(user, body, req)).rejects.toBe(refusal); + expect(writeAudit).not.toHaveBeenCalled(); + }); + + it('RCLC-007: a count the last report no longer holds propagates its stale 409 and writes no audit row', async () => { + const refusal = new DomainError(409, 'the last check found another count since, reload and confirm again', { + code: 'reclaim_confirm_stale', + }); + const { controller, writeAudit } = makeController({ confirm: vi.fn().mockRejectedValue(refusal) }); + const body = { category: 'covers', unreferenced: 400 } as StorageReclaimConfirmDto; + await expect(controller.confirm(user, body, req)).rejects.toBe(refusal); expect(writeAudit).not.toHaveBeenCalled(); }); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts index 5fd453179..f49408e47 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts @@ -217,6 +217,7 @@ describe('StorageReclaimRegistryService', () => { id, claims: [{ category: 'covers', prefix: '' }], namedKeys: async () => new Set(), + namedKeysByCategory: async () => new Map(), referenced: async () => new Set(), }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts index a86a5f974..35f0a5327 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts @@ -1,24 +1,31 @@ /** * The storage reclaim pass (#2924) over a real StorageModule: a real registry - * and LocalDriver on temp folders, the marks table in the snapshot db, and - * fake reference sources standing in for the owner domains. The pass marks - * first, deletes a week later only with the switch on and only after asking - * every source about its batch again. A skip keeps the marks; a share over a - * fifth waits for an admin's confirmation; an owner marker of another install - * keeps its hands off; a restore resets the switch, the marks and the reports - * but keeps this install's id. RCLS-001 to RCLS-036. + * and LocalDriver on temp folders, the marks table in the snapshot db, the + * install id file in a temp data dir, and fake reference sources standing in + * for the owner domains. The pass reads every source once, marks first, + * deletes a week later only with the switch on and only after reading every + * source again, fresh, right before the batch. A skip keeps the marks; a share + * over a fifth waits for an admin's confirmation of the count they saw, which + * covers a margin for two weeks and is used up once the share is back under; + * an owner marker of another install keeps its hands off; a restore, or a + * database naming another install, resets the switch, the marks, the queue + * and the reports but keeps this install's id. RCLS-001 to RCLS-043. */ import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; import { AuditLog } from '../../../../src/db/entities/AuditLog.entity'; import { StorageReclaimMarks } from '../../../../src/db/entities/StorageReclaimMarks.entity'; +import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity'; import { StorageReclaimMarksRepository } from '../../../../src/db/repositories/StorageReclaimMarks.repository'; import { dbNow } from '../../../../src/db/types'; +import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; import { logError } from '../../../../src/nest/audit/audit-log.logger'; import { DomainError } from '../../../../src/nest/common/domain-error'; import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service'; import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; +import { StorageInstallService } from '../../../../src/nest/storage/storage-install.service'; import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; +import { StorageReclaimGuardService } from '../../../../src/nest/storage/storage-reclaim-guard.service'; import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service'; import { StorageReclaimService } from '../../../../src/nest/storage/storage-reclaim.service'; import { @@ -29,6 +36,7 @@ import { type StorageReferenceSource, type StorageReferenceSourceId, } from '../../../../src/nest/storage/storage-reclaim.types'; +import { STORAGE_RELEASE_JOB } from '../../../../src/nest/storage/storage-release.service'; import { StorageModule } from '../../../../src/nest/storage/storage.module'; import { StorageService } from '../../../../src/nest/storage/storage.service'; import type { ServedCategory } from '../../../../src/nest/storage/storage.types'; @@ -58,6 +66,7 @@ let t: TestModule; let orm: TestOrm; let uploadsRoot: string; let backupsRoot: string; +let dataRoot: string; let reclaim: StorageReclaimService; let storage: StorageService; let jobs: StorageJobsService; @@ -68,6 +77,8 @@ let installId: string; let named: Map>; /** Called on every question a source is asked, before it answers: `keys` is null for a whole-category read. */ let onAsk: ((category: ServedCategory, keys: readonly string[] | null) => void | Promise) | null; +/** Called when a source is read once for the whole pass, before it answers. */ +let onRead: ((id: StorageReferenceSourceId) => void | Promise) | null; /** The keys the fake sources name in `category`, created empty on first use. */ function namedIn(category: ServedCategory): Set { @@ -82,12 +93,22 @@ function namedIn(category: ServedCategory): Set { function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource { const categories: readonly ServedCategory[] = id === 'auth' ? ['avatars'] : id === 'files' ? ['files'] : STORAGE_RECLAIMABLE_CATEGORIES; + const lowered = (category: ServedCategory) => + new Set([...(named.get(category) ?? [])].map((key) => key.toLowerCase())); return { id, claims: categories.map((category): StorageReferenceClaim => ({ category, prefix: '' })), async namedKeys(category) { await onAsk?.(category, null); - return new Set([...(named.get(category) ?? [])].map((key) => key.toLowerCase())); + return lowered(category); + }, + async namedKeysByCategory(asked) { + await onRead?.(id); + return new Map( + asked + .filter((category) => categories.includes(category)) + .map((category): [ServedCategory, Set] => [category, lowered(category)]), + ); }, async referenced(category, keys) { await onAsk?.(category, keys); @@ -127,16 +148,6 @@ async function marksOf(category: string): Promise a.key.localeCompare(b.key)); } -/** What a synchronous call threw, or null. */ -function thrownBy(call: () => unknown): unknown { - try { - call(); - } catch (err) { - return err; - } - return null; -} - async function markWeekOld(category: ServedCategory, keys: string[]): Promise { await orm.repo(StorageReclaimMarks).insertMissing( category, @@ -148,6 +159,7 @@ async function markWeekOld(category: ServedCategory, keys: string[]): Promise { uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-uploads-')); backupsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-backups-')); + dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-reclaim-data-')); orm = await createTestOrm(db); await setAppSetting(orm, 'storage.backends', backends()); t = await createTestModule({ @@ -155,6 +167,7 @@ beforeAll(async () => { imports: [StorageModule], init: true, overrides: [ + { provide: DataPathsService, useValue: { ...new DataPathsService(), installIdFile: path.join(dataRoot, 'id') } }, { provide: DatabaseLifecycle, useValue: { @@ -170,7 +183,7 @@ beforeAll(async () => { jobs = t.get(StorageJobsService); references = t.get(StorageReclaimRegistryService); for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id)); - installId = (await readAppSetting(orm, 'install_id')) ?? ''; + installId = t.get(StorageInstallService).installId() ?? ''; }); beforeEach(async () => { @@ -180,10 +193,13 @@ beforeEach(async () => { if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true }); } await deleteRows(orm, StorageReclaimMarks); + await deleteRows(orm, StorageReleaseQueue); await deleteRows(orm, AppSettings, { key: { $like: 'storage.reclaim%' } }); + t.get(StorageReclaimGuardService).forget(); await deleteRows(orm, AuditLog); named = new Map(); onAsk = null; + onRead = null; }); afterAll(async () => { @@ -192,6 +208,7 @@ afterAll(async () => { db.close(); fs.rmSync(uploadsRoot, { recursive: true, force: true }); fs.rmSync(backupsRoot, { recursive: true, force: true }); + fs.rmSync(dataRoot, { recursive: true, force: true }); }); const T0 = new Date(Date.now() + 2 * DAY); @@ -206,6 +223,8 @@ async function markedWeekOld(orphans: string[], kept = 4): Promise { describe('StorageReclaimService: mark, then sweep', () => { it('RCLS-001: a first pass reports and marks; the report is persisted and both states carry it', async () => { expect(installId).toMatch(/^[0-9a-f-]{36}$/); + expect(fs.readFileSync(path.join(dataRoot, 'id'), 'utf8').trim()).toBe(installId); + expect(await readAppSetting(orm, 'install_id')).toBe(installId); await seedCovers(['orphan.jpg']); const report = await reclaim.run({ now: T0 }); expect(report.mode).toBe('report'); @@ -273,10 +292,10 @@ describe('StorageReclaimService: mark, then sweep', () => { expect(await marksOf('covers')).toEqual([{ key: 'orphan.jpg', first_seen_at: dbNow(WEEK_LATER) }]); }); - it('RCLS-005: the fresh re-check right before the delete keeps an object named in between', async () => { + it('RCLS-005: the fresh read right before the delete keeps an object named in between', async () => { await markedWeekOld(['orphan.jpg']); onAsk = (category, keys) => { - if (category === 'covers' && keys !== null) namedIn('covers').add('orphan.jpg'); + if (category === 'covers' && keys === null) namedIn('covers').add('orphan.jpg'); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.eligible.objects).toBe(1); @@ -337,37 +356,64 @@ describe('StorageReclaimService: a large unreferenced share waits for the admin' ]); }); - it('RCLS-009: an admin confirmation of the shown count lets the next pass delete; a larger share waits again', async () => { + it('RCLS-009: confirming the count shown clears the prompt at once and lets the next pass delete', async () => { await seedCovers(['o1.jpg', 'o2.jpg'], 1); await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); await reclaim.setDeleteEnabled(true); await reclaim.run({ now: WEEK_LATER }); - const confirmed = await reclaim.confirm('covers'); - expect(confirmed.lastRun?.categories.covers.confirmationRequired).toBe(true); - expect(JSON.parse(String(await readAppSetting(orm, 'storage.reclaim_confirmed')))).toEqual({ - installId, - counts: { covers: 2 }, - }); + const before = Date.now(); + const confirmed = await reclaim.confirm('covers', 2); + // The state folds the confirmation in: the prompt goes before the next pass. + expect(confirmed.lastRun?.categories.covers.confirmationRequired).toBe(false); + const stored = JSON.parse(String(await readAppSetting(orm, 'storage.reclaim_confirmed'))); + expect(stored).toEqual({ installId, categories: { covers: { count: 2, confirmedAt: expect.any(Number) } } }); + expect(stored.categories.covers.confirmedAt).toBeGreaterThanOrEqual(before); - await put('covers', 'o3.jpg'); - await put('covers', 'o4.jpg'); - await put('covers', 'o5.jpg'); - await markWeekOld('covers', ['o3.jpg', 'o4.jpg', 'o5.jpg']); - const bigger = await reclaim.run({ now: WEEK_LATER }); - expect(bigger.categories.covers.confirmationRequired).toBe(true); - expect(bigger.categories.covers.deleted.objects).toBe(0); - - await reclaim.confirm('covers'); const after = await reclaim.run({ now: WEEK_LATER }); - expect(after.categories.covers).toMatchObject({ confirmationRequired: false, deleted: { objects: 5, bytes: 25 } }); + expect(after.categories.covers).toMatchObject({ confirmationRequired: false, deleted: { objects: 2, bytes: 10 } }); expect(fs.existsSync(objectPath('covers', 'kept-0.jpg'))).toBe(true); }); + it('RCLS-037: a confirmation covers half as much again, at least a hundred more; a surge past that waits again', async () => { + await seedCovers(['o1.jpg', 'o2.jpg'], 1); + await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); + await reclaim.run({ now: WEEK_LATER }); + await reclaim.confirm('covers', 2); + // A few more orphans than were shown: still covered (2 + 100). + const few = ['o3.jpg', 'o4.jpg', 'o5.jpg']; + await Promise.all(few.map((key) => put('covers', key))); + const covered = await reclaim.run({ now: WEEK_LATER }); + expect(covered.categories.covers).toMatchObject({ unreferenced: { objects: 5 }, confirmationRequired: false }); + // A surge well past the margin waits for a new confirmation. + const surge = Array.from({ length: 105 }, (_unused, i) => `s${i}.jpg`); + await Promise.all(surge.map((key) => put('covers', key))); + const waiting = await reclaim.run({ now: WEEK_LATER }); + expect(waiting.categories.covers).toMatchObject({ unreferenced: { objects: 110 }, confirmationRequired: true }); + }); + + it('RCLS-038: a confirmation runs out after two weeks, and is used up once the share is back under a fifth', async () => { + await seedCovers(['o1.jpg', 'o2.jpg'], 1); + await reclaim.run({ now: WEEK_LATER }); + await reclaim.confirm('covers', 2); + const late = await reclaim.run({ now: new Date(Date.now() + 15 * DAY) }); + expect(late.categories.covers.confirmationRequired).toBe(true); + // The pass that found it run out dropped it. + expect(await readAppSetting(orm, 'storage.reclaim_confirmed')).toBeNull(); + + await reclaim.run({ now: WEEK_LATER }); + await reclaim.confirm('covers', 2); + await Promise.all(Array.from({ length: 20 }, (_unused, i) => put('covers', `named-${i}.jpg`))); + for (let i = 0; i < 20; i += 1) namedIn('covers').add(`named-${i}.jpg`); + const under = await reclaim.run({ now: WEEK_LATER }); + expect(under.categories.covers.confirmationRequired).toBe(false); + expect(await readAppSetting(orm, 'storage.reclaim_confirmed')).toBeNull(); + }); + it('RCLS-010: a confirmation is refused with a code when the last report holds nothing waiting', async () => { - await expect(reclaim.confirm('covers')).rejects.toMatchObject({ code: 'reclaim_nothing_to_confirm' }); + await expect(reclaim.confirm('covers', 1)).rejects.toMatchObject({ code: 'reclaim_nothing_to_confirm' }); await seedCovers(['orphan.jpg']); await reclaim.run({ now: T0 }); - const refusal = await reclaim.confirm('covers').catch((err: unknown) => err); + const refusal = await reclaim.confirm('covers', 1).catch((err: unknown) => err); expect(refusal).toBeInstanceOf(DomainError); expect((refusal as DomainError).getStatus()).toBe(409); expect((refusal as DomainError).getResponse()).toEqual({ @@ -376,18 +422,38 @@ describe('StorageReclaimService: a large unreferenced share waits for the admin' }); }); - it('RCLS-011: a confirmation another install wrote counts for nothing here', async () => { + it('RCLS-039: a count the admin was shown that the last report no longer holds is refused as stale', async () => { + const orphans = Array.from({ length: 300 }, (_unused, i) => `o${i}.jpg`); + await seedCovers(orphans, 1); + await reclaim.run({ now: T0 }); + const stale = await reclaim.confirm('covers', 5).catch((err: unknown) => err); + expect(stale).toBeInstanceOf(DomainError); + expect((stale as DomainError).getStatus()).toBe(409); + expect((stale as DomainError).getResponse()).toMatchObject({ code: 'reclaim_confirm_stale' }); + // Within a tenth and fifty objects of it is the count the admin saw. + await expect(reclaim.confirm('covers', 280)).resolves.toMatchObject({ deleteEnabled: false }); + await expect(reclaim.confirm('covers', 240)).rejects.toMatchObject({ code: 'reclaim_confirm_stale' }); + expect(await readAppSetting(orm, 'storage.reclaim_confirmed')).toContain('"count":280'); + }); + + it('RCLS-011: a confirmation another install wrote counts for nothing here, and an old-format row neither', async () => { await seedCovers(['o1.jpg', 'o2.jpg'], 1); await markWeekOld('covers', ['o1.jpg', 'o2.jpg']); await setAppSetting( orm, 'storage.reclaim_confirmed', - JSON.stringify({ installId: '11111111-2222-4333-8444-555555555555', counts: { covers: 99 } }), + JSON.stringify({ + installId: '11111111-2222-4333-8444-555555555555', + categories: { covers: { count: 99, confirmedAt: Date.now() } }, + }), ); await reclaim.setDeleteEnabled(true); const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.confirmationRequired).toBe(true); expect(report.categories.covers.deleted.objects).toBe(0); + await setAppSetting(orm, 'storage.reclaim_confirmed', JSON.stringify({ installId, counts: { covers: 99 } })); + const old = await reclaim.run({ now: WEEK_LATER }); + expect(old.categories.covers.confirmationRequired).toBe(true); }); }); @@ -405,6 +471,7 @@ describe('StorageReclaimService: categories it may not judge', () => { id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], namedKeys: async () => new Set(), + namedKeysByCategory: async () => new Map(), referenced: async () => new Set(), }; vi.spyOn(references, 'sources').mockReturnValue([...all.filter((source) => source.id !== 'trips'), partialTrips]); @@ -413,21 +480,24 @@ describe('StorageReclaimService: categories it may not judge', () => { expect((await marksOf('covers')).map((mark) => mark.key)).toEqual(['orphan.jpg']); }); - it('RCLS-013: a source answering for a subtree only is not asked about root keys before a delete', async () => { + it('RCLS-013: a source answering for a subtree is read with the rest, and never asked key by key', async () => { await markedWeekOld(['orphan.jpg']); - const subtree = vi.fn(async () => new Set()); + const subtreeRead = vi.fn(async () => new Set()); + const subtreeAsked = vi.fn(async () => new Set()); vi.spyOn(references, 'sources').mockReturnValue([ ...references.sources(), { id: 'trips', claims: [{ category: 'covers', prefix: 'sub/' }], - namedKeys: async () => new Set(), - referenced: subtree, + namedKeys: subtreeRead, + namedKeysByCategory: async () => new Map>([['covers', new Set()]]), + referenced: subtreeAsked, }, ]); const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(1); - expect(subtree).not.toHaveBeenCalled(); + expect(subtreeRead).toHaveBeenCalledWith('covers'); + expect(subtreeAsked).not.toHaveBeenCalled(); }); it('RCLS-014: a category whose location holds another category is skipped with its marks', async () => { @@ -488,17 +558,18 @@ describe('StorageReclaimService: categories it may not judge', () => { expect(await marksOf('covers')).toEqual([{ key: 'x.jpg', first_seen_at: dbNow(new Date(0)) }]); }); - it('RCLS-018: a source that fails skips the category, never reads it as unreferenced, and keeps its marks', async () => { + it('RCLS-018: a source that fails skips the categories it answers for, never reads them as unreferenced', async () => { + await put('files', 'orphan.pdf'); + await markWeekOld('files', ['orphan.pdf']); await seedCovers(['orphan.jpg']); - await markWeekOld('covers', ['orphan.jpg']); - onAsk = (category) => { - if (category === 'covers') throw new Error('trips table locked'); + onRead = (id) => { + if (id === 'files') throw new Error('trip_files table locked'); }; const report = await reclaim.run({ now: T0 }); - expect(report.categories.covers).toMatchObject({ objects: 5, skipped: 'source_failed' }); - expect(report.categories.places.skipped).toBeNull(); - expect(logError).toHaveBeenCalledWith(expect.stringContaining('trips table locked')); - expect(await marksOf('covers')).toHaveLength(1); + expect(report.categories.files).toMatchObject({ objects: 1, skipped: 'source_failed' }); + expect(report.categories.covers.skipped).toBeNull(); + expect(vi.mocked(logError).mock.calls.filter(([line]) => String(line).includes('table locked'))).toHaveLength(2); + expect(await marksOf('files')).toEqual([{ key: 'orphan.pdf', first_seen_at: dbNow(new Date(0)) }]); }); }); @@ -506,7 +577,7 @@ describe('StorageReclaimService: when a pass runs at all', () => { it('RCLS-019: a pass refuses with a coded 409 on a demo instance and while another storage job holds the slot', async () => { const demo = vi.spyOn(t.get(RuntimeEnvService), 'isDemoMode').mockReturnValue(true); await expect(reclaim.run()).rejects.toMatchObject({ code: 'reclaim_demo' }); - expect(thrownBy(() => reclaim.start())).toMatchObject({ code: 'reclaim_demo' }); + await expect(reclaim.start()).rejects.toMatchObject({ code: 'reclaim_demo' }); demo.mockRestore(); expect(jobs.claimExclusive('a test')).toBe(true); @@ -525,8 +596,8 @@ describe('StorageReclaimService: when a pass runs at all', () => { it('RCLS-020: while a pass holds the slot, a migration is refused with its name, and the slot is free after', async () => { let refusal: unknown = null; - onAsk = async (category) => { - if (category !== 'covers' || refusal) return; + onRead = async () => { + if (refusal) return; refusal = await jobs.startMigration('covers', 'backups-local').catch((err: unknown) => err); }; await seedCovers(['orphan.jpg']); @@ -542,10 +613,10 @@ describe('StorageReclaimService: when a pass runs at all', () => { const gate = new Promise((resolve) => { release = resolve; }); - onAsk = async () => gate; - reclaim.start(); + onRead = async () => gate; + await reclaim.start(); expect((await reclaim.state()).running).toBe(true); - expect(thrownBy(() => reclaim.start())).toMatchObject({ code: 'reclaim_busy' }); + await expect(reclaim.start()).rejects.toMatchObject({ code: 'reclaim_busy' }); release(); await jobs.exclusiveIdle(); const state = await reclaim.state(); @@ -567,10 +638,10 @@ describe('StorageReclaimService: when a pass runs at all', () => { }); describe('StorageReclaimService: the delete itself', () => { - it('RCLS-023: a re-check that fails keeps the object', async () => { + it('RCLS-023: a fresh read that fails keeps the object', async () => { await markedWeekOld(['orphan.jpg']); onAsk = (_category, keys) => { - if (keys !== null) throw new Error('lookup timed out'); + if (keys === null) throw new Error('lookup timed out'); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(0); @@ -594,19 +665,21 @@ describe('StorageReclaimService: the delete itself', () => { expect(await marksOf('covers')).toEqual([]); }); - it('RCLS-025: the re-check before the delete asks about the whole batch at once, and the listing reads each source once', async () => { + it('RCLS-025: a pass reads each source once for every category, and once more, whole, right before a batch goes', async () => { await markedWeekOld(['a.jpg', 'b.jpg', 'c.jpg'], 12); + const reads: StorageReferenceSourceId[] = []; const asked: Array = []; + onRead = (id) => { + reads.push(id); + }; onAsk = (category, keys) => { if (category === 'covers') asked.push(keys === null ? null : [...keys].sort()); }; const report = await reclaim.run({ now: WEEK_LATER }); expect(report.categories.covers.deleted.objects).toBe(3); + expect(reads.sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort()); const freeForm = STORAGE_REFERENCE_SOURCE_IDS.filter((id) => id !== 'auth' && id !== 'files').length; - expect(asked.filter((keys) => keys === null)).toHaveLength(freeForm); - expect(asked.filter((keys) => keys !== null)).toEqual( - Array.from({ length: freeForm }, () => ['a.jpg', 'b.jpg', 'c.jpg']), - ); + expect(asked).toEqual(Array.from({ length: freeForm }, () => null)); }); it('RCLS-026: a delete that fails stops deleting in that category, keeps the mark and says why once', async () => { @@ -655,13 +728,24 @@ describe('StorageReclaimService: the delete itself', () => { expect(fs.existsSync(objectPath('places', 'place-orphan.jpg'))).toBe(true); }); - it('RCLS-030: nothing is deleted while a backup reads the uploads', async () => { + it('RCLS-030: nothing is deleted while a backup reads the uploads, and the report says so', async () => { await markedWeekOld(['orphan.jpg']); + await put('places', 'place-orphan.jpg'); + await markWeekOld('places', ['place-orphan.jpg']); + ['p1.jpg', 'p2.jpg', 'p3.jpg', 'p4.jpg', 'p5.jpg'].forEach((key) => namedIn('places').add(key)); + await Promise.all(['p1.jpg', 'p2.jpg', 'p3.jpg', 'p4.jpg', 'p5.jpg'].map((key) => put('places', key))); jobs.beginBackup(); try { const report = await reclaim.run({ now: WEEK_LATER }); expect(report.mode).toBe('delete'); - expect(report.categories.covers).toMatchObject({ eligible: { objects: 1 }, deleted: { objects: 0 } }); + expect(report.categories.covers).toMatchObject({ + eligible: { objects: 1 }, + deleted: { objects: 0 }, + skipped: 'backup_running', + }); + expect(report.categories.places).toMatchObject({ deleted: { objects: 0 }, skipped: 'backup_running' }); + // A category with nothing due has nothing to say. + expect(report.categories.avatars.skipped).toBeNull(); expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); } finally { jobs.endBackup(); @@ -684,6 +768,9 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => { for (const category of ['avatars', 'places'] as const) { expect(report.categories[category]).toMatchObject({ objects: 0, skipped: 'interrupted' }); } + for (const category of ['photos', 'photos-google', 'photos-trek', 'backups'] as const) { + expect(report.categories[category].skipped).toBe('not_reclaimed'); + } expect((await reclaim.state()).lastRun?.stopped).toBe('database_reopened'); expect(await countRows(orm, AuditLog, { action: 'storage.reclaim_deleted' })).toBe(1); }); @@ -711,7 +798,7 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => { let restoreAsked = false; let restoreWent = false; onAsk = (_category, keys) => { - if (keys === null || restoreAsked) return; + if (keys !== null || restoreAsked) return; restoreAsked = true; restoreStarted = jobs.beginRestore().then(() => { restoreWent = true; @@ -729,9 +816,14 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => { } }); - it('RCLS-034: a reopen turns deletion off and drops marks, confirmations and reports, but keeps the install id', async () => { + it('RCLS-034: a reopen turns deletion off and drops marks, confirmations, the queue and reports, keeping the id', async () => { await markedWeekOld(['orphan.jpg']); - await setAppSetting(orm, 'storage.reclaim_confirmed', JSON.stringify({ installId, counts: { covers: 1 } })); + await setAppSetting( + orm, + 'storage.reclaim_confirmed', + JSON.stringify({ installId, categories: { covers: { count: 1, confirmedAt: Date.now() } } }), + ); + await orm.repo(StorageReleaseQueue).enqueue([{ category: 'covers', key: 'queued.jpg' }], dbNow(new Date())); // A restored database carries another install's id and switch. await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab'); await setAppSetting(orm, 'storage.reclaim_delete', '12345678-aaaa-4bbb-8ccc-1234567890ab'); @@ -740,8 +832,61 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => { const reset = ['storage.reclaim_delete', 'storage.reclaim_confirmed', 'storage.reclaim_report']; expect(await Promise.all(reset.map((key) => readAppSetting(orm, key)))).toEqual([null, null, null]); expect(await marksOf('covers')).toEqual([]); + expect(await countRows(orm, StorageReleaseQueue)).toBe(0); expect(await reclaim.state()).toEqual({ deleteEnabled: false, running: false, lastRun: null, lastDeleteRun: null }); }); + + it('RCLS-040: at boot, a database naming another installation loses its reclaim state and gets this id', async () => { + const install = t.get(StorageInstallService); + await markedWeekOld(['orphan.jpg']); + await orm.repo(StorageReleaseQueue).enqueue([{ category: 'covers', key: 'queued.jpg' }], dbNow(new Date())); + // As this installation's own database: nothing to reset. + expect(await install.reconcile()).toBe(false); + expect((await reclaim.state()).deleteEnabled).toBe(true); + // A database restored at boot, cloned or copied from another one. + await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + await setAppSetting(orm, 'storage.reclaim_delete', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + expect(await install.reconcile()).toBe(true); + expect(await readAppSetting(orm, 'install_id')).toBe(installId); + expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBeNull(); + expect(await readAppSetting(orm, 'storage.reclaim_report')).toBeNull(); + expect(await marksOf('covers')).toEqual([]); + expect(await countRows(orm, StorageReleaseQueue)).toBe(0); + }); + + it('RCLS-041: the install id file is made once, mode 0600; a garbled one keeps every delete off', async () => { + const file = path.join(dataRoot, 'id'); + expect(fs.readFileSync(file, 'utf8').trim()).toBe(installId); + if (process.platform !== 'win32') expect(fs.statSync(file).mode & 0o777).toBe(0o600); + const garbled = path.join(dataRoot, 'garbled'); + fs.writeFileSync(garbled, 'not an id\n'); + const install = t.get(StorageInstallService); + const paths = t.get(DataPathsService) as { installIdFile: string }; + /** The service reads its file once; these cases read another one afresh. */ + const reread = (file: string) => { + paths.installIdFile = file; + (install as unknown as { fileId: string | null | undefined }).fileId = undefined; + return install.installId(); + }; + try { + expect(reread(garbled)).toBeNull(); + await expect(reclaim.setDeleteEnabled(true)).rejects.toMatchObject({ code: 'reclaim_install_unknown' }); + expect((await reclaim.state()).deleteEnabled).toBe(false); + expect(await install.reconcile()).toBe(false); + } finally { + expect(reread(file)).toBe(installId); + } + // A missing file is made, once. + const fresh = path.join(dataRoot, 'nested', 'fresh-id'); + try { + const made = reread(fresh); + expect(made).toMatch(/^[0-9a-f-]{36}$/); + expect(fs.readFileSync(fresh, 'utf8').trim()).toBe(made); + expect(reread(fresh)).toBe(made); + } finally { + expect(reread(file)).toBe(installId); + } + }); }); describe('StorageReclaimService: the switch and the stored reports', () => { @@ -759,6 +904,31 @@ describe('StorageReclaimService: the switch and the stored reports', () => { expect(await readsOff('12345678-aaaa-4bbb-8ccc-1234567890ab')).toBe(false); }); + it('RCLS-042: without an install id a pass judges nothing and deletes nothing', async () => { + await markedWeekOld(['orphan.jpg']); + vi.spyOn(t.get(StorageInstallService), 'installId').mockReturnValue(null); + const report = await reclaim.run({ now: WEEK_LATER }); + expect(report.mode).toBe('report'); + expect(report.categories.covers).toMatchObject({ skipped: 'owner_unknown', deleted: { objects: 0 } }); + expect(fs.existsSync(objectPath('covers', 'orphan.jpg'))).toBe(true); + }); + + it('RCLS-043: a pass waits for a release tick holding the slot instead of losing its night', async () => { + expect(jobs.claimExclusive(STORAGE_RELEASE_JOB)).toBe(true); + const running = reclaim.run({ now: T0 }); + await Promise.resolve(); + expect(jobs.exclusiveHolder).toBe(STORAGE_RELEASE_JOB); + jobs.releaseExclusive(STORAGE_RELEASE_JOB); + expect((await running).stopped).toBeNull(); + // Any other holder still refuses at once. + expect(jobs.claimExclusive('a backfill')).toBe(true); + try { + await expect(reclaim.run({ now: T0 })).rejects.toMatchObject({ code: 'reclaim_busy' }); + } finally { + jobs.releaseExclusive('a backfill'); + } + }); + it('RCLS-036: a stored report that no longer parses reads as never run', async () => { await setAppSetting(orm, 'storage.reclaim_report', '{not json'); expect((await reclaim.state()).lastRun).toBeNull(); diff --git a/server/tests/unit/nest/storage/storage-release.job.test.ts b/server/tests/unit/nest/storage/storage-release.job.test.ts new file mode 100644 index 000000000..292f68b08 --- /dev/null +++ b/server/tests/unit/nest/storage/storage-release.job.test.ts @@ -0,0 +1,85 @@ +/** + * StorageReleaseJob: every ten minutes, the due rows of the release queue. + * It hands the registrar the tick's own promise, so the lease covers the whole + * tick, logs at debug only, and contains a failure in one error line. + * RELJ-001 to RELJ-004. + */ +import { logDebug, logError } from '../../../../src/nest/audit/audit-log.logger'; +import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service'; +import { StorageReleaseJob } from '../../../../src/nest/storage/storage-release.job'; +import type { ReleaseTickResult, StorageReleaseService } from '../../../../src/nest/storage/storage-release.service'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +function makeJob(processDue: () => Promise, enabled = true) { + const registrar = { + isEnabled: vi.fn(() => enabled), + register: vi.fn((_name: string, _expression: string, _onTick: () => void | Promise) => enabled), + }; + const release = { processDue: vi.fn(processDue) }; + const job = new StorageReleaseJob( + registrar as unknown as CronRegistrarService, + release as unknown as StorageReleaseService, + ); + return { job, registrar, release }; +} + +const ran = (deleted: number, kept: number): ReleaseTickResult => ({ skipped: null, deleted, kept }); + +beforeEach(() => vi.clearAllMocks()); + +describe('StorageReleaseJob', () => { + it('RELJ-001: registers every ten minutes and hands the registrar the tick promise itself', async () => { + let finish: (value: ReleaseTickResult) => void = () => undefined; + const { job, registrar, release } = makeJob(() => new Promise((resolve) => (finish = resolve))); + job.onApplicationBootstrap(); + expect(registrar.register).toHaveBeenCalledWith('storage-release', '*/10 * * * *', expect.any(Function)); + const [call] = registrar.register.mock.calls; + if (!call) throw new Error('the job registered nothing'); + const ticking = call[2](); + expect(ticking).toBeInstanceOf(Promise); + let settled = false; + void (ticking as Promise).then(() => (settled = true)); + await Promise.resolve(); + expect(settled).toBe(false); + finish(ran(0, 0)); + await ticking; + expect(release.processDue).toHaveBeenCalledTimes(1); + }); + + it('RELJ-002: does not register under the test gate', () => { + const { job, registrar, release } = makeJob(async () => ran(0, 0), false); + job.onApplicationBootstrap(); + expect(registrar.register).not.toHaveBeenCalled(); + expect(release.processDue).not.toHaveBeenCalled(); + }); + + it('RELJ-003: says at debug what a tick did or why it skipped, and nothing for an empty one', async () => { + const quiet = makeJob(async () => ran(0, 0)); + await quiet.job.tick(new Date(5)); + expect(quiet.release.processDue).toHaveBeenCalledWith(new Date(5)); + expect(logDebug).not.toHaveBeenCalled(); + await makeJob(async () => ran(3, 1)).job.tick(); + expect(logDebug).toHaveBeenCalledWith('Storage release: deleted 3, kept 1 still named'); + await makeJob(async () => ({ skipped: 'busy', deleted: 0, kept: 0 })).job.tick(); + expect(logDebug).toHaveBeenCalledWith('Storage release skipped: busy'); + expect(logError).not.toHaveBeenCalled(); + }); + + it('RELJ-004: a failure is contained in one error line', async () => { + await makeJob(async () => { + throw new Error('registry not initialized'); + }).job.tick(); + expect(logError).toHaveBeenCalledWith('Storage release: registry not initialized'); + await makeJob(vi.fn<() => Promise>().mockRejectedValue('plain')).job.tick(); + expect(logError).toHaveBeenCalledWith('Storage release: plain'); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-release.service.test.ts b/server/tests/unit/nest/storage/storage-release.service.test.ts index 1405152b8..7209ffc5b 100644 --- a/server/tests/unit/nest/storage/storage-release.service.test.ts +++ b/server/tests/unit/nest/storage/storage-release.service.test.ts @@ -1,15 +1,23 @@ /** * StorageReleaseService (#2924): what a user's delete or replace left unnamed - * goes right after it, through the same sources and parser as the nightly - * pass, over a real StorageModule (a LocalDriver on a temp folder) with fake - * sources standing in for the owner domains. An object any row still names - * stays; the pass's guards hold; a failure never reaches the caller. - * RELS-001 to RELS-010. + * is queued by the request, which deletes nothing, and goes once it is due, + * through the same sources and parser as the nightly pass, over a real + * StorageModule (a LocalDriver on a temp folder) with fake sources standing in + * for the owner domains. An object any row still names stays; the pass's + * guards hold, the owner marker included; a failure never reaches the caller + * and leaves its rows queued. RELS-001 to RELS-021. */ import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; +import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity'; +import { StorageReleaseQueueRepository } from '../../../../src/db/repositories/StorageReleaseQueue.repository'; +import { dbNow } from '../../../../src/db/types'; +import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service'; import { logWarn } from '../../../../src/nest/audit/audit-log.logger'; +import { StorageInstallService } from '../../../../src/nest/storage/storage-install.service'; import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service'; +import { StorageReclaimGuardService } from '../../../../src/nest/storage/storage-reclaim-guard.service'; import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service'; +import { MAX_LIKE_TERMS } from '../../../../src/nest/storage/storage-reclaim-source.service'; import { STORAGE_RECLAIMABLE_CATEGORIES, STORAGE_REFERENCE_SOURCE_IDS, @@ -18,12 +26,16 @@ import { type StorageReferenceSource, type StorageReferenceSourceId, } from '../../../../src/nest/storage/storage-reclaim.types'; -import { StorageReleaseService } from '../../../../src/nest/storage/storage-release.service'; +import { + RELEASE_DELAY_MS, + STORAGE_RELEASE_JOB, + StorageReleaseService, +} from '../../../../src/nest/storage/storage-release.service'; import { StorageModule } from '../../../../src/nest/storage/storage.module'; import { StorageService } from '../../../../src/nest/storage/storage.service'; import type { ServedCategory } from '../../../../src/nest/storage/storage.types'; import { createSnapshotTestDb } from '../../../helpers/db-mock'; -import { deleteRows } from '../../../helpers/factories/rows'; +import { countRows, deleteRows, findRows } from '../../../helpers/factories/rows'; import { setAppSetting } from '../../../helpers/factories/settings'; import { createTestModule, type TestModule } from '../../../helpers/test-module'; import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; @@ -42,13 +54,17 @@ vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => logWarn: vi.fn(), })); +const DAY = 24 * 60 * 60 * 1000; const db = createSnapshotTestDb(); let t: TestModule; let orm: TestOrm; let uploadsRoot: string; +let dataRoot: string; let release: StorageReleaseService; let storage: StorageService; let references: StorageReclaimRegistryService; +let jobs: StorageJobsService; +let installId: string; /** What the fake sources still name, per category, lowercased. */ let named: Map>; let asked: Array<{ id: StorageReferenceSourceId; category: ServedCategory; keys: string[] }>; @@ -63,6 +79,9 @@ function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource { namedKeys: async () => { throw new Error('a release never reads a whole category'); }, + namedKeysByCategory: async () => { + throw new Error('a release never reads a whole category'); + }, async referenced(category, keys) { asked.push({ id, category, keys: [...keys] }); if (failWith) throw failWith; @@ -80,25 +99,49 @@ function exists(category: string, key: string): boolean { return fs.existsSync(path.join(uploadsRoot, category, key)); } -beforeAll(async () => { - uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-uploads-')); - orm = await createTestOrm(db); +async function queued(category: string, key?: string): Promise { + return await countRows(orm, StorageReleaseQueue, key === undefined ? { category } : { category, key }); +} + +/** Once everything queued so far is due. */ +function due(): Date { + return new Date(Date.now() + RELEASE_DELAY_MS + 60_000); +} + +async function setBackends(extra: unknown[] = []): Promise { await setAppSetting( orm, 'storage.backends', - JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }]), + JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, ...extra]), ); - t = await createTestModule({ db, imports: [StorageModule], init: true }); +} + +beforeAll(async () => { + uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-uploads-')); + dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-data-')); + orm = await createTestOrm(db); + await setBackends(); + const paths = { ...new DataPathsService(), installIdFile: path.join(dataRoot, '.trek-install-id') }; + t = await createTestModule({ + db, + imports: [StorageModule], + init: true, + overrides: [{ provide: DataPathsService, useValue: paths }], + }); release = t.get(StorageReleaseService); storage = t.get(StorageService); references = t.get(StorageReclaimRegistryService); + jobs = t.get(StorageJobsService); + installId = t.get(StorageInstallService).installId() ?? ''; await storage.reloadConfig(); for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id)); }); -beforeEach(() => { +beforeEach(async () => { vi.restoreAllMocks(); vi.mocked(logWarn).mockClear(); + await deleteRows(orm, StorageReleaseQueue); + t.get(StorageReclaimGuardService).forget(); named = new Map(); asked = []; failWith = null; @@ -110,141 +153,291 @@ afterAll(async () => { await orm.close(); db.close(); fs.rmSync(uploadsRoot, { recursive: true, force: true }); + fs.rmSync(dataRoot, { recursive: true, force: true }); }); const UUID = '6f1e2d3c-4b5a-4968-8776-655443322110'; -describe('StorageReleaseService', () => { - it('RELS-001: deletes what no row names, keeps what one still does, and answers what went', async () => { +describe('StorageReleaseService: the request only queues', () => { + it('RELS-001: queues what the values name, deletes nothing itself, and the due tick deletes what no row names', async () => { + expect(installId).toMatch(/^[0-9a-f-]{36}$/); await put('covers', `${UUID}-old.jpg`); await put('covers', `${UUID}-shared.jpg`); named.set('covers', new Set([`${UUID}-shared.jpg`])); - const released = await release.releaseValues([`/uploads/covers/${UUID}-old.jpg`, `covers/${UUID}-shared.jpg`]); - expect(released).toEqual([{ category: 'covers', key: `${UUID}-old.jpg` }]); + const before = Date.now(); + expect(await release.releaseValues([`/uploads/covers/${UUID}-old.jpg`, `covers/${UUID}-shared.jpg`])).toBe(2); + expect(exists('covers', `${UUID}-old.jpg`)).toBe(true); + expect(asked).toEqual([]); + const rows = await findRows(orm, StorageReleaseQueue, { category: 'covers' }); + expect(rows.map((row) => row.key).sort()).toEqual([`${UUID}-old.jpg`, `${UUID}-shared.jpg`]); + for (const row of rows) expect(row.due_at >= dbNow(new Date(before + RELEASE_DELAY_MS - 1000))).toBe(true); + + expect(await release.processDue(due())).toEqual({ skipped: null, deleted: 1, kept: 1 }); expect(exists('covers', `${UUID}-old.jpg`)).toBe(false); expect(exists('covers', `${UUID}-shared.jpg`)).toBe(true); + expect(await queued('covers')).toBe(0); }); it('RELS-002: reads every spelling a column holds, and a bare name with its column options', async () => { - await put('places', `${UUID}.jpg`); - await put('avatars', `${UUID}.png`); - await put('files', `${UUID}. #4711`); - const released = await release.releaseValues([`https://trip.example.test/uploads/places/${UUID}.jpg?v=2`]); - expect(released).toEqual([{ category: 'places', key: `${UUID}.jpg` }]); - expect(await release.releaseValues([`${UUID}.png`], { bare: 'avatars' })).toEqual([ - { category: 'avatars', key: `${UUID}.png` }, - ]); - expect(await release.releaseValues([`legacy/${UUID}. #4711`], { bare: 'files', basename: true })).toEqual([ - { category: 'files', key: `${UUID}. #4711` }, - ]); - // Every source that claims the category was asked about the keys, narrowed, never about the whole category. - expect( - asked - .filter((question) => question.category === 'files') - .map((question) => question.id) - .sort(), - ).toEqual(['collections', 'files', 'journey', 'photos', 'place-contrib', 'places', 'trips'].sort()); + expect(await release.releaseValues([`https://trip.example.test/uploads/places/${UUID}.jpg?v=2`])).toBeGreaterThan( + 0, + ); + expect(await release.releaseValues([`${UUID}.png`], { bare: 'avatars' })).toBe(1); + expect(await release.releaseValues([`legacy/${UUID}. #4711`], { bare: 'files', basename: true })).toBeGreaterThan( + 0, + ); + expect(await queued('places', `${UUID}.jpg`)).toBe(1); + expect(await queued('avatars', `${UUID}.png`)).toBe(1); + expect(await queued('files', `${UUID}. #4711`)).toBe(1); }); - it('RELS-003: never touches a nested key, an owner marker or a category the ledger keeps', async () => { - await put('journey', `thumbs/${UUID}.jpg`); - await put('covers', storageOwnerMarkerKey(UUID)); - await put('photos', `${UUID}.jpg`); + it('RELS-003: never queues a nested key, an owner marker, a category the ledger keeps or a foreign URL', async () => { expect( await release.releaseValues([ `/uploads/journey/thumbs/${UUID}.jpg`, `/uploads/covers/${storageOwnerMarkerKey(UUID)}`, `/uploads/photos/${UUID}.jpg`, + `/uploads/backups/${UUID}.zip`, 'https://elsewhere.example.test/x.jpg', null, + undefined, ]), - ).toEqual([]); - expect(exists('journey', `thumbs/${UUID}.jpg`)).toBe(true); - expect(exists('covers', storageOwnerMarkerKey(UUID))).toBe(true); - expect(exists('photos', `${UUID}.jpg`)).toBe(true); + ).toBe(0); + expect(await countRows(orm, StorageReleaseQueue)).toBe(0); + }); + + it('RELS-004: one value queues a bounded number of keys, and a key queued again waits from the later time', async () => { + const many = Array.from({ length: 40 }, (_unused, i) => `/uploads/covers/${UUID}-${i}.jpg`).join(' '); + expect(await release.releaseValues([many])).toBeLessThanOrEqual(16); + await deleteRows(orm, StorageReleaseQueue); + await release.releaseValues([`/uploads/covers/${UUID}-again.jpg`]); + const first = (await findRows(orm, StorageReleaseQueue, { key: `${UUID}-again.jpg` }))[0]?.due_at ?? ''; + vi.spyOn(Date, 'now').mockReturnValue(Date.now() + 2 * DAY); + await release.releaseValues([`/uploads/covers/${UUID}-again.jpg`]); + vi.mocked(Date.now).mockRestore(); + const second = (await findRows(orm, StorageReleaseQueue, { key: `${UUID}-again.jpg` }))[0]?.due_at ?? ''; + expect(second > first).toBe(true); + expect(await queued('covers', `${UUID}-again.jpg`)).toBe(1); + }); + + it('RELS-005: a queue that fails is logged and answered as nothing queued, never thrown', async () => { + vi.spyOn(StorageReleaseQueueRepository.prototype, 'size').mockRejectedValueOnce(new Error('database is locked')); + await expect(release.releaseValues([`/uploads/covers/${UUID}-locked.jpg`])).resolves.toBe(0); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('database is locked')); + }); +}); + +describe('StorageReleaseService: a tick deletes only what is due and unnamed', () => { + it('RELS-006: nothing goes before it is due', async () => { + await put('places', `${UUID}-young.jpg`); + await release.releaseValues([`/uploads/places/${UUID}-young.jpg`]); + expect(await release.processDue(new Date())).toEqual({ skipped: null, deleted: 0, kept: 0 }); + expect(exists('places', `${UUID}-young.jpg`)).toBe(true); + expect(await queued('places', `${UUID}-young.jpg`)).toBe(1); + }); + + it('RELS-007: a value an undo wrote back before it was due keeps its object, and its row goes', async () => { + await put('covers', `${UUID}-undone.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-undone.jpg`]); + named.set('covers', new Set([`${UUID}-undone.jpg`])); + expect((await release.processDue(due())).kept).toBe(1); + expect(exists('covers', `${UUID}-undone.jpg`)).toBe(true); + expect(await queued('covers')).toBe(0); + }); + + it('RELS-008: every source that claims the category is asked, at most 50 keys a question, 200 rows a tick', async () => { + const keys = Array.from({ length: 230 }, (_unused, i) => `${UUID}-n${i}.jpg`); + expect(await release.releaseValues(keys.map((key) => `/uploads/places/${key}`))).toBe(230); + const result = await release.processDue(due()); + expect(result.deleted).toBe(200); + expect(await queued('places')).toBe(30); + expect(Math.max(...asked.map((question) => question.keys.length))).toBeLessThanOrEqual(50); + expect([...new Set(asked.map((question) => question.id))].sort()).toEqual( + ['collections', 'journey', 'photos', 'place-contrib', 'places', 'trips'].sort(), + ); + expect(MAX_LIKE_TERMS).toBeLessThanOrEqual(50); + }); + + it('RELS-009: a key named in another spelling, case included, keeps its object', async () => { + await put('covers', `${UUID}-case.jpg`); + named.set('covers', new Set([`${UUID}-case.jpg`])); + await release.releaseValues([`/uploads/covers/${UUID.toUpperCase()}-CASE.JPG`]); + await release.processDue(due()); + expect(exists('covers', `${UUID}-case.jpg`)).toBe(true); + }); + + it('RELS-010: a source that fails keeps the rows and never throws; the next tick deletes', async () => { + await put('covers', `${UUID}-f.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-f.jpg`]); + failWith = new Error('places table locked'); + expect((await release.processDue(due())).deleted).toBe(0); + expect(exists('covers', `${UUID}-f.jpg`)).toBe(true); + expect(await queued('covers', `${UUID}-f.jpg`)).toBe(1); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('places table locked')); + failWith = null; + expect((await release.processDue(due())).deleted).toBe(1); + expect(exists('covers', `${UUID}-f.jpg`)).toBe(false); + }); + + it('RELS-011: a delete that fails keeps its row and stops the category for the tick', async () => { + await put('covers', `${UUID}-d1.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-d1.jpg`]); + vi.spyOn(storage, 'delete').mockRejectedValue(new Error('permission denied')); + expect((await release.processDue(due())).deleted).toBe(0); + expect(exists('covers', `${UUID}-d1.jpg`)).toBe(true); + expect(await queued('covers', `${UUID}-d1.jpg`)).toBe(1); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('permission denied')); + }); + + it('RELS-012: an object the facade keeps on delete (a demo) stays, and its row goes', async () => { + await put('covers', `${UUID}-k.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-k.jpg`]); + vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true); + expect((await release.processDue(due())).kept).toBe(1); + expect(exists('covers', `${UUID}-k.jpg`)).toBe(true); + expect(await queued('covers')).toBe(0); + }); + + it('RELS-013: a row that waited thirty days past its time is left to the nightly pass', async () => { + await orm + .repo(StorageReleaseQueue) + .enqueue([{ category: 'covers', key: `${UUID}-ancient.jpg` }], dbNow(new Date(Date.now() - 31 * DAY))); + await release.processDue(new Date()); + expect(await queued('covers', `${UUID}-ancient.jpg`)).toBe(0); expect(asked).toEqual([]); }); +}); - it('RELS-004: a category missing a required source is left to the nightly pass', async () => { +describe('StorageReleaseService: the guards the nightly pass keeps', () => { + it('RELS-014: a category missing a required source keeps its rows', async () => { await put('avatars', `${UUID}-a.png`); + await release.releaseValues([`${UUID}-a.png`], { bare: 'avatars' }); vi.spyOn(references, 'sources').mockReturnValue(references.sources().filter((source) => source.id !== 'auth')); - expect(await release.releaseIfUnreferenced('avatars', [`${UUID}-a.png`])).toEqual([]); + await release.processDue(due()); expect(exists('avatars', `${UUID}-a.png`)).toBe(true); + expect(await queued('avatars')).toBe(1); }); - it('RELS-005: nothing goes while a restore runs', async () => { + it('RELS-015: nothing runs while a restore or a backup runs, or another storage job holds the slot', async () => { await put('covers', `${UUID}-r.jpg`); - const jobs = t.get(StorageJobsService); + await release.releaseValues([`/uploads/covers/${UUID}-r.jpg`]); await jobs.beginRestore(); try { - expect(await release.releaseIfUnreferenced('covers', [`${UUID}-r.jpg`])).toEqual([]); + expect((await release.processDue(due())).skipped).toBe('busy'); } finally { jobs.endRestore(); } + jobs.beginBackup(); + try { + expect((await release.processDue(due())).skipped).toBe('busy'); + } finally { + jobs.endBackup(); + } + expect(jobs.claimExclusive('a test job')).toBe(true); + try { + expect((await release.processDue(due())).skipped).toBe('busy'); + } finally { + jobs.releaseExclusive('a test job'); + } expect(exists('covers', `${UUID}-r.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); + // A tick holds the slot under its own name while it runs, and lets go after. + const holders: Array = []; + named.set('covers', new Set()); + vi.spyOn(storage, 'delete').mockImplementation(async () => { + holders.push(jobs.exclusiveHolder); + }); + await release.processDue(due()); + expect(holders).toEqual([STORAGE_RELEASE_JOB]); + expect(jobs.exclusiveHolder).toBeNull(); }); - it('RELS-006: a category whose location overlaps another is left alone', async () => { + it('RELS-016: a restore that begins mid-tick stops it and keeps the rest queued', async () => { + await put('covers', `${UUID}-m1.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-m1.jpg`]); + const realReferenced = references.sources().map((source) => source.referenced.bind(source)); + let restoring = false; + references.sources().forEach((source, i) => { + vi.spyOn(source, 'referenced').mockImplementation(async (category, keys) => { + if (!restoring) { + restoring = true; + void jobs.beginRestore(); + } + return await (realReferenced[i] as typeof source.referenced)(category, keys); + }); + }); + try { + expect((await release.processDue(due())).deleted).toBe(0); + } finally { + jobs.endRestore(); + } + expect(exists('covers', `${UUID}-m1.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); + }); + + it('RELS-017: a category whose location overlaps another keeps its rows', async () => { await put('covers', `${UUID}-s.jpg`); - await setAppSetting( - orm, - 'storage.backends', - JSON.stringify([ - { name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }, - { name: 'nested', type: 'local', options: { root: path.join(uploadsRoot, 'covers') } }, - ]), - ); + await release.releaseValues([`/uploads/covers/${UUID}-s.jpg`]); + await setBackends([{ name: 'nested', type: 'local', options: { root: path.join(uploadsRoot, 'covers') } }]); await setAppSetting(orm, 'storage.categories', JSON.stringify({ 'photos-trek': 'nested' })); await storage.reloadConfig(); try { - expect(await release.releaseIfUnreferenced('covers', [`${UUID}-s.jpg`])).toEqual([]); + await release.processDue(due()); expect(exists('covers', `${UUID}-s.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); } finally { - await setAppSetting( - orm, - 'storage.backends', - JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }]), - ); + await setBackends(); await deleteRows(orm, AppSettings, { key: 'storage.categories' }); await storage.reloadConfig(); } }); - it('RELS-007: a source that fails keeps everything and never throws', async () => { - await put('covers', `${UUID}-f.jpg`); - failWith = new Error('places table locked'); - await expect(release.releaseValues([`/uploads/covers/${UUID}-f.jpg`])).resolves.toEqual([]); - expect(exists('covers', `${UUID}-f.jpg`)).toBe(true); - expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('places table locked')); + it('RELS-018: another installation marker keeps the category untouched; this one writes its own', async () => { + await put('covers', `${UUID}-o.jpg`); + await put('covers', storageOwnerMarkerKey('another-install-0001')); + await release.releaseValues([`/uploads/covers/${UUID}-o.jpg`]); + try { + await release.processDue(due()); + expect(exists('covers', `${UUID}-o.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); + } finally { + await storage.delete('covers', storageOwnerMarkerKey('another-install-0001')); + } + // The verdict is cached until it is forgotten (a reopen) or runs out. + await release.processDue(due()); + expect(exists('covers', `${UUID}-o.jpg`)).toBe(true); + t.get(StorageReclaimGuardService).forget(); + await release.processDue(due()); + expect(exists('covers', `${UUID}-o.jpg`)).toBe(false); + expect(exists('covers', storageOwnerMarkerKey(installId))).toBe(true); }); - it('RELS-008: a delete that fails is logged and answered as kept, the others still go', async () => { - await put('covers', `${UUID}-d1.jpg`); - await put('covers', `${UUID}-d2.jpg`); - const realDelete = storage.delete.bind(storage); - vi.spyOn(storage, 'delete').mockImplementation(async (category, name) => { - if (name.endsWith('-d1.jpg')) throw new Error('permission denied'); - await realDelete(category, name); + it('RELS-019: an owner marker that cannot be looked for leaves the category as owner_unknown', async () => { + await put('covers', `${UUID}-u.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-u.jpg`]); + vi.spyOn(storage, 'list').mockImplementation(() => { + throw new Error('listing refused'); }); - const released = await release.releaseIfUnreferenced('covers', [`${UUID}-d1.jpg`, `${UUID}-d2.jpg`]); - expect(released).toEqual([`${UUID}-d2.jpg`]); - expect(exists('covers', `${UUID}-d1.jpg`)).toBe(true); - expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('permission denied')); + await release.processDue(due()); + expect(exists('covers', `${UUID}-u.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('listing refused')); }); - it('RELS-009: an object the facade keeps on delete (a demo) and one that is not there are not reported', async () => { - await put('covers', `${UUID}-k.jpg`); - const keeps = vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true); - expect(await release.releaseIfUnreferenced('covers', [`${UUID}-k.jpg`])).toEqual([]); - expect(exists('covers', `${UUID}-k.jpg`)).toBe(true); - keeps.mockRestore(); - expect(await release.releaseIfUnreferenced('covers', [`${UUID}-missing.jpg`])).toEqual([]); + it('RELS-020: without an install id nothing is released', async () => { + await put('covers', `${UUID}-i.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-i.jpg`]); + vi.spyOn(t.get(StorageInstallService), 'installId').mockReturnValue(null); + expect((await release.processDue(due())).skipped).toBe('no_install_id'); + expect(exists('covers', `${UUID}-i.jpg`)).toBe(true); }); - it('RELS-010: a key named in another spelling, case included, keeps its object', async () => { - await put('covers', `${UUID}-case.jpg`); - named.set('covers', new Set([`${UUID}-case.jpg`])); - expect(await release.releaseValues([`/uploads/covers/${UUID.toUpperCase()}-CASE.JPG`])).toEqual([]); - expect(exists('covers', `${UUID}-case.jpg`)).toBe(true); + it('RELS-021: a key with no run to narrow a read by is kept without a question, for the nightly pass', async () => { + await put('covers', 'short.jpg'); + await release.releaseValues(['/uploads/covers/short.jpg']); + expect((await release.processDue(due())).kept).toBe(1); + expect(asked).toEqual([]); + expect(exists('covers', 'short.jpg')).toBe(true); + expect(await queued('covers')).toBe(0); }); });