diff --git a/server/src/app-config/README.md b/server/src/app-config/README.md index 35facaed2..8c06cd676 100644 --- a/server/src/app-config/README.md +++ b/server/src/app-config/README.md @@ -60,7 +60,8 @@ tokens, `RuntimeEnvService`) and consumes the SAME derive functions. `uploads/` (on `SERVER_ROOT`, the package directory, never on `process.cwd()`) and names what lives in them: backups, the scratch dir, the logs, the JWT secret and encryption key files, the install id file the storage reclaim -tells a copied database apart by, and the database file (TREK_DB_FILE when set, +tells a copied database apart by, the reset file a restore leaves for it, and +the database file (TREK_DB_FILE when set, else `data/travel.db`). Never derive one of these paths from `__dirname` in a new file: a moved file would quietly move the data. diff --git a/server/src/app-config/data-paths.ts b/server/src/app-config/data-paths.ts index 5fd091336..2ab31c0b4 100644 --- a/server/src/app-config/data-paths.ts +++ b/server/src/app-config/data-paths.ts @@ -35,6 +35,14 @@ export interface DataPaths { * restored or copied from another install is told apart by it (storage reclaim). */ readonly installIdFile: string; + /** + * `data/.trek-reclaim-reset`: left by a restore right before it swaps the + * database in, removed once storage reclaim has dropped the state the swapped + * database brought. A start that finds it resets that state first, so a + * restore whose reopen failed, or one at boot, is covered even when the + * backup came from this installation. Never part of a backup. + */ + readonly reclaimResetFile: string; /** * The SQLite file a non-test process opens: TREK_DB_FILE when set, else * `data/travel.db`. Under NODE_ENV=test the connection is `:memory:` instead @@ -65,6 +73,7 @@ export function resolveDataPaths( jwtSecretFile: path.join(dataDir, '.jwt_secret'), encryptionKeyFile: path.join(dataDir, '.encryption_key'), installIdFile: path.join(dataDir, '.trek-install-id'), + reclaimResetFile: path.join(dataDir, '.trek-reclaim-reset'), dbFile: env.db.trekDbFile || path.join(dataDir, 'travel.db'), }; } diff --git a/server/src/db/repositories/StorageReleaseQueue.repository.ts b/server/src/db/repositories/StorageReleaseQueue.repository.ts index 253daf209..7ac8e2970 100644 --- a/server/src/db/repositories/StorageReleaseQueue.repository.ts +++ b/server/src/db/repositories/StorageReleaseQueue.repository.ts @@ -14,9 +14,9 @@ export interface ReleaseQueueRow { /** * `storage_release_queue`: the objects a delete or a replace left unnamed, * keyed by category and category-relative key, each with the time it may go. - * Owners only add rows; the release job reads the due ones of the categories - * it may judge, asks every reference source again and drops each row once it - * has decided, deleted or not. A key queued again waits from the later time. + * Owners only add rows; the release job reads the due ones of each category + * it may judge (a share of its tick each), asks every reference source again + * and drops each row once it has decided, deleted or not. A key queued again waits from the later time. */ export class StorageReleaseQueueRepository extends TrekRepository { private db() { @@ -49,15 +49,16 @@ export class StorageReleaseQueueRepository extends TrekRepository row.category); } - /** The rows of these categories due at `now`, the longest waiting first, at most `limit`. */ - async dueIn(categories: string[], now: string, limit: number): Promise { - if (categories.length === 0 || limit <= 0) return []; + /** The rows of one category due at `now`, the longest waiting first, at most `limit`. */ + async dueIn(category: string, now: string, limit: number): Promise { + if (limit <= 0) return []; return await this.db() .selectFrom('storage_release_queue') .select(['category', 'key', 'due_at']) - .where('category', 'in', categories) + .where('category', '=', category) .where('due_at', '<=', now) .orderBy('due_at') + .orderBy('key') .limit(limit) .execute(); } diff --git a/server/src/index.ts b/server/src/index.ts index 930de1f7b..0d74d6831 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -138,10 +138,12 @@ async function bootstrap(): Promise { async function restoreBeforeTheDatabaseOpens() { const { restoreOnFirstBoot } = require('./nest/backup/boot-restore') as typeof import('./nest/backup/boot-restore'); const env = readEnv(); + const paths = resolveDataPaths(env); return restoreOnFirstBoot({ archive: env.backup.restoreFromBackup, dbFile: resolveDbPath(), - dataDir: resolveDataPaths(env).dataDir, + dataDir: paths.dataDir, + reclaimResetFile: paths.reclaimResetFile, }); } diff --git a/server/src/nest/app-config/data-paths.service.ts b/server/src/nest/app-config/data-paths.service.ts index 5aa1d3a7c..ccbd30c2e 100644 --- a/server/src/nest/app-config/data-paths.service.ts +++ b/server/src/nest/app-config/data-paths.service.ts @@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common'; /** * The data layout for Nest classes: the data directory, uploads, backups, - * scratch space, logs, the key files, the install id and the database file. + * scratch space, logs, the key files, the install id, the reclaim reset + * marker and the database file. * * Providers that read a data path take this one (AdminService, * StorageRegistryService). Code that runs before the container exists or is @@ -24,6 +25,7 @@ export class DataPathsService implements DataPaths { readonly jwtSecretFile: string; readonly encryptionKeyFile: string; readonly installIdFile: string; + readonly reclaimResetFile: string; readonly dbFile: string; constructor() { @@ -36,6 +38,7 @@ export class DataPathsService implements DataPaths { this.jwtSecretFile = paths.jwtSecretFile; this.encryptionKeyFile = paths.encryptionKeyFile; this.installIdFile = paths.installIdFile; + this.reclaimResetFile = paths.reclaimResetFile; this.dbFile = paths.dbFile; } } diff --git a/server/src/nest/backup/backup-archive.ts b/server/src/nest/backup/backup-archive.ts index f3ef2ca71..4dadff107 100644 --- a/server/src/nest/backup/backup-archive.ts +++ b/server/src/nest/backup/backup-archive.ts @@ -135,3 +135,17 @@ export function checkBackupDatabase(extractDir: string): ArchiveRefusal | null { uploadedDb?.close(); } } + +/** + * Leaves `data/.trek-reclaim-reset` (data-paths.ts) before a restore swaps the + * database in. Storage reclaim removes it once it has dropped the state the + * swapped database brought (the deletion switch, the confirmations, the marks, + * the release queue); until then every start does that first. So a restart + * after a reopen that failed, and a restore at boot, never keep that state, + * whichever installation the backup came from. Throws when the file cannot be + * written: the restore then stops before it swaps anything. + */ +export function markReclaimResetPending(file: string): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${new Date().toISOString()}\n`, { mode: 0o600 }); +} diff --git a/server/src/nest/backup/backup.impl.ts b/server/src/nest/backup/backup.impl.ts index 004c10a68..152679aec 100644 --- a/server/src/nest/backup/backup.impl.ts +++ b/server/src/nest/backup/backup.impl.ts @@ -13,7 +13,7 @@ import { storageOwnerMarkerInstall } from '../storage/storage-reclaim.types'; import type { StorageService } from '../storage/storage.service'; import { StorageInvalidKeyError } from '../storage/storage.types'; import { VALID_INTERVALS } from './auto-backup.settings'; -import { extractBackupArchive } from './backup-archive'; +import { extractBackupArchive, markReclaimResetPending } from './backup-archive'; import { RequestContext } from '@mikro-orm/core'; import archiver from 'archiver'; @@ -40,7 +40,7 @@ const describeError = (err: unknown): string => (err instanceof Error ? err.mess // The scratch and key directory. The database itself may live elsewhere // (TREK_DB_FILE); the database port knows where. -const { dataDir, encryptionKeyFile } = resolveDataPaths(); +const { dataDir, encryptionKeyFile, reclaimResetFile } = resolveDataPaths(); const PRECOMPRESSED = /\.(jpe?g|png|webp|gif|heic|heif|avif|mp4|mov|m4v|webm|pdf|zip|gz)$/i; // Compressed upload cap for restore archives. Defaults to 500 MB, raisable via @@ -482,6 +482,9 @@ export async function restoreFromZip({ storage, database }: BackupDeps, zipPath: await database.keepCopyBeforeRestore(); const liveSessions = await readSessionsToCarry(); + // Read again by storage reclaim once the swapped database is open, here or + // on the next start when the reopen below fails. + markReclaimResetPending(reclaimResetFile); try { // Closes the connection, swaps the database in and reopens it, migrating diff --git a/server/src/nest/backup/boot-restore.ts b/server/src/nest/backup/boot-restore.ts index 056bb7a2d..1849b17fa 100644 --- a/server/src/nest/backup/boot-restore.ts +++ b/server/src/nest/backup/boot-restore.ts @@ -1,5 +1,5 @@ import { stageExtractedPluginTrees } from '../plugins/plugin-backup'; -import { checkBackupDatabase, extractBackupArchive } from './backup-archive'; +import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive'; import fs from 'fs'; import path from 'path'; @@ -45,6 +45,8 @@ export interface BootRestorePlan { dbFile: string; /** The data directory, for the scratch space and the at-rest key. */ dataDir: string; + /** `data/.trek-reclaim-reset` (data-paths.ts), left before the database is put in place. */ + reclaimResetFile: string; } export type BootRestoreOutcome = @@ -77,6 +79,9 @@ export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise { + const pending = resetPending(this.dataPaths.reclaimResetFile); const id = this.installId(); - if (id === null) return false; + if (id === null && !pending) return false; const stored = await this.appSettings.getValue(INSTALL_ID_KEY); - if (stored === id) return false; + if (stored === id && !pending) return false; const wasOn = await this.reset(id); - if (stored !== null) { + const off = wasOn ? ', and deletion is off until an admin turns it on' : ''; + if (pending) { + this.clearResetPending(); + logInfo(`[storage] the database was put in place by a restore: its reclaim state was dropped${off}`); + } else if (stored !== null) { logInfo( `[storage] the database names another installation (restored, cloned or copied): its reclaim state was ` + - `dropped${wasOn ? ', and deletion is off until an admin turns it on' : ''}`, + `dropped${off}`, ); } - return stored !== null; + return pending || stored !== null; + } + + /** + * Whether the database is this installation's as `reconcile` leaves it: it + * names `id` and no restore is waiting for its reset. False means a + * reconcile failed or has not run, so nothing may be released. + */ + async databaseIsThis(id: string): Promise { + if (resetPending(this.dataPaths.reclaimResetFile)) return false; + return (await this.appSettings.getValue(INSTALL_ID_KEY)) === id; } /** * The database was swapped (a restore, the demo reset): what ran on the old * one stops, and the new one gets this installation's id and no reclaim - * state. Contained, since a listener that throws fails the reopen. + * state, after which a restore's reset file goes. Contained, since a + * listener that throws fails the reopen. */ async afterReopen(): Promise { this.currentEpoch += 1; try { const id = this.installId(); const wasOn = await withRequestContext(this.orm, () => this.reset(id)); + this.clearResetPending(); if (wasOn) logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on'); } catch (err) { @@ -115,6 +139,15 @@ export class StorageInstallService implements OnApplicationBootstrap { } } + /** Removes a restore's reset file once its reset is done; one left behind only resets again on the next start. */ + private clearResetPending(): void { + try { + fs.rmSync(this.dataPaths.reclaimResetFile, { force: true }); + } catch (err) { + logWarn(`[storage] could not remove the reclaim reset file, the next start resets again: ${messageOf(err)}`); + } + } + /** Drops the reclaim state and writes `id` (when there is one); answers whether deletion was on. */ private async reset(id: string | null): Promise { return await this.uow.transactional(async () => { @@ -169,6 +202,21 @@ function placeOnce(file: string, text: string): void { } } +/** + * Whether a restore left the reset file. A file that cannot be looked at + * counts as there: resetting once more only switches deletion off. + */ +function resetPending(file: string): boolean { + try { + fs.statSync(file); + return true; + } catch (err) { + if (codeOf(err) === 'ENOENT') return false; + logWarn(`[storage] could not look for the reclaim reset file, the reclaim state is reset: ${messageOf(err)}`); + return true; + } +} + /** The id the file holds, 'missing' when there is no file, null when it is unreadable or holds no id. */ function readId(file: string): string | null | 'missing' { try { diff --git a/server/src/nest/storage/storage-reclaim-guard.service.ts b/server/src/nest/storage/storage-reclaim-guard.service.ts index d2abe5a83..266084418 100644 --- a/server/src/nest/storage/storage-reclaim-guard.service.ts +++ b/server/src/nest/storage/storage-reclaim-guard.service.ts @@ -14,8 +14,15 @@ type CategoryRefusal = 'not_reclaimed' | 'sources_missing' | 'shared_location'; /** Why the place a category lives in may not be judged: another installation's, or one nobody can tell. */ type OwnerRefusal = 'foreign_install' | 'owner_unknown'; -/** A verdict the release queue reuses for this long; the nightly pass renews it every night. */ -const OWNER_VERDICT_TTL_MS = 26 * 60 * 60 * 1000; +/** + * How long the release queue trusts a verdict that let a category through: a + * little more than one release interval (ten minutes), so a marker another + * installation left since stops the next tick but one at the latest, and the + * root is listed at most once in that time. + */ +const OWNER_ALLOWED_TTL_MS = 15 * 60 * 1000; +/** A refusal holds until the nightly pass judges again: it only ever keeps objects. */ +const OWNER_REFUSAL_TTL_MS = 26 * 60 * 60 * 1000; /** A failed look at the owner markers is not retried before this. */ const OWNER_FAILURE_TTL_MS = 60 * 60 * 1000; @@ -36,9 +43,14 @@ interface CachedVerdict { * * The owner verdict is cached per category with the location it was judged * at. The pass judges it from the listing it makes anyway; the release queue - * reuses that verdict for a day and lists the category's root itself only - * when there is none, so a release never deletes in a category the pass - * would refuse as another installation's. + * reuses a refusal until the next night, but an allowed verdict only for + * OWNER_ALLOWED_TTL_MS, and lists the category's root itself when there is no + * verdict it may reuse: at most once per category in that window, since a + * fresh verdict is cached again. The root is listed whole, filtered to the + * markers here: the local driver reads a prefix as a folder, so it cannot be + * asked for the `trek-owner-` names alone. So a release never deletes in a + * category another installation's marker has reached for longer than that + * window. */ @Injectable() export class StorageReclaimGuardService { @@ -87,15 +99,16 @@ export class StorageReclaimGuardService { ): Promise { const location = this.locationOf(category); const verdict = await this.judgeOwners(category, owners, installId); - this.verdicts.set(category, { verdict, location, until: now + OWNER_VERDICT_TTL_MS }); + const ttl = verdict === null ? OWNER_ALLOWED_TTL_MS : OWNER_REFUSAL_TTL_MS; + this.verdicts.set(category, { verdict, location, until: now + ttl }); return verdict; } /** - * The owner verdict for a category, as the last judgement left it when that - * holds for where the category lives now; otherwise its root is listed for - * owner markers and judged afresh. A listing that fails reads as - * `owner_unknown` for an hour. + * The owner verdict for a category, as the last judgement left it while that + * is fresh (see OWNER_ALLOWED_TTL_MS) and holds for where the category lives + * now; otherwise its root is listed for owner markers and judged afresh. A + * listing that fails reads as `owner_unknown` for an hour. */ async cachedOwnerVerdict( category: ServedCategory, diff --git a/server/src/nest/storage/storage-release.service.ts b/server/src/nest/storage/storage-release.service.ts index 75c1025f8..2424fafc2 100644 --- a/server/src/nest/storage/storage-release.service.ts +++ b/server/src/nest/storage/storage-release.service.ts @@ -53,7 +53,7 @@ const MAX_KEYS_PER_CALL = 1000; const QUEUE_LIMIT = 50_000; /** Rows per insert: three bound values each. */ const ENQUEUE_BATCH = 100; -/** Rows one tick looks at. */ +/** Rows one tick looks at, shared equally between the categories it may judge. */ const TICK_LIMIT = 200; /** Keys per question to the sources. */ const ASK_BATCH = 50; @@ -67,7 +67,7 @@ export const STORAGE_RELEASE_JOB = 'a storage release'; export interface ReleaseTickResult { /** Why the tick did nothing, or null when it ran. */ - skipped: 'busy' | 'no_install_id' | null; + skipped: 'busy' | 'no_install_id' | 'install_mismatch' | null; deleted: number; /** Rows dropped because a row still names the object, or the instance keeps it. */ kept: number; @@ -96,8 +96,11 @@ class ReleaseStopped extends Error {} * required source is missing, where the category's location overlaps another * one's, or where another installation's owner marker lies (or this one's * cannot be written). It does nothing while a restore or a backup runs or - * another storage job holds the slot, and without an install id. A refused - * category's rows wait, up to QUEUE_MAX_AGE_MS. Releases run whatever the + * another storage job holds the slot, without an install id, and while the + * database names another installation or a restore's reset is still pending + * (StorageInstallService.databaseIsThis). A refused category's rows wait, up + * to QUEUE_MAX_AGE_MS; what is dropped then, or never queued because the queue + * was full, is logged and left to the nightly pass. Releases run whatever the * nightly deletion switch says: they are what a delete always did. Queueing * never fails the caller's request. */ @@ -129,8 +132,12 @@ export class StorageReleaseService { try { const rows = releasable(values, options); if (rows.length === 0) return 0; - if ((await this.queue.size()) + rows.length > QUEUE_LIMIT) { - logWarn(`[storage] the release queue is full, ${rows.length} object(s) are left to the nightly pass`); + const waiting = await this.queue.size(); + if (waiting + rows.length > QUEUE_LIMIT) { + logWarn( + `[storage] the release queue is full (${waiting} of ${QUEUE_LIMIT} waiting): ${rows.length} released ` + + `object(s) are left to the nightly pass, which removes them only once deletion is on`, + ); return 0; } const dueAt = dbNow(new Date(Date.now() + RELEASE_DELAY_MS)); @@ -160,6 +167,8 @@ export class StorageReleaseService { const result: ReleaseTickResult = { skipped: null, deleted: 0, kept: 0 }; const installId = this.install.installId(); if (installId === null) return { ...result, skipped: 'no_install_id' }; + // A reconcile that failed at boot leaves another installation's queue, or a restore's, in place. + if (!(await this.install.databaseIsThis(installId))) return { ...result, skipped: 'install_mismatch' }; if (this.jobs.backupRunning || !this.jobs.claimExclusive(STORAGE_RELEASE_JOB)) return { ...result, skipped: 'busy' }; try { @@ -172,16 +181,11 @@ export class StorageReleaseService { /** @txIndependent see processDue. */ private async drain(installId: string, now: number, epoch: number, result: ReleaseTickResult): Promise { - await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS))); + await this.dropOverdue(now); const due = new Set(await this.queue.dueCategories(dbNow(new Date(now)))); if (due.size === 0) return; const allowed = await this.allowedCategories(due, installId, now); - const rows = await this.queue.dueIn([...allowed.keys()], dbNow(new Date(now)), TICK_LIMIT); - const byCategory = new Map(); - for (const row of rows) { - const category = row.category as ServedCategory; - byCategory.set(category, [...(byCategory.get(category) ?? []), row.key]); - } + const byCategory = await this.dueKeys([...allowed.keys()], now); try { await [...byCategory].reduce>(async (previous, [category, keys]) => { await previous; @@ -193,6 +197,34 @@ export class StorageReleaseService { } } + /** Drops the rows that waited QUEUE_MAX_AGE_MS past their time, their category refused all along. */ + private async dropOverdue(now: number): Promise { + const dropped = await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS))); + if (dropped === 0) return; + logWarn( + `[storage] ${dropped} released object(s) waited 30 days past their time in a category the release may not ` + + `judge: they are left to the nightly pass, which removes them only once deletion is on`, + ); + } + + /** + * The due keys of each category, the longest waiting first, at most an + * equal share of TICK_LIMIT each: a category whose deletes keep failing + * keeps only its own rows waiting, never another's. + */ + private async dueKeys(categories: readonly ServedCategory[], now: number): Promise> { + const byCategory = new Map(); + if (categories.length === 0) return byCategory; + const share = Math.ceil(TICK_LIMIT / categories.length); + await categories.reduce>(async (previous, category) => { + await previous; + const rows = await this.queue.dueIn(category, dbNow(new Date(now)), share); + const keys = rows.map((row) => row.key); + if (keys.length > 0) byCategory.set(category, keys); + }, Promise.resolve()); + return byCategory; + } + /** Of the categories with rows due, the ones the guards let a release judge now, each with the sources to ask. */ private async allowedCategories( due: ReadonlySet, diff --git a/server/tests/unit/app-config/data-paths.test.ts b/server/tests/unit/app-config/data-paths.test.ts index 9ebdd44dc..379f78eeb 100644 --- a/server/tests/unit/app-config/data-paths.test.ts +++ b/server/tests/unit/app-config/data-paths.test.ts @@ -42,6 +42,7 @@ describe('resolveDataPaths', () => { jwtSecretFile: path.join('/srv/trek', 'data', '.jwt_secret'), encryptionKeyFile: path.join('/srv/trek', 'data', '.encryption_key'), installIdFile: path.join('/srv/trek', 'data', '.trek-install-id'), + reclaimResetFile: path.join('/srv/trek', 'data', '.trek-reclaim-reset'), dbFile: path.join('/srv/trek', 'data', 'travel.db'), }); }); diff --git a/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts b/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts index ef68e326a..f394f915a 100644 --- a/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts +++ b/server/tests/unit/db/repositories/StorageReleaseQueue.repository.test.ts @@ -28,19 +28,24 @@ afterAll(async () => { }); describe('StorageReleaseQueueRepository', () => { - it('RELQ-001: queues keys per category and answers the due ones, the longest waiting first, up to a limit', async () => { + it('RELQ-001: queues keys per category and answers the due ones of one category, the longest waiting first, up to a limit', async () => { await queue.enqueue([{ category: 'covers', key: 'b.jpg' }], '2026-01-02 00:00:00'); await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00'); await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-01-01 00:00:00'); await queue.enqueue([{ category: 'covers', key: 'later.jpg' }], '2026-02-01 00:00:00'); expect(await queue.size()).toBe(4); - expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 10)).toEqual([ + expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 10)).toEqual([ { category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' }, { category: 'covers', key: 'b.jpg', due_at: '2026-01-02 00:00:00' }, ]); - expect(await queue.dueIn(['covers', 'places'], '2026-01-15 00:00:00', 1)).toHaveLength(1); - expect(await queue.dueIn([], '2026-01-15 00:00:00', 10)).toEqual([]); - expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 0)).toEqual([]); + expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 1)).toEqual([ + { category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' }, + ]); + expect(await queue.dueIn('places', '2026-01-15 00:00:00', 10)).toEqual([ + { category: 'places', key: 'a.jpg', due_at: '2026-01-01 00:00:00' }, + ]); + expect(await queue.dueIn('files', '2026-01-15 00:00:00', 10)).toEqual([]); + expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 0)).toEqual([]); }); it('RELQ-002: a key queued again waits from the later time, and an empty batch writes nothing', async () => { @@ -48,8 +53,8 @@ describe('StorageReleaseQueueRepository', () => { await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-03-01 00:00:00'); await queue.enqueue([], '2026-04-01 00:00:00'); expect(await queue.size()).toBe(1); - expect(await queue.dueIn(['covers'], '2026-02-01 00:00:00', 10)).toEqual([]); - expect(await queue.dueIn(['covers'], '2026-03-01 00:00:00', 10)).toHaveLength(1); + expect(await queue.dueIn('covers', '2026-02-01 00:00:00', 10)).toEqual([]); + expect(await queue.dueIn('covers', '2026-03-01 00:00:00', 10)).toHaveLength(1); }); it('RELQ-003: answers the categories holding a due row, each once', async () => { @@ -77,7 +82,7 @@ describe('StorageReleaseQueueRepository', () => { expect(await queue.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1); expect(await queue.deleteKeys('covers', [])).toBe(0); expect(await queue.deleteDueBefore('2026-02-01 00:00:00')).toBe(1); - expect(await queue.dueIn(['places'], '2026-03-01 00:00:00', 10)).toEqual([ + expect(await queue.dueIn('places', '2026-03-01 00:00:00', 10)).toEqual([ { category: 'places', key: 'a.jpg', due_at: '2026-03-01 00:00:00' }, ]); }); diff --git a/server/tests/unit/nest/backup.impl.test.ts b/server/tests/unit/nest/backup.impl.test.ts index e808eeb5f..1eb858602 100644 --- a/server/tests/unit/nest/backup.impl.test.ts +++ b/server/tests/unit/nest/backup.impl.test.ts @@ -39,6 +39,7 @@ const fsMock = vi.hoisted(() => ({ createReadStream: vi.fn(), rmSync: vi.fn(), copyFileSync: vi.fn(), + writeFileSync: vi.fn(), renameSync: vi.fn(), cpSync: vi.fn(), // Identity by default: when uploadsDir is a plain directory, realpathSync @@ -1428,6 +1429,35 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => { expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringContaining('the replaced database was kept as')); }); + it('BACKUP-045j: leaves the reclaim reset file before it closes the database, and swaps nothing when it cannot', async () => { + setupSuccessfulExtraction(); + setupAllTablesPresent(); + const callOrder: string[] = []; + fsMock.writeFileSync.mockImplementation((target: string) => { + callOrder.push(`write:${path.basename(String(target))}`); + }); + dbMock.closeDb.mockImplementation(() => { + callOrder.push('closeDb'); + }); + fsMock.unlinkSync.mockReturnValue(undefined); + fsMock.rmSync.mockReturnValue(undefined); + fsMock.existsSync.mockImplementation((p: string) => !String(p).includes('uploads')); + + await restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip'); + + expect(callOrder.slice(0, 2)).toEqual(['write:.trek-reclaim-reset', 'closeDb']); + + setupSuccessfulExtraction(); + setupAllTablesPresent(); + dbMock.closeDb.mockClear(); + fsMock.writeFileSync.mockImplementation(() => { + throw new Error('read-only file system'); + }); + await expect(restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip')).rejects.toThrow('read-only file system'); + expect(dbMock.closeDb).not.toHaveBeenCalled(); + fsMock.writeFileSync.mockReset(); + }); + it('BACKUP-045i: swaps the archive into the database the connection runs on when TREK_DB_FILE moves it', async () => { // The restore used to write data/travel.db whatever TREK_DB_FILE said: the // live database was never replaced and the restore reported success. diff --git a/server/tests/unit/nest/boot-restore.test.ts b/server/tests/unit/nest/boot-restore.test.ts index 3b12662fd..c3041e721 100644 --- a/server/tests/unit/nest/boot-restore.test.ts +++ b/server/tests/unit/nest/boot-restore.test.ts @@ -1,5 +1,5 @@ /** - * BOOT-RESTORE-001..008 — restoring a backup on a first start (#1089). + * BOOT-RESTORE-001..009 — restoring a backup on a first start (#1089). * * Real files, a real archive and a real SQLite file: what is being protected is * the order of things on disk, and a mocked filesystem would only confirm the @@ -21,12 +21,14 @@ vi.mock('../../../src/nest/plugins/plugin-backup', () => ({ stageExtractedPlugin let root: string; let dataDir: string; let dbFile: string; +let reclaimResetFile: string; beforeEach(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-boot-restore-')); dataDir = path.join(root, 'data'); fs.mkdirSync(dataDir); dbFile = path.join(dataDir, 'travel.db'); + reclaimResetFile = path.join(dataDir, '.trek-reclaim-reset'); stageMock.mockClear(); vi.spyOn(console, 'log').mockImplementation(() => {}); }); @@ -75,14 +77,17 @@ async function goodArchive(extra: Record = {}) { describe('restoreOnFirstBoot', () => { it('BOOT-RESTORE-001: without the variable nothing happens', async () => { - expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir })).toEqual({ restored: false, reason: 'unset' }); + expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir, reclaimResetFile })).toEqual({ + restored: false, + reason: 'unset', + }); expect(fs.existsSync(dbFile)).toBe(false); }); it("BOOT-RESTORE-002: on a first start the backup's database, key and uploads are put in place", async () => { const zipPath = await goodArchive(); - const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir }); + const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile }); expect(out.restored).toBe(true); const db = new Database(dbFile, { readonly: true }); @@ -94,6 +99,8 @@ describe('restoreOnFirstBoot', () => { expect(fs.readFileSync(path.join(out.uploads!, 'covers', 'c.jpg'), 'utf8')).toBe('jpeg'); expect(stageMock).toHaveBeenCalledWith(out.staging); expect(fs.existsSync(`${dbFile}.restore-tmp`)).toBe(false); + // Storage reclaim drops the restored database's reclaim state on this start, whichever install it is from. + expect(fs.existsSync(reclaimResetFile)).toBe(true); }); it('BOOT-RESTORE-003: with a database already there the backup is left alone, so a restart never rolls back', async () => { @@ -102,35 +109,37 @@ describe('restoreOnFirstBoot', () => { const before = fs.readFileSync(dbFile); const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).toEqual({ + expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).toEqual({ restored: false, reason: 'database-exists', }); expect(fs.readFileSync(dbFile).equals(before)).toBe(true); expect(warn).toHaveBeenCalledWith(expect.stringContaining('only restored on a first start')); + expect(fs.existsSync(reclaimResetFile)).toBe(false); }); it('BOOT-RESTORE-004: a missing archive stops the start instead of coming up empty', async () => { - await expect(restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir })).rejects.toBeInstanceOf( - BootRestoreError, - ); + await expect( + restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir, reclaimResetFile }), + ).rejects.toBeInstanceOf(BootRestoreError); expect(fs.existsSync(dbFile)).toBe(false); }); it('BOOT-RESTORE-005: an archive without a database is refused and leaves no scratch behind', async () => { const zipPath = await archive({ 'notes.txt': 'hello' }); - await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow( + await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow( 'RESTORE_FROM_BACKUP: Invalid backup: travel.db not found', ); expect(fs.existsSync(dbFile)).toBe(false); expect(fs.readdirSync(dataDir).filter((n) => n.startsWith('restore-boot-'))).toEqual([]); + expect(fs.existsSync(reclaimResetFile)).toBe(false); }); it('BOOT-RESTORE-006: a database that is not a TREK one is refused', async () => { const src = path.join(root, 'other.db'); trekDb(src, ['users']); const zipPath = await archive({ 'travel.db': { file: src } }); - await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow( + await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow( 'missing required table: trips', ); expect(fs.existsSync(dbFile)).toBe(false); @@ -138,7 +147,7 @@ describe('restoreOnFirstBoot', () => { it('BOOT-RESTORE-007: an entry escaping the archive is refused', async () => { const zipPath = await archive({ '../escape.txt': 'x', 'travel.db': 'not a db' }); - await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow( + await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow( /escapes the archive root|not a valid SQLite database/, ); expect(fs.existsSync(dbFile)).toBe(false); @@ -149,10 +158,18 @@ describe('restoreOnFirstBoot', () => { trekDb(src); const zipPath = await archive({ 'travel.db': { file: src } }); - const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir }); + const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile }); expect(out).toEqual({ restored: true, uploads: null, staging: expect.stringContaining('restore-boot-') }); expect(fs.existsSync(dbFile)).toBe(true); expect(fs.existsSync(path.join(dataDir, '.encryption_key'))).toBe(false); }); + + it('BOOT-RESTORE-009: a reclaim reset file that cannot be written stops the start before the database is put in place', async () => { + const zipPath = await goodArchive(); + // A directory where the file goes: writing it fails, as on a read-only data folder. + fs.mkdirSync(reclaimResetFile); + await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(); + expect(fs.existsSync(dbFile)).toBe(false); + }); }); diff --git a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts index 35f0a5327..03bd24653 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.service.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.service.test.ts @@ -9,7 +9,8 @@ * covers a margin for two weeks and is used up once the share is back under; * an owner marker of another install keeps its hands off; a restore, or a * database naming another install, resets the switch, the marks, the queue - * and the reports but keeps this install's id. RCLS-001 to RCLS-043. + * and the reports but keeps this install's id, and so does a restore's reset + * file whatever id the database names. RCLS-001 to RCLS-045. */ import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; import { AuditLog } from '../../../../src/db/entities/AuditLog.entity'; @@ -19,7 +20,7 @@ import { StorageReclaimMarksRepository } from '../../../../src/db/repositories/S import { dbNow } from '../../../../src/db/types'; import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; -import { logError } from '../../../../src/nest/audit/audit-log.logger'; +import { logError, logWarn } from '../../../../src/nest/audit/audit-log.logger'; import { DomainError } from '../../../../src/nest/common/domain-error'; import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service'; import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service'; @@ -167,7 +168,14 @@ beforeAll(async () => { imports: [StorageModule], init: true, overrides: [ - { provide: DataPathsService, useValue: { ...new DataPathsService(), installIdFile: path.join(dataRoot, 'id') } }, + { + provide: DataPathsService, + useValue: { + ...new DataPathsService(), + installIdFile: path.join(dataRoot, 'id'), + reclaimResetFile: path.join(dataRoot, 'reclaim-reset'), + }, + }, { provide: DatabaseLifecycle, useValue: { @@ -189,6 +197,7 @@ beforeAll(async () => { beforeEach(async () => { vi.restoreAllMocks(); vi.mocked(logError).mockClear(); + vi.mocked(logWarn).mockClear(); for (const entry of fs.readdirSync(uploadsRoot)) { if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true }); } @@ -854,6 +863,59 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => { expect(await countRows(orm, StorageReleaseQueue)).toBe(0); }); + it('RCLS-044: a restore left the reset file: a backup of this installation loses its state too, then it goes', async () => { + const install = t.get(StorageInstallService); + const sentinel = path.join(dataRoot, 'reclaim-reset'); + await markedWeekOld(['orphan.jpg']); + await orm.repo(StorageReleaseQueue).enqueue([{ category: 'covers', key: 'queued.jpg' }], dbNow(new Date())); + // A restore at boot, or a restart after a restore whose reopen failed: the id is this installation's own. + fs.writeFileSync(sentinel, 'pending'); + expect(await install.databaseIsThis(installId)).toBe(false); + expect(await install.reconcile()).toBe(true); + expect(fs.existsSync(sentinel)).toBe(false); + expect(await readAppSetting(orm, 'install_id')).toBe(installId); + expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBeNull(); + expect(await readAppSetting(orm, 'storage.reclaim_report')).toBeNull(); + expect(await marksOf('covers')).toEqual([]); + expect(await countRows(orm, StorageReleaseQueue)).toBe(0); + expect(await install.databaseIsThis(installId)).toBe(true); + // Done once: the next start keeps what this installation does from then on. + await reclaim.setDeleteEnabled(true); + expect(await install.reconcile()).toBe(false); + expect((await reclaim.state()).deleteEnabled).toBe(true); + // A reopen that went through takes the file away with its reset. + fs.writeFileSync(sentinel, 'pending'); + await reopen(); + expect(fs.existsSync(sentinel)).toBe(false); + expect((await reclaim.state()).deleteEnabled).toBe(false); + }); + + it('RCLS-045: a reset file that cannot be removed is reset again on the next start, and the database is not this', async () => { + const install = t.get(StorageInstallService); + const sentinel = path.join(dataRoot, 'reclaim-reset'); + fs.writeFileSync(sentinel, 'pending'); + const realRm = fs.rmSync; + vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (target === sentinel) throw Object.assign(new Error('read-only file system'), { code: 'EROFS' }); + realRm(target, options); + }); + try { + await reclaim.setDeleteEnabled(true); + expect(await install.reconcile()).toBe(true); + expect(fs.existsSync(sentinel)).toBe(true); + expect(await install.databaseIsThis(installId)).toBe(false); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('the next start resets again')); + } finally { + vi.mocked(fs.rmSync).mockRestore(); + fs.rmSync(sentinel, { force: true }); + } + expect(await install.databaseIsThis(installId)).toBe(true); + // A database that names another installation is not this one either. + await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + expect(await install.databaseIsThis(installId)).toBe(false); + expect(await install.reconcile()).toBe(true); + }); + it('RCLS-041: the install id file is made once, mode 0600; a garbled one keeps every delete off', async () => { const file = path.join(dataRoot, 'id'); expect(fs.readFileSync(file, 'utf8').trim()).toBe(installId); diff --git a/server/tests/unit/nest/storage/storage-release.service.test.ts b/server/tests/unit/nest/storage/storage-release.service.test.ts index 7209ffc5b..f2f006c12 100644 --- a/server/tests/unit/nest/storage/storage-release.service.test.ts +++ b/server/tests/unit/nest/storage/storage-release.service.test.ts @@ -5,7 +5,9 @@ * StorageModule (a LocalDriver on a temp folder) with fake sources standing in * for the owner domains. An object any row still names stays; the pass's * guards hold, the owner marker included; a failure never reaches the caller - * and leaves its rows queued. RELS-001 to RELS-021. + * and leaves its rows queued; one failing category never holds up another, and + * a database that is not this installation's releases nothing. RELS-001 to + * RELS-026. */ import { AppSettings } from '../../../../src/db/entities/AppSettings.entity'; import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity'; @@ -121,7 +123,11 @@ beforeAll(async () => { dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-data-')); orm = await createTestOrm(db); await setBackends(); - const paths = { ...new DataPathsService(), installIdFile: path.join(dataRoot, '.trek-install-id') }; + const paths = { + ...new DataPathsService(), + installIdFile: path.join(dataRoot, '.trek-install-id'), + reclaimResetFile: path.join(dataRoot, '.trek-reclaim-reset'), + }; t = await createTestModule({ db, imports: [StorageModule], @@ -441,3 +447,103 @@ describe('StorageReleaseService: the guards the nightly pass keeps', () => { expect(await queued('covers')).toBe(0); }); }); + +describe('StorageReleaseService: what a round-3 review found', () => { + it('RELS-022: a database naming another installation, or a restore whose reset is pending, releases nothing', async () => { + await put('covers', `${UUID}-x.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-x.jpg`]); + await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab'); + try { + expect((await release.processDue(due())).skipped).toBe('install_mismatch'); + } finally { + await setAppSetting(orm, 'install_id', installId); + } + const sentinel = path.join(dataRoot, '.trek-reclaim-reset'); + fs.writeFileSync(sentinel, 'pending'); + try { + expect((await release.processDue(due())).skipped).toBe('install_mismatch'); + } finally { + fs.rmSync(sentinel, { force: true }); + } + expect(exists('covers', `${UUID}-x.jpg`)).toBe(true); + expect(await queued('covers')).toBe(1); + expect(await release.processDue(due())).toEqual({ skipped: null, deleted: 1, kept: 0 }); + }); + + it('RELS-023: a category whose deletes keep failing takes only its share of a tick, the others still go', async () => { + const older = Array.from({ length: 230 }, (_unused, i) => `${UUID}-c${i}.jpg`); + await orm.repo(StorageReleaseQueue).enqueue( + older.map((key) => ({ category: 'covers', key })), + dbNow(new Date(Date.now() - DAY)), + ); + await put('places', `${UUID}-p.jpg`); + await release.releaseValues([`/uploads/places/${UUID}-p.jpg`]); + const realDelete = storage.delete.bind(storage); + vi.spyOn(storage, 'delete').mockImplementation(async (category, key) => { + if (category === 'covers') throw new Error('covers bucket refuses deletes'); + await realDelete(category, key); + }); + // The 230 older covers rows would fill a tick of 200 on their own, every tick. + expect((await release.processDue(due())).deleted).toBe(1); + expect(exists('places', `${UUID}-p.jpg`)).toBe(false); + expect(await queued('places')).toBe(0); + expect(await queued('covers')).toBe(230); + }); + + it('RELS-024: rows dropped after thirty days and a full queue are logged at warn with their counts', async () => { + await orm.repo(StorageReleaseQueue).enqueue( + [ + { category: 'covers', key: `${UUID}-old1.jpg` }, + { category: 'places', key: `${UUID}-old2.jpg` }, + ], + dbNow(new Date(Date.now() - 31 * DAY)), + ); + await release.processDue(new Date()); + expect(logWarn).toHaveBeenCalledWith(expect.stringMatching(/^\[storage\] 2 released object\(s\) waited 30 days/)); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('only once deletion is on')); + vi.mocked(logWarn).mockClear(); + vi.spyOn(StorageReleaseQueueRepository.prototype, 'size').mockResolvedValueOnce(50_000); + expect(await release.releaseValues([`/uploads/covers/${UUID}-full.jpg`])).toBe(0); + expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('(50000 of 50000 waiting): 1 released object(s)')); + expect(await countRows(orm, StorageReleaseQueue)).toBe(0); + }); + + it('RELS-025: an allowed owner verdict holds for one window only, a refusal until the next night', async () => { + const marker = storageOwnerMarkerKey('another-install-0002'); + const lists = vi.spyOn(storage, 'list'); + const start = due().getTime(); + await put('covers', `${UUID}-w1.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-w1.jpg`]); + expect((await release.processDue(new Date(start))).deleted).toBe(1); + expect(lists).toHaveBeenCalledTimes(1); + // Another installation's marker shows up: within the window the verdict stands, without a second listing. + await put('covers', marker); + try { + await put('covers', `${UUID}-w2.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-w2.jpg`]); + expect((await release.processDue(new Date(start + 10 * 60_000))).deleted).toBe(1); + expect(lists).toHaveBeenCalledTimes(1); + // Past it the root is listed again and the category refused. + await put('covers', `${UUID}-w3.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-w3.jpg`]); + expect((await release.processDue(new Date(start + 16 * 60_000))).deleted).toBe(0); + expect(lists).toHaveBeenCalledTimes(2); + } finally { + await storage.delete('covers', marker); + } + // The refusal holds hours later, though the marker is gone, until the nightly pass judges again. + expect((await release.processDue(new Date(start + 6 * 60 * 60_000))).deleted).toBe(0); + expect(lists).toHaveBeenCalledTimes(2); + expect(exists('covers', `${UUID}-w3.jpg`)).toBe(true); + expect((await release.processDue(new Date(start + 27 * 60 * 60_000))).deleted).toBe(1); + expect(exists('covers', `${UUID}-w3.jpg`)).toBe(false); + }); + + it('RELS-026: the owner marker is looked for in the category root only, one shallow listing', async () => { + const lists = vi.spyOn(storage, 'list'); + await put('covers', `${UUID}-root.jpg`); + await release.releaseValues([`/uploads/covers/${UUID}-root.jpg`]); + await release.processDue(due()); + expect(lists.mock.calls).toEqual([['covers', '', { shallow: true }]]); + }); +});