fix(storage): reset reclaim state after any restore, re-check owners often, share release ticks

- A restore (admin panel or RESTORE_FROM_BACKUP) now leaves
  data/.trek-reclaim-reset before it swaps the database in. Boot reconcile
  resets the reclaim state when it finds the file, whatever install id the
  database names, and removes it afterwards; a successful reopen removes it
  too. A restart after a failed reopen, or a boot restore of this
  installation's own backup, no longer keeps the old switch, marks and
  confirmations. The id mismatch reset stays.
- The release job skips a tick (install_mismatch) while the database names
  another installation or a reset is still pending.
- An allowed owner verdict is cached for 15 minutes, a refusal for 26 hours.
  The local driver reads a list prefix as a folder, so the category root is
  still listed whole, at most once per category per window.
- Each tick reads every allowed category's due rows separately, an equal
  share of the 200 each, so a category whose deletes keep failing cannot
  starve the others.
- Rows dropped after 30 days overdue and releases refused by a full queue
  are logged at warn with their counts.
This commit is contained in:
Maurice
2026-11-01 18:31:41 +01:00
parent 9d2ffc91af
commit bef8fc20f1
17 changed files with 432 additions and 80 deletions
+2 -1
View File
@@ -60,7 +60,8 @@ tokens, `RuntimeEnvService`) and consumes the SAME derive functions.
`uploads/` (on `SERVER_ROOT`, the package directory, never on `process.cwd()`)
and names what lives in them: backups, the scratch dir, the logs, the JWT
secret and encryption key files, the install id file the storage reclaim
tells a copied database apart by, and the database file (TREK_DB_FILE when set,
tells a copied database apart by, the reset file a restore leaves for it, and
the database file (TREK_DB_FILE when set,
else `data/travel.db`). Never derive one of these paths from `__dirname` in a
new file: a moved file would quietly move the data.
+9
View File
@@ -35,6 +35,14 @@ export interface DataPaths {
* restored or copied from another install is told apart by it (storage reclaim).
*/
readonly installIdFile: string;
/**
* `data/.trek-reclaim-reset`: left by a restore right before it swaps the
* database in, removed once storage reclaim has dropped the state the swapped
* database brought. A start that finds it resets that state first, so a
* restore whose reopen failed, or one at boot, is covered even when the
* backup came from this installation. Never part of a backup.
*/
readonly reclaimResetFile: string;
/**
* The SQLite file a non-test process opens: TREK_DB_FILE when set, else
* `data/travel.db`. Under NODE_ENV=test the connection is `:memory:` instead
@@ -65,6 +73,7 @@ export function resolveDataPaths(
jwtSecretFile: path.join(dataDir, '.jwt_secret'),
encryptionKeyFile: path.join(dataDir, '.encryption_key'),
installIdFile: path.join(dataDir, '.trek-install-id'),
reclaimResetFile: path.join(dataDir, '.trek-reclaim-reset'),
dbFile: env.db.trekDbFile || path.join(dataDir, 'travel.db'),
};
}
@@ -14,9 +14,9 @@ export interface ReleaseQueueRow {
/**
* `storage_release_queue`: the objects a delete or a replace left unnamed,
* keyed by category and category-relative key, each with the time it may go.
* Owners only add rows; the release job reads the due ones of the categories
* it may judge, asks every reference source again and drops each row once it
* has decided, deleted or not. A key queued again waits from the later time.
* Owners only add rows; the release job reads the due ones of each category
* it may judge (a share of its tick each), asks every reference source again
* and drops each row once it has decided, deleted or not. A key queued again waits from the later time.
*/
export class StorageReleaseQueueRepository extends TrekRepository<StorageReleaseQueue> {
private db() {
@@ -49,15 +49,16 @@ export class StorageReleaseQueueRepository extends TrekRepository<StorageRelease
return rows.map((row) => row.category);
}
/** The rows of these categories due at `now`, the longest waiting first, at most `limit`. */
async dueIn(categories: string[], now: string, limit: number): Promise<ReleaseQueueRow[]> {
if (categories.length === 0 || limit <= 0) return [];
/** The rows of one category due at `now`, the longest waiting first, at most `limit`. */
async dueIn(category: string, now: string, limit: number): Promise<ReleaseQueueRow[]> {
if (limit <= 0) return [];
return await this.db()
.selectFrom('storage_release_queue')
.select(['category', 'key', 'due_at'])
.where('category', 'in', categories)
.where('category', '=', category)
.where('due_at', '<=', now)
.orderBy('due_at')
.orderBy('key')
.limit(limit)
.execute();
}
+3 -1
View File
@@ -138,10 +138,12 @@ async function bootstrap(): Promise<void> {
async function restoreBeforeTheDatabaseOpens() {
const { restoreOnFirstBoot } = require('./nest/backup/boot-restore') as typeof import('./nest/backup/boot-restore');
const env = readEnv();
const paths = resolveDataPaths(env);
return restoreOnFirstBoot({
archive: env.backup.restoreFromBackup,
dbFile: resolveDbPath(),
dataDir: resolveDataPaths(env).dataDir,
dataDir: paths.dataDir,
reclaimResetFile: paths.reclaimResetFile,
});
}
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
/**
* The data layout for Nest classes: the data directory, uploads, backups,
* scratch space, logs, the key files, the install id and the database file.
* scratch space, logs, the key files, the install id, the reclaim reset
* marker and the database file.
*
* Providers that read a data path take this one (AdminService,
* StorageRegistryService). Code that runs before the container exists or is
@@ -24,6 +25,7 @@ export class DataPathsService implements DataPaths {
readonly jwtSecretFile: string;
readonly encryptionKeyFile: string;
readonly installIdFile: string;
readonly reclaimResetFile: string;
readonly dbFile: string;
constructor() {
@@ -36,6 +38,7 @@ export class DataPathsService implements DataPaths {
this.jwtSecretFile = paths.jwtSecretFile;
this.encryptionKeyFile = paths.encryptionKeyFile;
this.installIdFile = paths.installIdFile;
this.reclaimResetFile = paths.reclaimResetFile;
this.dbFile = paths.dbFile;
}
}
+14
View File
@@ -135,3 +135,17 @@ export function checkBackupDatabase(extractDir: string): ArchiveRefusal | null {
uploadedDb?.close();
}
}
/**
* Leaves `data/.trek-reclaim-reset` (data-paths.ts) before a restore swaps the
* database in. Storage reclaim removes it once it has dropped the state the
* swapped database brought (the deletion switch, the confirmations, the marks,
* the release queue); until then every start does that first. So a restart
* after a reopen that failed, and a restore at boot, never keep that state,
* whichever installation the backup came from. Throws when the file cannot be
* written: the restore then stops before it swaps anything.
*/
export function markReclaimResetPending(file: string): void {
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${new Date().toISOString()}\n`, { mode: 0o600 });
}
+5 -2
View File
@@ -13,7 +13,7 @@ import { storageOwnerMarkerInstall } from '../storage/storage-reclaim.types';
import type { StorageService } from '../storage/storage.service';
import { StorageInvalidKeyError } from '../storage/storage.types';
import { VALID_INTERVALS } from './auto-backup.settings';
import { extractBackupArchive } from './backup-archive';
import { extractBackupArchive, markReclaimResetPending } from './backup-archive';
import { RequestContext } from '@mikro-orm/core';
import archiver from 'archiver';
@@ -40,7 +40,7 @@ const describeError = (err: unknown): string => (err instanceof Error ? err.mess
// The scratch and key directory. The database itself may live elsewhere
// (TREK_DB_FILE); the database port knows where.
const { dataDir, encryptionKeyFile } = resolveDataPaths();
const { dataDir, encryptionKeyFile, reclaimResetFile } = resolveDataPaths();
const PRECOMPRESSED = /\.(jpe?g|png|webp|gif|heic|heif|avif|mp4|mov|m4v|webm|pdf|zip|gz)$/i;
// Compressed upload cap for restore archives. Defaults to 500 MB, raisable via
@@ -482,6 +482,9 @@ export async function restoreFromZip({ storage, database }: BackupDeps, zipPath:
await database.keepCopyBeforeRestore();
const liveSessions = await readSessionsToCarry();
// Read again by storage reclaim once the swapped database is open, here or
// on the next start when the reopen below fails.
markReclaimResetPending(reclaimResetFile);
try {
// Closes the connection, swaps the database in and reopens it, migrating
+6 -1
View File
@@ -1,5 +1,5 @@
import { stageExtractedPluginTrees } from '../plugins/plugin-backup';
import { checkBackupDatabase, extractBackupArchive } from './backup-archive';
import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive';
import fs from 'fs';
import path from 'path';
@@ -45,6 +45,8 @@ export interface BootRestorePlan {
dbFile: string;
/** The data directory, for the scratch space and the at-rest key. */
dataDir: string;
/** `data/.trek-reclaim-reset` (data-paths.ts), left before the database is put in place. */
reclaimResetFile: string;
}
export type BootRestoreOutcome =
@@ -77,6 +79,9 @@ export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRes
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable.error}`);
}
// Storage reclaim drops what the restored database knew about its storage on
// the start that follows, even when the backup is this installation's own.
markReclaimResetPending(plan.reclaimResetFile);
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
const tmp = `${plan.dbFile}.restore-tmp`;
@@ -24,20 +24,26 @@ const INSTALL_ID = /^[a-z0-9-]{8,64}$/;
* (`data/.trek-install-id`, made on the first start with a random id, mode
* 0600 like the at-rest key). No backup archive carries it and no restore
* writes it, so it stays with the data dir while a database can come from
* anywhere: a backup restore (in the app or at boot), a restart after a
* restore that failed half way, a copied `travel.db`.
* anywhere.
*
* At boot, and after every reopen of the database, the database is reconciled
* with it: a database that names another id, or none, gets this one, and the
* reclaim state it brought (the deletion switch, the confirmations, both
* reports, the marks and the release queue) is dropped, since it describes
* another installation's storage. After a reopen that happens whatever the id
* says, as before: the rows a restore brings are an older history even when
* they are this installation's own.
* The reclaim state a database holds (the deletion switch, the confirmations,
* both reports, the marks and the release queue) is dropped, and the database
* gets this installation's id, in exactly these cases:
* - at boot, when the database names another id or none: a copied `travel.db`,
* a backup of another installation, a database from before install ids;
* - at boot, when a restore left `data/.trek-reclaim-reset` (data-paths.ts)
* behind, whatever the id says: a restore at boot (RESTORE_FROM_BACKUP), and
* a restore in the admin panel whose reopen failed or that the process did
* not outlive. The file goes once the state is dropped;
* - after every reopen of the database (a restore in the admin panel, the demo
* reset), whatever the id says: the rows a restore brings are an older
* history even when they are this installation's own.
*
* What it cannot tell apart is a data dir copied whole, the file with it: the
* wiki asks to give a clone its own data dir or delete the file. When the file
* can be neither read nor written, there is no id: nothing is deleted then.
* What it cannot tell apart is a database of this installation put in place
* by hand while TREK was stopped, and a data dir copied whole, the file with
* it: the wiki asks to give a clone its own data dir or delete the file. When
* the file can be neither read nor written, there is no id: nothing is deleted
* then.
*/
@Injectable()
export class StorageInstallService implements OnApplicationBootstrap {
@@ -80,34 +86,52 @@ export class StorageInstallService implements OnApplicationBootstrap {
/**
* Makes the database name this installation. One that names another, or
* none, loses its reclaim state first. Answers whether it reset something
* a database had brought along.
* none, loses its reclaim state first, and so does any database a restore
* left the reset file for. Answers whether it reset something a database had
* brought along.
*/
async reconcile(): Promise<boolean> {
const pending = resetPending(this.dataPaths.reclaimResetFile);
const id = this.installId();
if (id === null) return false;
if (id === null && !pending) return false;
const stored = await this.appSettings.getValue(INSTALL_ID_KEY);
if (stored === id) return false;
if (stored === id && !pending) return false;
const wasOn = await this.reset(id);
if (stored !== null) {
const off = wasOn ? ', and deletion is off until an admin turns it on' : '';
if (pending) {
this.clearResetPending();
logInfo(`[storage] the database was put in place by a restore: its reclaim state was dropped${off}`);
} else if (stored !== null) {
logInfo(
`[storage] the database names another installation (restored, cloned or copied): its reclaim state was ` +
`dropped${wasOn ? ', and deletion is off until an admin turns it on' : ''}`,
`dropped${off}`,
);
}
return stored !== null;
return pending || stored !== null;
}
/**
* Whether the database is this installation's as `reconcile` leaves it: it
* names `id` and no restore is waiting for its reset. False means a
* reconcile failed or has not run, so nothing may be released.
*/
async databaseIsThis(id: string): Promise<boolean> {
if (resetPending(this.dataPaths.reclaimResetFile)) return false;
return (await this.appSettings.getValue(INSTALL_ID_KEY)) === id;
}
/**
* The database was swapped (a restore, the demo reset): what ran on the old
* one stops, and the new one gets this installation's id and no reclaim
* state. Contained, since a listener that throws fails the reopen.
* state, after which a restore's reset file goes. Contained, since a
* listener that throws fails the reopen.
*/
async afterReopen(): Promise<void> {
this.currentEpoch += 1;
try {
const id = this.installId();
const wasOn = await withRequestContext(this.orm, () => this.reset(id));
this.clearResetPending();
if (wasOn)
logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on');
} catch (err) {
@@ -115,6 +139,15 @@ export class StorageInstallService implements OnApplicationBootstrap {
}
}
/** Removes a restore's reset file once its reset is done; one left behind only resets again on the next start. */
private clearResetPending(): void {
try {
fs.rmSync(this.dataPaths.reclaimResetFile, { force: true });
} catch (err) {
logWarn(`[storage] could not remove the reclaim reset file, the next start resets again: ${messageOf(err)}`);
}
}
/** Drops the reclaim state and writes `id` (when there is one); answers whether deletion was on. */
private async reset(id: string | null): Promise<boolean> {
return await this.uow.transactional(async () => {
@@ -169,6 +202,21 @@ function placeOnce(file: string, text: string): void {
}
}
/**
* Whether a restore left the reset file. A file that cannot be looked at
* counts as there: resetting once more only switches deletion off.
*/
function resetPending(file: string): boolean {
try {
fs.statSync(file);
return true;
} catch (err) {
if (codeOf(err) === 'ENOENT') return false;
logWarn(`[storage] could not look for the reclaim reset file, the reclaim state is reset: ${messageOf(err)}`);
return true;
}
}
/** The id the file holds, 'missing' when there is no file, null when it is unreadable or holds no id. */
function readId(file: string): string | null | 'missing' {
try {
@@ -14,8 +14,15 @@ type CategoryRefusal = 'not_reclaimed' | 'sources_missing' | 'shared_location';
/** Why the place a category lives in may not be judged: another installation's, or one nobody can tell. */
type OwnerRefusal = 'foreign_install' | 'owner_unknown';
/** A verdict the release queue reuses for this long; the nightly pass renews it every night. */
const OWNER_VERDICT_TTL_MS = 26 * 60 * 60 * 1000;
/**
* How long the release queue trusts a verdict that let a category through: a
* little more than one release interval (ten minutes), so a marker another
* installation left since stops the next tick but one at the latest, and the
* root is listed at most once in that time.
*/
const OWNER_ALLOWED_TTL_MS = 15 * 60 * 1000;
/** A refusal holds until the nightly pass judges again: it only ever keeps objects. */
const OWNER_REFUSAL_TTL_MS = 26 * 60 * 60 * 1000;
/** A failed look at the owner markers is not retried before this. */
const OWNER_FAILURE_TTL_MS = 60 * 60 * 1000;
@@ -36,9 +43,14 @@ interface CachedVerdict {
*
* The owner verdict is cached per category with the location it was judged
* at. The pass judges it from the listing it makes anyway; the release queue
* reuses that verdict for a day and lists the category's root itself only
* when there is none, so a release never deletes in a category the pass
* would refuse as another installation's.
* reuses a refusal until the next night, but an allowed verdict only for
* OWNER_ALLOWED_TTL_MS, and lists the category's root itself when there is no
* verdict it may reuse: at most once per category in that window, since a
* fresh verdict is cached again. The root is listed whole, filtered to the
* markers here: the local driver reads a prefix as a folder, so it cannot be
* asked for the `trek-owner-` names alone. So a release never deletes in a
* category another installation's marker has reached for longer than that
* window.
*/
@Injectable()
export class StorageReclaimGuardService {
@@ -87,15 +99,16 @@ export class StorageReclaimGuardService {
): Promise<OwnerRefusal | null> {
const location = this.locationOf(category);
const verdict = await this.judgeOwners(category, owners, installId);
this.verdicts.set(category, { verdict, location, until: now + OWNER_VERDICT_TTL_MS });
const ttl = verdict === null ? OWNER_ALLOWED_TTL_MS : OWNER_REFUSAL_TTL_MS;
this.verdicts.set(category, { verdict, location, until: now + ttl });
return verdict;
}
/**
* The owner verdict for a category, as the last judgement left it when that
* holds for where the category lives now; otherwise its root is listed for
* owner markers and judged afresh. A listing that fails reads as
* `owner_unknown` for an hour.
* The owner verdict for a category, as the last judgement left it while that
* is fresh (see OWNER_ALLOWED_TTL_MS) and holds for where the category lives
* now; otherwise its root is listed for owner markers and judged afresh. A
* listing that fails reads as `owner_unknown` for an hour.
*/
async cachedOwnerVerdict(
category: ServedCategory,
@@ -53,7 +53,7 @@ const MAX_KEYS_PER_CALL = 1000;
const QUEUE_LIMIT = 50_000;
/** Rows per insert: three bound values each. */
const ENQUEUE_BATCH = 100;
/** Rows one tick looks at. */
/** Rows one tick looks at, shared equally between the categories it may judge. */
const TICK_LIMIT = 200;
/** Keys per question to the sources. */
const ASK_BATCH = 50;
@@ -67,7 +67,7 @@ export const STORAGE_RELEASE_JOB = 'a storage release';
export interface ReleaseTickResult {
/** Why the tick did nothing, or null when it ran. */
skipped: 'busy' | 'no_install_id' | null;
skipped: 'busy' | 'no_install_id' | 'install_mismatch' | null;
deleted: number;
/** Rows dropped because a row still names the object, or the instance keeps it. */
kept: number;
@@ -96,8 +96,11 @@ class ReleaseStopped extends Error {}
* required source is missing, where the category's location overlaps another
* one's, or where another installation's owner marker lies (or this one's
* cannot be written). It does nothing while a restore or a backup runs or
* another storage job holds the slot, and without an install id. A refused
* category's rows wait, up to QUEUE_MAX_AGE_MS. Releases run whatever the
* another storage job holds the slot, without an install id, and while the
* database names another installation or a restore's reset is still pending
* (StorageInstallService.databaseIsThis). A refused category's rows wait, up
* to QUEUE_MAX_AGE_MS; what is dropped then, or never queued because the queue
* was full, is logged and left to the nightly pass. Releases run whatever the
* nightly deletion switch says: they are what a delete always did. Queueing
* never fails the caller's request.
*/
@@ -129,8 +132,12 @@ export class StorageReleaseService {
try {
const rows = releasable(values, options);
if (rows.length === 0) return 0;
if ((await this.queue.size()) + rows.length > QUEUE_LIMIT) {
logWarn(`[storage] the release queue is full, ${rows.length} object(s) are left to the nightly pass`);
const waiting = await this.queue.size();
if (waiting + rows.length > QUEUE_LIMIT) {
logWarn(
`[storage] the release queue is full (${waiting} of ${QUEUE_LIMIT} waiting): ${rows.length} released ` +
`object(s) are left to the nightly pass, which removes them only once deletion is on`,
);
return 0;
}
const dueAt = dbNow(new Date(Date.now() + RELEASE_DELAY_MS));
@@ -160,6 +167,8 @@ export class StorageReleaseService {
const result: ReleaseTickResult = { skipped: null, deleted: 0, kept: 0 };
const installId = this.install.installId();
if (installId === null) return { ...result, skipped: 'no_install_id' };
// A reconcile that failed at boot leaves another installation's queue, or a restore's, in place.
if (!(await this.install.databaseIsThis(installId))) return { ...result, skipped: 'install_mismatch' };
if (this.jobs.backupRunning || !this.jobs.claimExclusive(STORAGE_RELEASE_JOB))
return { ...result, skipped: 'busy' };
try {
@@ -172,16 +181,11 @@ export class StorageReleaseService {
/** @txIndependent see processDue. */
private async drain(installId: string, now: number, epoch: number, result: ReleaseTickResult): Promise<void> {
await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS)));
await this.dropOverdue(now);
const due = new Set(await this.queue.dueCategories(dbNow(new Date(now))));
if (due.size === 0) return;
const allowed = await this.allowedCategories(due, installId, now);
const rows = await this.queue.dueIn([...allowed.keys()], dbNow(new Date(now)), TICK_LIMIT);
const byCategory = new Map<ServedCategory, string[]>();
for (const row of rows) {
const category = row.category as ServedCategory;
byCategory.set(category, [...(byCategory.get(category) ?? []), row.key]);
}
const byCategory = await this.dueKeys([...allowed.keys()], now);
try {
await [...byCategory].reduce<Promise<void>>(async (previous, [category, keys]) => {
await previous;
@@ -193,6 +197,34 @@ export class StorageReleaseService {
}
}
/** Drops the rows that waited QUEUE_MAX_AGE_MS past their time, their category refused all along. */
private async dropOverdue(now: number): Promise<void> {
const dropped = await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS)));
if (dropped === 0) return;
logWarn(
`[storage] ${dropped} released object(s) waited 30 days past their time in a category the release may not ` +
`judge: they are left to the nightly pass, which removes them only once deletion is on`,
);
}
/**
* The due keys of each category, the longest waiting first, at most an
* equal share of TICK_LIMIT each: a category whose deletes keep failing
* keeps only its own rows waiting, never another's.
*/
private async dueKeys(categories: readonly ServedCategory[], now: number): Promise<Map<ServedCategory, string[]>> {
const byCategory = new Map<ServedCategory, string[]>();
if (categories.length === 0) return byCategory;
const share = Math.ceil(TICK_LIMIT / categories.length);
await categories.reduce<Promise<void>>(async (previous, category) => {
await previous;
const rows = await this.queue.dueIn(category, dbNow(new Date(now)), share);
const keys = rows.map((row) => row.key);
if (keys.length > 0) byCategory.set(category, keys);
}, Promise.resolve());
return byCategory;
}
/** Of the categories with rows due, the ones the guards let a release judge now, each with the sources to ask. */
private async allowedCategories(
due: ReadonlySet<string>,
@@ -42,6 +42,7 @@ describe('resolveDataPaths', () => {
jwtSecretFile: path.join('/srv/trek', 'data', '.jwt_secret'),
encryptionKeyFile: path.join('/srv/trek', 'data', '.encryption_key'),
installIdFile: path.join('/srv/trek', 'data', '.trek-install-id'),
reclaimResetFile: path.join('/srv/trek', 'data', '.trek-reclaim-reset'),
dbFile: path.join('/srv/trek', 'data', 'travel.db'),
});
});
@@ -28,19 +28,24 @@ afterAll(async () => {
});
describe('StorageReleaseQueueRepository', () => {
it('RELQ-001: queues keys per category and answers the due ones, the longest waiting first, up to a limit', async () => {
it('RELQ-001: queues keys per category and answers the due ones of one category, the longest waiting first, up to a limit', async () => {
await queue.enqueue([{ category: 'covers', key: 'b.jpg' }], '2026-01-02 00:00:00');
await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00');
await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-01-01 00:00:00');
await queue.enqueue([{ category: 'covers', key: 'later.jpg' }], '2026-02-01 00:00:00');
expect(await queue.size()).toBe(4);
expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 10)).toEqual([
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 10)).toEqual([
{ category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
{ category: 'covers', key: 'b.jpg', due_at: '2026-01-02 00:00:00' },
]);
expect(await queue.dueIn(['covers', 'places'], '2026-01-15 00:00:00', 1)).toHaveLength(1);
expect(await queue.dueIn([], '2026-01-15 00:00:00', 10)).toEqual([]);
expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 0)).toEqual([]);
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 1)).toEqual([
{ category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
]);
expect(await queue.dueIn('places', '2026-01-15 00:00:00', 10)).toEqual([
{ category: 'places', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
]);
expect(await queue.dueIn('files', '2026-01-15 00:00:00', 10)).toEqual([]);
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 0)).toEqual([]);
});
it('RELQ-002: a key queued again waits from the later time, and an empty batch writes nothing', async () => {
@@ -48,8 +53,8 @@ describe('StorageReleaseQueueRepository', () => {
await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-03-01 00:00:00');
await queue.enqueue([], '2026-04-01 00:00:00');
expect(await queue.size()).toBe(1);
expect(await queue.dueIn(['covers'], '2026-02-01 00:00:00', 10)).toEqual([]);
expect(await queue.dueIn(['covers'], '2026-03-01 00:00:00', 10)).toHaveLength(1);
expect(await queue.dueIn('covers', '2026-02-01 00:00:00', 10)).toEqual([]);
expect(await queue.dueIn('covers', '2026-03-01 00:00:00', 10)).toHaveLength(1);
});
it('RELQ-003: answers the categories holding a due row, each once', async () => {
@@ -77,7 +82,7 @@ describe('StorageReleaseQueueRepository', () => {
expect(await queue.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1);
expect(await queue.deleteKeys('covers', [])).toBe(0);
expect(await queue.deleteDueBefore('2026-02-01 00:00:00')).toBe(1);
expect(await queue.dueIn(['places'], '2026-03-01 00:00:00', 10)).toEqual([
expect(await queue.dueIn('places', '2026-03-01 00:00:00', 10)).toEqual([
{ category: 'places', key: 'a.jpg', due_at: '2026-03-01 00:00:00' },
]);
});
@@ -39,6 +39,7 @@ const fsMock = vi.hoisted(() => ({
createReadStream: vi.fn(),
rmSync: vi.fn(),
copyFileSync: vi.fn(),
writeFileSync: vi.fn(),
renameSync: vi.fn(),
cpSync: vi.fn(),
// Identity by default: when uploadsDir is a plain directory, realpathSync
@@ -1428,6 +1429,35 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => {
expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringContaining('the replaced database was kept as'));
});
it('BACKUP-045j: leaves the reclaim reset file before it closes the database, and swaps nothing when it cannot', async () => {
setupSuccessfulExtraction();
setupAllTablesPresent();
const callOrder: string[] = [];
fsMock.writeFileSync.mockImplementation((target: string) => {
callOrder.push(`write:${path.basename(String(target))}`);
});
dbMock.closeDb.mockImplementation(() => {
callOrder.push('closeDb');
});
fsMock.unlinkSync.mockReturnValue(undefined);
fsMock.rmSync.mockReturnValue(undefined);
fsMock.existsSync.mockImplementation((p: string) => !String(p).includes('uploads'));
await restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip');
expect(callOrder.slice(0, 2)).toEqual(['write:.trek-reclaim-reset', 'closeDb']);
setupSuccessfulExtraction();
setupAllTablesPresent();
dbMock.closeDb.mockClear();
fsMock.writeFileSync.mockImplementation(() => {
throw new Error('read-only file system');
});
await expect(restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip')).rejects.toThrow('read-only file system');
expect(dbMock.closeDb).not.toHaveBeenCalled();
fsMock.writeFileSync.mockReset();
});
it('BACKUP-045i: swaps the archive into the database the connection runs on when TREK_DB_FILE moves it', async () => {
// The restore used to write data/travel.db whatever TREK_DB_FILE said: the
// live database was never replaced and the restore reported success.
+28 -11
View File
@@ -1,5 +1,5 @@
/**
* BOOT-RESTORE-001..008 — restoring a backup on a first start (#1089).
* BOOT-RESTORE-001..009 — restoring a backup on a first start (#1089).
*
* Real files, a real archive and a real SQLite file: what is being protected is
* the order of things on disk, and a mocked filesystem would only confirm the
@@ -21,12 +21,14 @@ vi.mock('../../../src/nest/plugins/plugin-backup', () => ({ stageExtractedPlugin
let root: string;
let dataDir: string;
let dbFile: string;
let reclaimResetFile: string;
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-boot-restore-'));
dataDir = path.join(root, 'data');
fs.mkdirSync(dataDir);
dbFile = path.join(dataDir, 'travel.db');
reclaimResetFile = path.join(dataDir, '.trek-reclaim-reset');
stageMock.mockClear();
vi.spyOn(console, 'log').mockImplementation(() => {});
});
@@ -75,14 +77,17 @@ async function goodArchive(extra: Record<string, string> = {}) {
describe('restoreOnFirstBoot', () => {
it('BOOT-RESTORE-001: without the variable nothing happens', async () => {
expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir })).toEqual({ restored: false, reason: 'unset' });
expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir, reclaimResetFile })).toEqual({
restored: false,
reason: 'unset',
});
expect(fs.existsSync(dbFile)).toBe(false);
});
it("BOOT-RESTORE-002: on a first start the backup's database, key and uploads are put in place", async () => {
const zipPath = await goodArchive();
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir });
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile });
expect(out.restored).toBe(true);
const db = new Database(dbFile, { readonly: true });
@@ -94,6 +99,8 @@ describe('restoreOnFirstBoot', () => {
expect(fs.readFileSync(path.join(out.uploads!, 'covers', 'c.jpg'), 'utf8')).toBe('jpeg');
expect(stageMock).toHaveBeenCalledWith(out.staging);
expect(fs.existsSync(`${dbFile}.restore-tmp`)).toBe(false);
// Storage reclaim drops the restored database's reclaim state on this start, whichever install it is from.
expect(fs.existsSync(reclaimResetFile)).toBe(true);
});
it('BOOT-RESTORE-003: with a database already there the backup is left alone, so a restart never rolls back', async () => {
@@ -102,35 +109,37 @@ describe('restoreOnFirstBoot', () => {
const before = fs.readFileSync(dbFile);
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).toEqual({
expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).toEqual({
restored: false,
reason: 'database-exists',
});
expect(fs.readFileSync(dbFile).equals(before)).toBe(true);
expect(warn).toHaveBeenCalledWith(expect.stringContaining('only restored on a first start'));
expect(fs.existsSync(reclaimResetFile)).toBe(false);
});
it('BOOT-RESTORE-004: a missing archive stops the start instead of coming up empty', async () => {
await expect(restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir })).rejects.toBeInstanceOf(
BootRestoreError,
);
await expect(
restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir, reclaimResetFile }),
).rejects.toBeInstanceOf(BootRestoreError);
expect(fs.existsSync(dbFile)).toBe(false);
});
it('BOOT-RESTORE-005: an archive without a database is refused and leaves no scratch behind', async () => {
const zipPath = await archive({ 'notes.txt': 'hello' });
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
'RESTORE_FROM_BACKUP: Invalid backup: travel.db not found',
);
expect(fs.existsSync(dbFile)).toBe(false);
expect(fs.readdirSync(dataDir).filter((n) => n.startsWith('restore-boot-'))).toEqual([]);
expect(fs.existsSync(reclaimResetFile)).toBe(false);
});
it('BOOT-RESTORE-006: a database that is not a TREK one is refused', async () => {
const src = path.join(root, 'other.db');
trekDb(src, ['users']);
const zipPath = await archive({ 'travel.db': { file: src } });
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
'missing required table: trips',
);
expect(fs.existsSync(dbFile)).toBe(false);
@@ -138,7 +147,7 @@ describe('restoreOnFirstBoot', () => {
it('BOOT-RESTORE-007: an entry escaping the archive is refused', async () => {
const zipPath = await archive({ '../escape.txt': 'x', 'travel.db': 'not a db' });
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
/escapes the archive root|not a valid SQLite database/,
);
expect(fs.existsSync(dbFile)).toBe(false);
@@ -149,10 +158,18 @@ describe('restoreOnFirstBoot', () => {
trekDb(src);
const zipPath = await archive({ 'travel.db': { file: src } });
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir });
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile });
expect(out).toEqual({ restored: true, uploads: null, staging: expect.stringContaining('restore-boot-') });
expect(fs.existsSync(dbFile)).toBe(true);
expect(fs.existsSync(path.join(dataDir, '.encryption_key'))).toBe(false);
});
it('BOOT-RESTORE-009: a reclaim reset file that cannot be written stops the start before the database is put in place', async () => {
const zipPath = await goodArchive();
// A directory where the file goes: writing it fails, as on a read-only data folder.
fs.mkdirSync(reclaimResetFile);
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow();
expect(fs.existsSync(dbFile)).toBe(false);
});
});
@@ -9,7 +9,8 @@
* covers a margin for two weeks and is used up once the share is back under;
* an owner marker of another install keeps its hands off; a restore, or a
* database naming another install, resets the switch, the marks, the queue
* and the reports but keeps this install's id. RCLS-001 to RCLS-043.
* and the reports but keeps this install's id, and so does a restore's reset
* file whatever id the database names. RCLS-001 to RCLS-045.
*/
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
import { AuditLog } from '../../../../src/db/entities/AuditLog.entity';
@@ -19,7 +20,7 @@ import { StorageReclaimMarksRepository } from '../../../../src/db/repositories/S
import { dbNow } from '../../../../src/db/types';
import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service';
import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
import { logError } from '../../../../src/nest/audit/audit-log.logger';
import { logError, logWarn } from '../../../../src/nest/audit/audit-log.logger';
import { DomainError } from '../../../../src/nest/common/domain-error';
import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service';
import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service';
@@ -167,7 +168,14 @@ beforeAll(async () => {
imports: [StorageModule],
init: true,
overrides: [
{ provide: DataPathsService, useValue: { ...new DataPathsService(), installIdFile: path.join(dataRoot, 'id') } },
{
provide: DataPathsService,
useValue: {
...new DataPathsService(),
installIdFile: path.join(dataRoot, 'id'),
reclaimResetFile: path.join(dataRoot, 'reclaim-reset'),
},
},
{
provide: DatabaseLifecycle,
useValue: {
@@ -189,6 +197,7 @@ beforeAll(async () => {
beforeEach(async () => {
vi.restoreAllMocks();
vi.mocked(logError).mockClear();
vi.mocked(logWarn).mockClear();
for (const entry of fs.readdirSync(uploadsRoot)) {
if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true });
}
@@ -854,6 +863,59 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => {
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
});
it('RCLS-044: a restore left the reset file: a backup of this installation loses its state too, then it goes', async () => {
const install = t.get(StorageInstallService);
const sentinel = path.join(dataRoot, 'reclaim-reset');
await markedWeekOld(['orphan.jpg']);
await orm.repo(StorageReleaseQueue).enqueue([{ category: 'covers', key: 'queued.jpg' }], dbNow(new Date()));
// A restore at boot, or a restart after a restore whose reopen failed: the id is this installation's own.
fs.writeFileSync(sentinel, 'pending');
expect(await install.databaseIsThis(installId)).toBe(false);
expect(await install.reconcile()).toBe(true);
expect(fs.existsSync(sentinel)).toBe(false);
expect(await readAppSetting(orm, 'install_id')).toBe(installId);
expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBeNull();
expect(await readAppSetting(orm, 'storage.reclaim_report')).toBeNull();
expect(await marksOf('covers')).toEqual([]);
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
expect(await install.databaseIsThis(installId)).toBe(true);
// Done once: the next start keeps what this installation does from then on.
await reclaim.setDeleteEnabled(true);
expect(await install.reconcile()).toBe(false);
expect((await reclaim.state()).deleteEnabled).toBe(true);
// A reopen that went through takes the file away with its reset.
fs.writeFileSync(sentinel, 'pending');
await reopen();
expect(fs.existsSync(sentinel)).toBe(false);
expect((await reclaim.state()).deleteEnabled).toBe(false);
});
it('RCLS-045: a reset file that cannot be removed is reset again on the next start, and the database is not this', async () => {
const install = t.get(StorageInstallService);
const sentinel = path.join(dataRoot, 'reclaim-reset');
fs.writeFileSync(sentinel, 'pending');
const realRm = fs.rmSync;
vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => {
if (target === sentinel) throw Object.assign(new Error('read-only file system'), { code: 'EROFS' });
realRm(target, options);
});
try {
await reclaim.setDeleteEnabled(true);
expect(await install.reconcile()).toBe(true);
expect(fs.existsSync(sentinel)).toBe(true);
expect(await install.databaseIsThis(installId)).toBe(false);
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('the next start resets again'));
} finally {
vi.mocked(fs.rmSync).mockRestore();
fs.rmSync(sentinel, { force: true });
}
expect(await install.databaseIsThis(installId)).toBe(true);
// A database that names another installation is not this one either.
await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab');
expect(await install.databaseIsThis(installId)).toBe(false);
expect(await install.reconcile()).toBe(true);
});
it('RCLS-041: the install id file is made once, mode 0600; a garbled one keeps every delete off', async () => {
const file = path.join(dataRoot, 'id');
expect(fs.readFileSync(file, 'utf8').trim()).toBe(installId);
@@ -5,7 +5,9 @@
* StorageModule (a LocalDriver on a temp folder) with fake sources standing in
* for the owner domains. An object any row still names stays; the pass's
* guards hold, the owner marker included; a failure never reaches the caller
* and leaves its rows queued. RELS-001 to RELS-021.
* and leaves its rows queued; one failing category never holds up another, and
* a database that is not this installation's releases nothing. RELS-001 to
* RELS-026.
*/
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity';
@@ -121,7 +123,11 @@ beforeAll(async () => {
dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-data-'));
orm = await createTestOrm(db);
await setBackends();
const paths = { ...new DataPathsService(), installIdFile: path.join(dataRoot, '.trek-install-id') };
const paths = {
...new DataPathsService(),
installIdFile: path.join(dataRoot, '.trek-install-id'),
reclaimResetFile: path.join(dataRoot, '.trek-reclaim-reset'),
};
t = await createTestModule({
db,
imports: [StorageModule],
@@ -441,3 +447,103 @@ describe('StorageReleaseService: the guards the nightly pass keeps', () => {
expect(await queued('covers')).toBe(0);
});
});
describe('StorageReleaseService: what a round-3 review found', () => {
it('RELS-022: a database naming another installation, or a restore whose reset is pending, releases nothing', async () => {
await put('covers', `${UUID}-x.jpg`);
await release.releaseValues([`/uploads/covers/${UUID}-x.jpg`]);
await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab');
try {
expect((await release.processDue(due())).skipped).toBe('install_mismatch');
} finally {
await setAppSetting(orm, 'install_id', installId);
}
const sentinel = path.join(dataRoot, '.trek-reclaim-reset');
fs.writeFileSync(sentinel, 'pending');
try {
expect((await release.processDue(due())).skipped).toBe('install_mismatch');
} finally {
fs.rmSync(sentinel, { force: true });
}
expect(exists('covers', `${UUID}-x.jpg`)).toBe(true);
expect(await queued('covers')).toBe(1);
expect(await release.processDue(due())).toEqual({ skipped: null, deleted: 1, kept: 0 });
});
it('RELS-023: a category whose deletes keep failing takes only its share of a tick, the others still go', async () => {
const older = Array.from({ length: 230 }, (_unused, i) => `${UUID}-c${i}.jpg`);
await orm.repo(StorageReleaseQueue).enqueue(
older.map((key) => ({ category: 'covers', key })),
dbNow(new Date(Date.now() - DAY)),
);
await put('places', `${UUID}-p.jpg`);
await release.releaseValues([`/uploads/places/${UUID}-p.jpg`]);
const realDelete = storage.delete.bind(storage);
vi.spyOn(storage, 'delete').mockImplementation(async (category, key) => {
if (category === 'covers') throw new Error('covers bucket refuses deletes');
await realDelete(category, key);
});
// The 230 older covers rows would fill a tick of 200 on their own, every tick.
expect((await release.processDue(due())).deleted).toBe(1);
expect(exists('places', `${UUID}-p.jpg`)).toBe(false);
expect(await queued('places')).toBe(0);
expect(await queued('covers')).toBe(230);
});
it('RELS-024: rows dropped after thirty days and a full queue are logged at warn with their counts', async () => {
await orm.repo(StorageReleaseQueue).enqueue(
[
{ category: 'covers', key: `${UUID}-old1.jpg` },
{ category: 'places', key: `${UUID}-old2.jpg` },
],
dbNow(new Date(Date.now() - 31 * DAY)),
);
await release.processDue(new Date());
expect(logWarn).toHaveBeenCalledWith(expect.stringMatching(/^\[storage\] 2 released object\(s\) waited 30 days/));
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('only once deletion is on'));
vi.mocked(logWarn).mockClear();
vi.spyOn(StorageReleaseQueueRepository.prototype, 'size').mockResolvedValueOnce(50_000);
expect(await release.releaseValues([`/uploads/covers/${UUID}-full.jpg`])).toBe(0);
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('(50000 of 50000 waiting): 1 released object(s)'));
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
});
it('RELS-025: an allowed owner verdict holds for one window only, a refusal until the next night', async () => {
const marker = storageOwnerMarkerKey('another-install-0002');
const lists = vi.spyOn(storage, 'list');
const start = due().getTime();
await put('covers', `${UUID}-w1.jpg`);
await release.releaseValues([`/uploads/covers/${UUID}-w1.jpg`]);
expect((await release.processDue(new Date(start))).deleted).toBe(1);
expect(lists).toHaveBeenCalledTimes(1);
// Another installation's marker shows up: within the window the verdict stands, without a second listing.
await put('covers', marker);
try {
await put('covers', `${UUID}-w2.jpg`);
await release.releaseValues([`/uploads/covers/${UUID}-w2.jpg`]);
expect((await release.processDue(new Date(start + 10 * 60_000))).deleted).toBe(1);
expect(lists).toHaveBeenCalledTimes(1);
// Past it the root is listed again and the category refused.
await put('covers', `${UUID}-w3.jpg`);
await release.releaseValues([`/uploads/covers/${UUID}-w3.jpg`]);
expect((await release.processDue(new Date(start + 16 * 60_000))).deleted).toBe(0);
expect(lists).toHaveBeenCalledTimes(2);
} finally {
await storage.delete('covers', marker);
}
// The refusal holds hours later, though the marker is gone, until the nightly pass judges again.
expect((await release.processDue(new Date(start + 6 * 60 * 60_000))).deleted).toBe(0);
expect(lists).toHaveBeenCalledTimes(2);
expect(exists('covers', `${UUID}-w3.jpg`)).toBe(true);
expect((await release.processDue(new Date(start + 27 * 60 * 60_000))).deleted).toBe(1);
expect(exists('covers', `${UUID}-w3.jpg`)).toBe(false);
});
it('RELS-026: the owner marker is looked for in the category root only, one shallow listing', async () => {
const lists = vi.spyOn(storage, 'list');
await put('covers', `${UUID}-root.jpg`);
await release.releaseValues([`/uploads/covers/${UUID}-root.jpg`]);
await release.processDue(due());
expect(lists.mock.calls).toEqual([['covers', '', { shallow: true }]]);
});
});