mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-13 07:40:59 +00:00
fix(storage): reset reclaim state after any restore, re-check owners often, share release ticks
- A restore (admin panel or RESTORE_FROM_BACKUP) now leaves data/.trek-reclaim-reset before it swaps the database in. Boot reconcile resets the reclaim state when it finds the file, whatever install id the database names, and removes it afterwards; a successful reopen removes it too. A restart after a failed reopen, or a boot restore of this installation's own backup, no longer keeps the old switch, marks and confirmations. The id mismatch reset stays. - The release job skips a tick (install_mismatch) while the database names another installation or a reset is still pending. - An allowed owner verdict is cached for 15 minutes, a refusal for 26 hours. The local driver reads a list prefix as a folder, so the category root is still listed whole, at most once per category per window. - Each tick reads every allowed category's due rows separately, an equal share of the 200 each, so a category whose deletes keep failing cannot starve the others. - Rows dropped after 30 days overdue and releases refused by a full queue are logged at warn with their counts.
This commit is contained in:
@@ -60,7 +60,8 @@ tokens, `RuntimeEnvService`) and consumes the SAME derive functions.
|
||||
`uploads/` (on `SERVER_ROOT`, the package directory, never on `process.cwd()`)
|
||||
and names what lives in them: backups, the scratch dir, the logs, the JWT
|
||||
secret and encryption key files, the install id file the storage reclaim
|
||||
tells a copied database apart by, and the database file (TREK_DB_FILE when set,
|
||||
tells a copied database apart by, the reset file a restore leaves for it, and
|
||||
the database file (TREK_DB_FILE when set,
|
||||
else `data/travel.db`). Never derive one of these paths from `__dirname` in a
|
||||
new file: a moved file would quietly move the data.
|
||||
|
||||
|
||||
@@ -35,6 +35,14 @@ export interface DataPaths {
|
||||
* restored or copied from another install is told apart by it (storage reclaim).
|
||||
*/
|
||||
readonly installIdFile: string;
|
||||
/**
|
||||
* `data/.trek-reclaim-reset`: left by a restore right before it swaps the
|
||||
* database in, removed once storage reclaim has dropped the state the swapped
|
||||
* database brought. A start that finds it resets that state first, so a
|
||||
* restore whose reopen failed, or one at boot, is covered even when the
|
||||
* backup came from this installation. Never part of a backup.
|
||||
*/
|
||||
readonly reclaimResetFile: string;
|
||||
/**
|
||||
* The SQLite file a non-test process opens: TREK_DB_FILE when set, else
|
||||
* `data/travel.db`. Under NODE_ENV=test the connection is `:memory:` instead
|
||||
@@ -65,6 +73,7 @@ export function resolveDataPaths(
|
||||
jwtSecretFile: path.join(dataDir, '.jwt_secret'),
|
||||
encryptionKeyFile: path.join(dataDir, '.encryption_key'),
|
||||
installIdFile: path.join(dataDir, '.trek-install-id'),
|
||||
reclaimResetFile: path.join(dataDir, '.trek-reclaim-reset'),
|
||||
dbFile: env.db.trekDbFile || path.join(dataDir, 'travel.db'),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,9 +14,9 @@ export interface ReleaseQueueRow {
|
||||
/**
|
||||
* `storage_release_queue`: the objects a delete or a replace left unnamed,
|
||||
* keyed by category and category-relative key, each with the time it may go.
|
||||
* Owners only add rows; the release job reads the due ones of the categories
|
||||
* it may judge, asks every reference source again and drops each row once it
|
||||
* has decided, deleted or not. A key queued again waits from the later time.
|
||||
* Owners only add rows; the release job reads the due ones of each category
|
||||
* it may judge (a share of its tick each), asks every reference source again
|
||||
* and drops each row once it has decided, deleted or not. A key queued again waits from the later time.
|
||||
*/
|
||||
export class StorageReleaseQueueRepository extends TrekRepository<StorageReleaseQueue> {
|
||||
private db() {
|
||||
@@ -49,15 +49,16 @@ export class StorageReleaseQueueRepository extends TrekRepository<StorageRelease
|
||||
return rows.map((row) => row.category);
|
||||
}
|
||||
|
||||
/** The rows of these categories due at `now`, the longest waiting first, at most `limit`. */
|
||||
async dueIn(categories: string[], now: string, limit: number): Promise<ReleaseQueueRow[]> {
|
||||
if (categories.length === 0 || limit <= 0) return [];
|
||||
/** The rows of one category due at `now`, the longest waiting first, at most `limit`. */
|
||||
async dueIn(category: string, now: string, limit: number): Promise<ReleaseQueueRow[]> {
|
||||
if (limit <= 0) return [];
|
||||
return await this.db()
|
||||
.selectFrom('storage_release_queue')
|
||||
.select(['category', 'key', 'due_at'])
|
||||
.where('category', 'in', categories)
|
||||
.where('category', '=', category)
|
||||
.where('due_at', '<=', now)
|
||||
.orderBy('due_at')
|
||||
.orderBy('key')
|
||||
.limit(limit)
|
||||
.execute();
|
||||
}
|
||||
|
||||
+3
-1
@@ -138,10 +138,12 @@ async function bootstrap(): Promise<void> {
|
||||
async function restoreBeforeTheDatabaseOpens() {
|
||||
const { restoreOnFirstBoot } = require('./nest/backup/boot-restore') as typeof import('./nest/backup/boot-restore');
|
||||
const env = readEnv();
|
||||
const paths = resolveDataPaths(env);
|
||||
return restoreOnFirstBoot({
|
||||
archive: env.backup.restoreFromBackup,
|
||||
dbFile: resolveDbPath(),
|
||||
dataDir: resolveDataPaths(env).dataDir,
|
||||
dataDir: paths.dataDir,
|
||||
reclaimResetFile: paths.reclaimResetFile,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,8 @@ import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* The data layout for Nest classes: the data directory, uploads, backups,
|
||||
* scratch space, logs, the key files, the install id and the database file.
|
||||
* scratch space, logs, the key files, the install id, the reclaim reset
|
||||
* marker and the database file.
|
||||
*
|
||||
* Providers that read a data path take this one (AdminService,
|
||||
* StorageRegistryService). Code that runs before the container exists or is
|
||||
@@ -24,6 +25,7 @@ export class DataPathsService implements DataPaths {
|
||||
readonly jwtSecretFile: string;
|
||||
readonly encryptionKeyFile: string;
|
||||
readonly installIdFile: string;
|
||||
readonly reclaimResetFile: string;
|
||||
readonly dbFile: string;
|
||||
|
||||
constructor() {
|
||||
@@ -36,6 +38,7 @@ export class DataPathsService implements DataPaths {
|
||||
this.jwtSecretFile = paths.jwtSecretFile;
|
||||
this.encryptionKeyFile = paths.encryptionKeyFile;
|
||||
this.installIdFile = paths.installIdFile;
|
||||
this.reclaimResetFile = paths.reclaimResetFile;
|
||||
this.dbFile = paths.dbFile;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,3 +135,17 @@ export function checkBackupDatabase(extractDir: string): ArchiveRefusal | null {
|
||||
uploadedDb?.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Leaves `data/.trek-reclaim-reset` (data-paths.ts) before a restore swaps the
|
||||
* database in. Storage reclaim removes it once it has dropped the state the
|
||||
* swapped database brought (the deletion switch, the confirmations, the marks,
|
||||
* the release queue); until then every start does that first. So a restart
|
||||
* after a reopen that failed, and a restore at boot, never keep that state,
|
||||
* whichever installation the backup came from. Throws when the file cannot be
|
||||
* written: the restore then stops before it swaps anything.
|
||||
*/
|
||||
export function markReclaimResetPending(file: string): void {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, `${new Date().toISOString()}\n`, { mode: 0o600 });
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import { storageOwnerMarkerInstall } from '../storage/storage-reclaim.types';
|
||||
import type { StorageService } from '../storage/storage.service';
|
||||
import { StorageInvalidKeyError } from '../storage/storage.types';
|
||||
import { VALID_INTERVALS } from './auto-backup.settings';
|
||||
import { extractBackupArchive } from './backup-archive';
|
||||
import { extractBackupArchive, markReclaimResetPending } from './backup-archive';
|
||||
import { RequestContext } from '@mikro-orm/core';
|
||||
|
||||
import archiver from 'archiver';
|
||||
@@ -40,7 +40,7 @@ const describeError = (err: unknown): string => (err instanceof Error ? err.mess
|
||||
|
||||
// The scratch and key directory. The database itself may live elsewhere
|
||||
// (TREK_DB_FILE); the database port knows where.
|
||||
const { dataDir, encryptionKeyFile } = resolveDataPaths();
|
||||
const { dataDir, encryptionKeyFile, reclaimResetFile } = resolveDataPaths();
|
||||
const PRECOMPRESSED = /\.(jpe?g|png|webp|gif|heic|heif|avif|mp4|mov|m4v|webm|pdf|zip|gz)$/i;
|
||||
|
||||
// Compressed upload cap for restore archives. Defaults to 500 MB, raisable via
|
||||
@@ -482,6 +482,9 @@ export async function restoreFromZip({ storage, database }: BackupDeps, zipPath:
|
||||
|
||||
await database.keepCopyBeforeRestore();
|
||||
const liveSessions = await readSessionsToCarry();
|
||||
// Read again by storage reclaim once the swapped database is open, here or
|
||||
// on the next start when the reopen below fails.
|
||||
markReclaimResetPending(reclaimResetFile);
|
||||
|
||||
try {
|
||||
// Closes the connection, swaps the database in and reopens it, migrating
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { stageExtractedPluginTrees } from '../plugins/plugin-backup';
|
||||
import { checkBackupDatabase, extractBackupArchive } from './backup-archive';
|
||||
import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive';
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
@@ -45,6 +45,8 @@ export interface BootRestorePlan {
|
||||
dbFile: string;
|
||||
/** The data directory, for the scratch space and the at-rest key. */
|
||||
dataDir: string;
|
||||
/** `data/.trek-reclaim-reset` (data-paths.ts), left before the database is put in place. */
|
||||
reclaimResetFile: string;
|
||||
}
|
||||
|
||||
export type BootRestoreOutcome =
|
||||
@@ -77,6 +79,9 @@ export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRes
|
||||
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable.error}`);
|
||||
}
|
||||
|
||||
// Storage reclaim drops what the restored database knew about its storage on
|
||||
// the start that follows, even when the backup is this installation's own.
|
||||
markReclaimResetPending(plan.reclaimResetFile);
|
||||
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
|
||||
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
|
||||
const tmp = `${plan.dbFile}.restore-tmp`;
|
||||
|
||||
@@ -24,20 +24,26 @@ const INSTALL_ID = /^[a-z0-9-]{8,64}$/;
|
||||
* (`data/.trek-install-id`, made on the first start with a random id, mode
|
||||
* 0600 like the at-rest key). No backup archive carries it and no restore
|
||||
* writes it, so it stays with the data dir while a database can come from
|
||||
* anywhere: a backup restore (in the app or at boot), a restart after a
|
||||
* restore that failed half way, a copied `travel.db`.
|
||||
* anywhere.
|
||||
*
|
||||
* At boot, and after every reopen of the database, the database is reconciled
|
||||
* with it: a database that names another id, or none, gets this one, and the
|
||||
* reclaim state it brought (the deletion switch, the confirmations, both
|
||||
* reports, the marks and the release queue) is dropped, since it describes
|
||||
* another installation's storage. After a reopen that happens whatever the id
|
||||
* says, as before: the rows a restore brings are an older history even when
|
||||
* they are this installation's own.
|
||||
* The reclaim state a database holds (the deletion switch, the confirmations,
|
||||
* both reports, the marks and the release queue) is dropped, and the database
|
||||
* gets this installation's id, in exactly these cases:
|
||||
* - at boot, when the database names another id or none: a copied `travel.db`,
|
||||
* a backup of another installation, a database from before install ids;
|
||||
* - at boot, when a restore left `data/.trek-reclaim-reset` (data-paths.ts)
|
||||
* behind, whatever the id says: a restore at boot (RESTORE_FROM_BACKUP), and
|
||||
* a restore in the admin panel whose reopen failed or that the process did
|
||||
* not outlive. The file goes once the state is dropped;
|
||||
* - after every reopen of the database (a restore in the admin panel, the demo
|
||||
* reset), whatever the id says: the rows a restore brings are an older
|
||||
* history even when they are this installation's own.
|
||||
*
|
||||
* What it cannot tell apart is a data dir copied whole, the file with it: the
|
||||
* wiki asks to give a clone its own data dir or delete the file. When the file
|
||||
* can be neither read nor written, there is no id: nothing is deleted then.
|
||||
* What it cannot tell apart is a database of this installation put in place
|
||||
* by hand while TREK was stopped, and a data dir copied whole, the file with
|
||||
* it: the wiki asks to give a clone its own data dir or delete the file. When
|
||||
* the file can be neither read nor written, there is no id: nothing is deleted
|
||||
* then.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageInstallService implements OnApplicationBootstrap {
|
||||
@@ -80,34 +86,52 @@ export class StorageInstallService implements OnApplicationBootstrap {
|
||||
|
||||
/**
|
||||
* Makes the database name this installation. One that names another, or
|
||||
* none, loses its reclaim state first. Answers whether it reset something
|
||||
* a database had brought along.
|
||||
* none, loses its reclaim state first, and so does any database a restore
|
||||
* left the reset file for. Answers whether it reset something a database had
|
||||
* brought along.
|
||||
*/
|
||||
async reconcile(): Promise<boolean> {
|
||||
const pending = resetPending(this.dataPaths.reclaimResetFile);
|
||||
const id = this.installId();
|
||||
if (id === null) return false;
|
||||
if (id === null && !pending) return false;
|
||||
const stored = await this.appSettings.getValue(INSTALL_ID_KEY);
|
||||
if (stored === id) return false;
|
||||
if (stored === id && !pending) return false;
|
||||
const wasOn = await this.reset(id);
|
||||
if (stored !== null) {
|
||||
const off = wasOn ? ', and deletion is off until an admin turns it on' : '';
|
||||
if (pending) {
|
||||
this.clearResetPending();
|
||||
logInfo(`[storage] the database was put in place by a restore: its reclaim state was dropped${off}`);
|
||||
} else if (stored !== null) {
|
||||
logInfo(
|
||||
`[storage] the database names another installation (restored, cloned or copied): its reclaim state was ` +
|
||||
`dropped${wasOn ? ', and deletion is off until an admin turns it on' : ''}`,
|
||||
`dropped${off}`,
|
||||
);
|
||||
}
|
||||
return stored !== null;
|
||||
return pending || stored !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the database is this installation's as `reconcile` leaves it: it
|
||||
* names `id` and no restore is waiting for its reset. False means a
|
||||
* reconcile failed or has not run, so nothing may be released.
|
||||
*/
|
||||
async databaseIsThis(id: string): Promise<boolean> {
|
||||
if (resetPending(this.dataPaths.reclaimResetFile)) return false;
|
||||
return (await this.appSettings.getValue(INSTALL_ID_KEY)) === id;
|
||||
}
|
||||
|
||||
/**
|
||||
* The database was swapped (a restore, the demo reset): what ran on the old
|
||||
* one stops, and the new one gets this installation's id and no reclaim
|
||||
* state. Contained, since a listener that throws fails the reopen.
|
||||
* state, after which a restore's reset file goes. Contained, since a
|
||||
* listener that throws fails the reopen.
|
||||
*/
|
||||
async afterReopen(): Promise<void> {
|
||||
this.currentEpoch += 1;
|
||||
try {
|
||||
const id = this.installId();
|
||||
const wasOn = await withRequestContext(this.orm, () => this.reset(id));
|
||||
this.clearResetPending();
|
||||
if (wasOn)
|
||||
logInfo('[storage] the restored database had reclaim deletion on: it is off until an admin turns it on');
|
||||
} catch (err) {
|
||||
@@ -115,6 +139,15 @@ export class StorageInstallService implements OnApplicationBootstrap {
|
||||
}
|
||||
}
|
||||
|
||||
/** Removes a restore's reset file once its reset is done; one left behind only resets again on the next start. */
|
||||
private clearResetPending(): void {
|
||||
try {
|
||||
fs.rmSync(this.dataPaths.reclaimResetFile, { force: true });
|
||||
} catch (err) {
|
||||
logWarn(`[storage] could not remove the reclaim reset file, the next start resets again: ${messageOf(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drops the reclaim state and writes `id` (when there is one); answers whether deletion was on. */
|
||||
private async reset(id: string | null): Promise<boolean> {
|
||||
return await this.uow.transactional(async () => {
|
||||
@@ -169,6 +202,21 @@ function placeOnce(file: string, text: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a restore left the reset file. A file that cannot be looked at
|
||||
* counts as there: resetting once more only switches deletion off.
|
||||
*/
|
||||
function resetPending(file: string): boolean {
|
||||
try {
|
||||
fs.statSync(file);
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (codeOf(err) === 'ENOENT') return false;
|
||||
logWarn(`[storage] could not look for the reclaim reset file, the reclaim state is reset: ${messageOf(err)}`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/** The id the file holds, 'missing' when there is no file, null when it is unreadable or holds no id. */
|
||||
function readId(file: string): string | null | 'missing' {
|
||||
try {
|
||||
|
||||
@@ -14,8 +14,15 @@ type CategoryRefusal = 'not_reclaimed' | 'sources_missing' | 'shared_location';
|
||||
/** Why the place a category lives in may not be judged: another installation's, or one nobody can tell. */
|
||||
type OwnerRefusal = 'foreign_install' | 'owner_unknown';
|
||||
|
||||
/** A verdict the release queue reuses for this long; the nightly pass renews it every night. */
|
||||
const OWNER_VERDICT_TTL_MS = 26 * 60 * 60 * 1000;
|
||||
/**
|
||||
* How long the release queue trusts a verdict that let a category through: a
|
||||
* little more than one release interval (ten minutes), so a marker another
|
||||
* installation left since stops the next tick but one at the latest, and the
|
||||
* root is listed at most once in that time.
|
||||
*/
|
||||
const OWNER_ALLOWED_TTL_MS = 15 * 60 * 1000;
|
||||
/** A refusal holds until the nightly pass judges again: it only ever keeps objects. */
|
||||
const OWNER_REFUSAL_TTL_MS = 26 * 60 * 60 * 1000;
|
||||
/** A failed look at the owner markers is not retried before this. */
|
||||
const OWNER_FAILURE_TTL_MS = 60 * 60 * 1000;
|
||||
|
||||
@@ -36,9 +43,14 @@ interface CachedVerdict {
|
||||
*
|
||||
* The owner verdict is cached per category with the location it was judged
|
||||
* at. The pass judges it from the listing it makes anyway; the release queue
|
||||
* reuses that verdict for a day and lists the category's root itself only
|
||||
* when there is none, so a release never deletes in a category the pass
|
||||
* would refuse as another installation's.
|
||||
* reuses a refusal until the next night, but an allowed verdict only for
|
||||
* OWNER_ALLOWED_TTL_MS, and lists the category's root itself when there is no
|
||||
* verdict it may reuse: at most once per category in that window, since a
|
||||
* fresh verdict is cached again. The root is listed whole, filtered to the
|
||||
* markers here: the local driver reads a prefix as a folder, so it cannot be
|
||||
* asked for the `trek-owner-` names alone. So a release never deletes in a
|
||||
* category another installation's marker has reached for longer than that
|
||||
* window.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimGuardService {
|
||||
@@ -87,15 +99,16 @@ export class StorageReclaimGuardService {
|
||||
): Promise<OwnerRefusal | null> {
|
||||
const location = this.locationOf(category);
|
||||
const verdict = await this.judgeOwners(category, owners, installId);
|
||||
this.verdicts.set(category, { verdict, location, until: now + OWNER_VERDICT_TTL_MS });
|
||||
const ttl = verdict === null ? OWNER_ALLOWED_TTL_MS : OWNER_REFUSAL_TTL_MS;
|
||||
this.verdicts.set(category, { verdict, location, until: now + ttl });
|
||||
return verdict;
|
||||
}
|
||||
|
||||
/**
|
||||
* The owner verdict for a category, as the last judgement left it when that
|
||||
* holds for where the category lives now; otherwise its root is listed for
|
||||
* owner markers and judged afresh. A listing that fails reads as
|
||||
* `owner_unknown` for an hour.
|
||||
* The owner verdict for a category, as the last judgement left it while that
|
||||
* is fresh (see OWNER_ALLOWED_TTL_MS) and holds for where the category lives
|
||||
* now; otherwise its root is listed for owner markers and judged afresh. A
|
||||
* listing that fails reads as `owner_unknown` for an hour.
|
||||
*/
|
||||
async cachedOwnerVerdict(
|
||||
category: ServedCategory,
|
||||
|
||||
@@ -53,7 +53,7 @@ const MAX_KEYS_PER_CALL = 1000;
|
||||
const QUEUE_LIMIT = 50_000;
|
||||
/** Rows per insert: three bound values each. */
|
||||
const ENQUEUE_BATCH = 100;
|
||||
/** Rows one tick looks at. */
|
||||
/** Rows one tick looks at, shared equally between the categories it may judge. */
|
||||
const TICK_LIMIT = 200;
|
||||
/** Keys per question to the sources. */
|
||||
const ASK_BATCH = 50;
|
||||
@@ -67,7 +67,7 @@ export const STORAGE_RELEASE_JOB = 'a storage release';
|
||||
|
||||
export interface ReleaseTickResult {
|
||||
/** Why the tick did nothing, or null when it ran. */
|
||||
skipped: 'busy' | 'no_install_id' | null;
|
||||
skipped: 'busy' | 'no_install_id' | 'install_mismatch' | null;
|
||||
deleted: number;
|
||||
/** Rows dropped because a row still names the object, or the instance keeps it. */
|
||||
kept: number;
|
||||
@@ -96,8 +96,11 @@ class ReleaseStopped extends Error {}
|
||||
* required source is missing, where the category's location overlaps another
|
||||
* one's, or where another installation's owner marker lies (or this one's
|
||||
* cannot be written). It does nothing while a restore or a backup runs or
|
||||
* another storage job holds the slot, and without an install id. A refused
|
||||
* category's rows wait, up to QUEUE_MAX_AGE_MS. Releases run whatever the
|
||||
* another storage job holds the slot, without an install id, and while the
|
||||
* database names another installation or a restore's reset is still pending
|
||||
* (StorageInstallService.databaseIsThis). A refused category's rows wait, up
|
||||
* to QUEUE_MAX_AGE_MS; what is dropped then, or never queued because the queue
|
||||
* was full, is logged and left to the nightly pass. Releases run whatever the
|
||||
* nightly deletion switch says: they are what a delete always did. Queueing
|
||||
* never fails the caller's request.
|
||||
*/
|
||||
@@ -129,8 +132,12 @@ export class StorageReleaseService {
|
||||
try {
|
||||
const rows = releasable(values, options);
|
||||
if (rows.length === 0) return 0;
|
||||
if ((await this.queue.size()) + rows.length > QUEUE_LIMIT) {
|
||||
logWarn(`[storage] the release queue is full, ${rows.length} object(s) are left to the nightly pass`);
|
||||
const waiting = await this.queue.size();
|
||||
if (waiting + rows.length > QUEUE_LIMIT) {
|
||||
logWarn(
|
||||
`[storage] the release queue is full (${waiting} of ${QUEUE_LIMIT} waiting): ${rows.length} released ` +
|
||||
`object(s) are left to the nightly pass, which removes them only once deletion is on`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
const dueAt = dbNow(new Date(Date.now() + RELEASE_DELAY_MS));
|
||||
@@ -160,6 +167,8 @@ export class StorageReleaseService {
|
||||
const result: ReleaseTickResult = { skipped: null, deleted: 0, kept: 0 };
|
||||
const installId = this.install.installId();
|
||||
if (installId === null) return { ...result, skipped: 'no_install_id' };
|
||||
// A reconcile that failed at boot leaves another installation's queue, or a restore's, in place.
|
||||
if (!(await this.install.databaseIsThis(installId))) return { ...result, skipped: 'install_mismatch' };
|
||||
if (this.jobs.backupRunning || !this.jobs.claimExclusive(STORAGE_RELEASE_JOB))
|
||||
return { ...result, skipped: 'busy' };
|
||||
try {
|
||||
@@ -172,16 +181,11 @@ export class StorageReleaseService {
|
||||
|
||||
/** @txIndependent see processDue. */
|
||||
private async drain(installId: string, now: number, epoch: number, result: ReleaseTickResult): Promise<void> {
|
||||
await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS)));
|
||||
await this.dropOverdue(now);
|
||||
const due = new Set(await this.queue.dueCategories(dbNow(new Date(now))));
|
||||
if (due.size === 0) return;
|
||||
const allowed = await this.allowedCategories(due, installId, now);
|
||||
const rows = await this.queue.dueIn([...allowed.keys()], dbNow(new Date(now)), TICK_LIMIT);
|
||||
const byCategory = new Map<ServedCategory, string[]>();
|
||||
for (const row of rows) {
|
||||
const category = row.category as ServedCategory;
|
||||
byCategory.set(category, [...(byCategory.get(category) ?? []), row.key]);
|
||||
}
|
||||
const byCategory = await this.dueKeys([...allowed.keys()], now);
|
||||
try {
|
||||
await [...byCategory].reduce<Promise<void>>(async (previous, [category, keys]) => {
|
||||
await previous;
|
||||
@@ -193,6 +197,34 @@ export class StorageReleaseService {
|
||||
}
|
||||
}
|
||||
|
||||
/** Drops the rows that waited QUEUE_MAX_AGE_MS past their time, their category refused all along. */
|
||||
private async dropOverdue(now: number): Promise<void> {
|
||||
const dropped = await this.queue.deleteDueBefore(dbNow(new Date(now - QUEUE_MAX_AGE_MS)));
|
||||
if (dropped === 0) return;
|
||||
logWarn(
|
||||
`[storage] ${dropped} released object(s) waited 30 days past their time in a category the release may not ` +
|
||||
`judge: they are left to the nightly pass, which removes them only once deletion is on`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The due keys of each category, the longest waiting first, at most an
|
||||
* equal share of TICK_LIMIT each: a category whose deletes keep failing
|
||||
* keeps only its own rows waiting, never another's.
|
||||
*/
|
||||
private async dueKeys(categories: readonly ServedCategory[], now: number): Promise<Map<ServedCategory, string[]>> {
|
||||
const byCategory = new Map<ServedCategory, string[]>();
|
||||
if (categories.length === 0) return byCategory;
|
||||
const share = Math.ceil(TICK_LIMIT / categories.length);
|
||||
await categories.reduce<Promise<void>>(async (previous, category) => {
|
||||
await previous;
|
||||
const rows = await this.queue.dueIn(category, dbNow(new Date(now)), share);
|
||||
const keys = rows.map((row) => row.key);
|
||||
if (keys.length > 0) byCategory.set(category, keys);
|
||||
}, Promise.resolve());
|
||||
return byCategory;
|
||||
}
|
||||
|
||||
/** Of the categories with rows due, the ones the guards let a release judge now, each with the sources to ask. */
|
||||
private async allowedCategories(
|
||||
due: ReadonlySet<string>,
|
||||
|
||||
@@ -42,6 +42,7 @@ describe('resolveDataPaths', () => {
|
||||
jwtSecretFile: path.join('/srv/trek', 'data', '.jwt_secret'),
|
||||
encryptionKeyFile: path.join('/srv/trek', 'data', '.encryption_key'),
|
||||
installIdFile: path.join('/srv/trek', 'data', '.trek-install-id'),
|
||||
reclaimResetFile: path.join('/srv/trek', 'data', '.trek-reclaim-reset'),
|
||||
dbFile: path.join('/srv/trek', 'data', 'travel.db'),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -28,19 +28,24 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
describe('StorageReleaseQueueRepository', () => {
|
||||
it('RELQ-001: queues keys per category and answers the due ones, the longest waiting first, up to a limit', async () => {
|
||||
it('RELQ-001: queues keys per category and answers the due ones of one category, the longest waiting first, up to a limit', async () => {
|
||||
await queue.enqueue([{ category: 'covers', key: 'b.jpg' }], '2026-01-02 00:00:00');
|
||||
await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-01-01 00:00:00');
|
||||
await queue.enqueue([{ category: 'places', key: 'a.jpg' }], '2026-01-01 00:00:00');
|
||||
await queue.enqueue([{ category: 'covers', key: 'later.jpg' }], '2026-02-01 00:00:00');
|
||||
expect(await queue.size()).toBe(4);
|
||||
expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 10)).toEqual([
|
||||
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 10)).toEqual([
|
||||
{ category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
|
||||
{ category: 'covers', key: 'b.jpg', due_at: '2026-01-02 00:00:00' },
|
||||
]);
|
||||
expect(await queue.dueIn(['covers', 'places'], '2026-01-15 00:00:00', 1)).toHaveLength(1);
|
||||
expect(await queue.dueIn([], '2026-01-15 00:00:00', 10)).toEqual([]);
|
||||
expect(await queue.dueIn(['covers'], '2026-01-15 00:00:00', 0)).toEqual([]);
|
||||
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 1)).toEqual([
|
||||
{ category: 'covers', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
|
||||
]);
|
||||
expect(await queue.dueIn('places', '2026-01-15 00:00:00', 10)).toEqual([
|
||||
{ category: 'places', key: 'a.jpg', due_at: '2026-01-01 00:00:00' },
|
||||
]);
|
||||
expect(await queue.dueIn('files', '2026-01-15 00:00:00', 10)).toEqual([]);
|
||||
expect(await queue.dueIn('covers', '2026-01-15 00:00:00', 0)).toEqual([]);
|
||||
});
|
||||
|
||||
it('RELQ-002: a key queued again waits from the later time, and an empty batch writes nothing', async () => {
|
||||
@@ -48,8 +53,8 @@ describe('StorageReleaseQueueRepository', () => {
|
||||
await queue.enqueue([{ category: 'covers', key: 'a.jpg' }], '2026-03-01 00:00:00');
|
||||
await queue.enqueue([], '2026-04-01 00:00:00');
|
||||
expect(await queue.size()).toBe(1);
|
||||
expect(await queue.dueIn(['covers'], '2026-02-01 00:00:00', 10)).toEqual([]);
|
||||
expect(await queue.dueIn(['covers'], '2026-03-01 00:00:00', 10)).toHaveLength(1);
|
||||
expect(await queue.dueIn('covers', '2026-02-01 00:00:00', 10)).toEqual([]);
|
||||
expect(await queue.dueIn('covers', '2026-03-01 00:00:00', 10)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('RELQ-003: answers the categories holding a due row, each once', async () => {
|
||||
@@ -77,7 +82,7 @@ describe('StorageReleaseQueueRepository', () => {
|
||||
expect(await queue.deleteKeys('covers', ['a.jpg', 'nope.jpg'])).toBe(1);
|
||||
expect(await queue.deleteKeys('covers', [])).toBe(0);
|
||||
expect(await queue.deleteDueBefore('2026-02-01 00:00:00')).toBe(1);
|
||||
expect(await queue.dueIn(['places'], '2026-03-01 00:00:00', 10)).toEqual([
|
||||
expect(await queue.dueIn('places', '2026-03-01 00:00:00', 10)).toEqual([
|
||||
{ category: 'places', key: 'a.jpg', due_at: '2026-03-01 00:00:00' },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -39,6 +39,7 @@ const fsMock = vi.hoisted(() => ({
|
||||
createReadStream: vi.fn(),
|
||||
rmSync: vi.fn(),
|
||||
copyFileSync: vi.fn(),
|
||||
writeFileSync: vi.fn(),
|
||||
renameSync: vi.fn(),
|
||||
cpSync: vi.fn(),
|
||||
// Identity by default: when uploadsDir is a plain directory, realpathSync
|
||||
@@ -1428,6 +1429,35 @@ describe('BACKUP-045 restoreFromZip — full success path (no uploads)', () => {
|
||||
expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringContaining('the replaced database was kept as'));
|
||||
});
|
||||
|
||||
it('BACKUP-045j: leaves the reclaim reset file before it closes the database, and swaps nothing when it cannot', async () => {
|
||||
setupSuccessfulExtraction();
|
||||
setupAllTablesPresent();
|
||||
const callOrder: string[] = [];
|
||||
fsMock.writeFileSync.mockImplementation((target: string) => {
|
||||
callOrder.push(`write:${path.basename(String(target))}`);
|
||||
});
|
||||
dbMock.closeDb.mockImplementation(() => {
|
||||
callOrder.push('closeDb');
|
||||
});
|
||||
fsMock.unlinkSync.mockReturnValue(undefined);
|
||||
fsMock.rmSync.mockReturnValue(undefined);
|
||||
fsMock.existsSync.mockImplementation((p: string) => !String(p).includes('uploads'));
|
||||
|
||||
await restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip');
|
||||
|
||||
expect(callOrder.slice(0, 2)).toEqual(['write:.trek-reclaim-reset', 'closeDb']);
|
||||
|
||||
setupSuccessfulExtraction();
|
||||
setupAllTablesPresent();
|
||||
dbMock.closeDb.mockClear();
|
||||
fsMock.writeFileSync.mockImplementation(() => {
|
||||
throw new Error('read-only file system');
|
||||
});
|
||||
await expect(restoreFromZip(deps(stubStorage()), '/data/tmp/upload.zip')).rejects.toThrow('read-only file system');
|
||||
expect(dbMock.closeDb).not.toHaveBeenCalled();
|
||||
fsMock.writeFileSync.mockReset();
|
||||
});
|
||||
|
||||
it('BACKUP-045i: swaps the archive into the database the connection runs on when TREK_DB_FILE moves it', async () => {
|
||||
// The restore used to write data/travel.db whatever TREK_DB_FILE said: the
|
||||
// live database was never replaced and the restore reported success.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* BOOT-RESTORE-001..008 — restoring a backup on a first start (#1089).
|
||||
* BOOT-RESTORE-001..009 — restoring a backup on a first start (#1089).
|
||||
*
|
||||
* Real files, a real archive and a real SQLite file: what is being protected is
|
||||
* the order of things on disk, and a mocked filesystem would only confirm the
|
||||
@@ -21,12 +21,14 @@ vi.mock('../../../src/nest/plugins/plugin-backup', () => ({ stageExtractedPlugin
|
||||
let root: string;
|
||||
let dataDir: string;
|
||||
let dbFile: string;
|
||||
let reclaimResetFile: string;
|
||||
|
||||
beforeEach(() => {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-boot-restore-'));
|
||||
dataDir = path.join(root, 'data');
|
||||
fs.mkdirSync(dataDir);
|
||||
dbFile = path.join(dataDir, 'travel.db');
|
||||
reclaimResetFile = path.join(dataDir, '.trek-reclaim-reset');
|
||||
stageMock.mockClear();
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
});
|
||||
@@ -75,14 +77,17 @@ async function goodArchive(extra: Record<string, string> = {}) {
|
||||
|
||||
describe('restoreOnFirstBoot', () => {
|
||||
it('BOOT-RESTORE-001: without the variable nothing happens', async () => {
|
||||
expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir })).toEqual({ restored: false, reason: 'unset' });
|
||||
expect(await restoreOnFirstBoot({ archive: null, dbFile, dataDir, reclaimResetFile })).toEqual({
|
||||
restored: false,
|
||||
reason: 'unset',
|
||||
});
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
});
|
||||
|
||||
it("BOOT-RESTORE-002: on a first start the backup's database, key and uploads are put in place", async () => {
|
||||
const zipPath = await goodArchive();
|
||||
|
||||
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir });
|
||||
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile });
|
||||
|
||||
expect(out.restored).toBe(true);
|
||||
const db = new Database(dbFile, { readonly: true });
|
||||
@@ -94,6 +99,8 @@ describe('restoreOnFirstBoot', () => {
|
||||
expect(fs.readFileSync(path.join(out.uploads!, 'covers', 'c.jpg'), 'utf8')).toBe('jpeg');
|
||||
expect(stageMock).toHaveBeenCalledWith(out.staging);
|
||||
expect(fs.existsSync(`${dbFile}.restore-tmp`)).toBe(false);
|
||||
// Storage reclaim drops the restored database's reclaim state on this start, whichever install it is from.
|
||||
expect(fs.existsSync(reclaimResetFile)).toBe(true);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-003: with a database already there the backup is left alone, so a restart never rolls back', async () => {
|
||||
@@ -102,35 +109,37 @@ describe('restoreOnFirstBoot', () => {
|
||||
const before = fs.readFileSync(dbFile);
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
|
||||
expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).toEqual({
|
||||
expect(await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).toEqual({
|
||||
restored: false,
|
||||
reason: 'database-exists',
|
||||
});
|
||||
expect(fs.readFileSync(dbFile).equals(before)).toBe(true);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining('only restored on a first start'));
|
||||
expect(fs.existsSync(reclaimResetFile)).toBe(false);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-004: a missing archive stops the start instead of coming up empty', async () => {
|
||||
await expect(restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir })).rejects.toBeInstanceOf(
|
||||
BootRestoreError,
|
||||
);
|
||||
await expect(
|
||||
restoreOnFirstBoot({ archive: path.join(root, 'nope.zip'), dbFile, dataDir, reclaimResetFile }),
|
||||
).rejects.toBeInstanceOf(BootRestoreError);
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-005: an archive without a database is refused and leaves no scratch behind', async () => {
|
||||
const zipPath = await archive({ 'notes.txt': 'hello' });
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
|
||||
'RESTORE_FROM_BACKUP: Invalid backup: travel.db not found',
|
||||
);
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
expect(fs.readdirSync(dataDir).filter((n) => n.startsWith('restore-boot-'))).toEqual([]);
|
||||
expect(fs.existsSync(reclaimResetFile)).toBe(false);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-006: a database that is not a TREK one is refused', async () => {
|
||||
const src = path.join(root, 'other.db');
|
||||
trekDb(src, ['users']);
|
||||
const zipPath = await archive({ 'travel.db': { file: src } });
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
|
||||
'missing required table: trips',
|
||||
);
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
@@ -138,7 +147,7 @@ describe('restoreOnFirstBoot', () => {
|
||||
|
||||
it('BOOT-RESTORE-007: an entry escaping the archive is refused', async () => {
|
||||
const zipPath = await archive({ '../escape.txt': 'x', 'travel.db': 'not a db' });
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir })).rejects.toThrow(
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow(
|
||||
/escapes the archive root|not a valid SQLite database/,
|
||||
);
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
@@ -149,10 +158,18 @@ describe('restoreOnFirstBoot', () => {
|
||||
trekDb(src);
|
||||
const zipPath = await archive({ 'travel.db': { file: src } });
|
||||
|
||||
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir });
|
||||
const out = await restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile });
|
||||
|
||||
expect(out).toEqual({ restored: true, uploads: null, staging: expect.stringContaining('restore-boot-') });
|
||||
expect(fs.existsSync(dbFile)).toBe(true);
|
||||
expect(fs.existsSync(path.join(dataDir, '.encryption_key'))).toBe(false);
|
||||
});
|
||||
|
||||
it('BOOT-RESTORE-009: a reclaim reset file that cannot be written stops the start before the database is put in place', async () => {
|
||||
const zipPath = await goodArchive();
|
||||
// A directory where the file goes: writing it fails, as on a read-only data folder.
|
||||
fs.mkdirSync(reclaimResetFile);
|
||||
await expect(restoreOnFirstBoot({ archive: zipPath, dbFile, dataDir, reclaimResetFile })).rejects.toThrow();
|
||||
expect(fs.existsSync(dbFile)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
* covers a margin for two weeks and is used up once the share is back under;
|
||||
* an owner marker of another install keeps its hands off; a restore, or a
|
||||
* database naming another install, resets the switch, the marks, the queue
|
||||
* and the reports but keeps this install's id. RCLS-001 to RCLS-043.
|
||||
* and the reports but keeps this install's id, and so does a restore's reset
|
||||
* file whatever id the database names. RCLS-001 to RCLS-045.
|
||||
*/
|
||||
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
|
||||
import { AuditLog } from '../../../../src/db/entities/AuditLog.entity';
|
||||
@@ -19,7 +20,7 @@ import { StorageReclaimMarksRepository } from '../../../../src/db/repositories/S
|
||||
import { dbNow } from '../../../../src/db/types';
|
||||
import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service';
|
||||
import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import { logError } from '../../../../src/nest/audit/audit-log.logger';
|
||||
import { logError, logWarn } from '../../../../src/nest/audit/audit-log.logger';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import { DatabaseLifecycle } from '../../../../src/nest/database/database-lifecycle.service';
|
||||
import { StorageAdminService } from '../../../../src/nest/storage/storage-admin.service';
|
||||
@@ -167,7 +168,14 @@ beforeAll(async () => {
|
||||
imports: [StorageModule],
|
||||
init: true,
|
||||
overrides: [
|
||||
{ provide: DataPathsService, useValue: { ...new DataPathsService(), installIdFile: path.join(dataRoot, 'id') } },
|
||||
{
|
||||
provide: DataPathsService,
|
||||
useValue: {
|
||||
...new DataPathsService(),
|
||||
installIdFile: path.join(dataRoot, 'id'),
|
||||
reclaimResetFile: path.join(dataRoot, 'reclaim-reset'),
|
||||
},
|
||||
},
|
||||
{
|
||||
provide: DatabaseLifecycle,
|
||||
useValue: {
|
||||
@@ -189,6 +197,7 @@ beforeAll(async () => {
|
||||
beforeEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.mocked(logError).mockClear();
|
||||
vi.mocked(logWarn).mockClear();
|
||||
for (const entry of fs.readdirSync(uploadsRoot)) {
|
||||
if (entry !== '.tmp') fs.rmSync(path.join(uploadsRoot, entry), { recursive: true, force: true });
|
||||
}
|
||||
@@ -854,6 +863,59 @@ describe('StorageReclaimService: a pass that stops, and a restore', () => {
|
||||
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
|
||||
});
|
||||
|
||||
it('RCLS-044: a restore left the reset file: a backup of this installation loses its state too, then it goes', async () => {
|
||||
const install = t.get(StorageInstallService);
|
||||
const sentinel = path.join(dataRoot, 'reclaim-reset');
|
||||
await markedWeekOld(['orphan.jpg']);
|
||||
await orm.repo(StorageReleaseQueue).enqueue([{ category: 'covers', key: 'queued.jpg' }], dbNow(new Date()));
|
||||
// A restore at boot, or a restart after a restore whose reopen failed: the id is this installation's own.
|
||||
fs.writeFileSync(sentinel, 'pending');
|
||||
expect(await install.databaseIsThis(installId)).toBe(false);
|
||||
expect(await install.reconcile()).toBe(true);
|
||||
expect(fs.existsSync(sentinel)).toBe(false);
|
||||
expect(await readAppSetting(orm, 'install_id')).toBe(installId);
|
||||
expect(await readAppSetting(orm, 'storage.reclaim_delete')).toBeNull();
|
||||
expect(await readAppSetting(orm, 'storage.reclaim_report')).toBeNull();
|
||||
expect(await marksOf('covers')).toEqual([]);
|
||||
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
|
||||
expect(await install.databaseIsThis(installId)).toBe(true);
|
||||
// Done once: the next start keeps what this installation does from then on.
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
expect(await install.reconcile()).toBe(false);
|
||||
expect((await reclaim.state()).deleteEnabled).toBe(true);
|
||||
// A reopen that went through takes the file away with its reset.
|
||||
fs.writeFileSync(sentinel, 'pending');
|
||||
await reopen();
|
||||
expect(fs.existsSync(sentinel)).toBe(false);
|
||||
expect((await reclaim.state()).deleteEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it('RCLS-045: a reset file that cannot be removed is reset again on the next start, and the database is not this', async () => {
|
||||
const install = t.get(StorageInstallService);
|
||||
const sentinel = path.join(dataRoot, 'reclaim-reset');
|
||||
fs.writeFileSync(sentinel, 'pending');
|
||||
const realRm = fs.rmSync;
|
||||
vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => {
|
||||
if (target === sentinel) throw Object.assign(new Error('read-only file system'), { code: 'EROFS' });
|
||||
realRm(target, options);
|
||||
});
|
||||
try {
|
||||
await reclaim.setDeleteEnabled(true);
|
||||
expect(await install.reconcile()).toBe(true);
|
||||
expect(fs.existsSync(sentinel)).toBe(true);
|
||||
expect(await install.databaseIsThis(installId)).toBe(false);
|
||||
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('the next start resets again'));
|
||||
} finally {
|
||||
vi.mocked(fs.rmSync).mockRestore();
|
||||
fs.rmSync(sentinel, { force: true });
|
||||
}
|
||||
expect(await install.databaseIsThis(installId)).toBe(true);
|
||||
// A database that names another installation is not this one either.
|
||||
await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab');
|
||||
expect(await install.databaseIsThis(installId)).toBe(false);
|
||||
expect(await install.reconcile()).toBe(true);
|
||||
});
|
||||
|
||||
it('RCLS-041: the install id file is made once, mode 0600; a garbled one keeps every delete off', async () => {
|
||||
const file = path.join(dataRoot, 'id');
|
||||
expect(fs.readFileSync(file, 'utf8').trim()).toBe(installId);
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
* StorageModule (a LocalDriver on a temp folder) with fake sources standing in
|
||||
* for the owner domains. An object any row still names stays; the pass's
|
||||
* guards hold, the owner marker included; a failure never reaches the caller
|
||||
* and leaves its rows queued. RELS-001 to RELS-021.
|
||||
* and leaves its rows queued; one failing category never holds up another, and
|
||||
* a database that is not this installation's releases nothing. RELS-001 to
|
||||
* RELS-026.
|
||||
*/
|
||||
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
|
||||
import { StorageReleaseQueue } from '../../../../src/db/entities/StorageReleaseQueue.entity';
|
||||
@@ -121,7 +123,11 @@ beforeAll(async () => {
|
||||
dataRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-data-'));
|
||||
orm = await createTestOrm(db);
|
||||
await setBackends();
|
||||
const paths = { ...new DataPathsService(), installIdFile: path.join(dataRoot, '.trek-install-id') };
|
||||
const paths = {
|
||||
...new DataPathsService(),
|
||||
installIdFile: path.join(dataRoot, '.trek-install-id'),
|
||||
reclaimResetFile: path.join(dataRoot, '.trek-reclaim-reset'),
|
||||
};
|
||||
t = await createTestModule({
|
||||
db,
|
||||
imports: [StorageModule],
|
||||
@@ -441,3 +447,103 @@ describe('StorageReleaseService: the guards the nightly pass keeps', () => {
|
||||
expect(await queued('covers')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('StorageReleaseService: what a round-3 review found', () => {
|
||||
it('RELS-022: a database naming another installation, or a restore whose reset is pending, releases nothing', async () => {
|
||||
await put('covers', `${UUID}-x.jpg`);
|
||||
await release.releaseValues([`/uploads/covers/${UUID}-x.jpg`]);
|
||||
await setAppSetting(orm, 'install_id', '12345678-aaaa-4bbb-8ccc-1234567890ab');
|
||||
try {
|
||||
expect((await release.processDue(due())).skipped).toBe('install_mismatch');
|
||||
} finally {
|
||||
await setAppSetting(orm, 'install_id', installId);
|
||||
}
|
||||
const sentinel = path.join(dataRoot, '.trek-reclaim-reset');
|
||||
fs.writeFileSync(sentinel, 'pending');
|
||||
try {
|
||||
expect((await release.processDue(due())).skipped).toBe('install_mismatch');
|
||||
} finally {
|
||||
fs.rmSync(sentinel, { force: true });
|
||||
}
|
||||
expect(exists('covers', `${UUID}-x.jpg`)).toBe(true);
|
||||
expect(await queued('covers')).toBe(1);
|
||||
expect(await release.processDue(due())).toEqual({ skipped: null, deleted: 1, kept: 0 });
|
||||
});
|
||||
|
||||
it('RELS-023: a category whose deletes keep failing takes only its share of a tick, the others still go', async () => {
|
||||
const older = Array.from({ length: 230 }, (_unused, i) => `${UUID}-c${i}.jpg`);
|
||||
await orm.repo(StorageReleaseQueue).enqueue(
|
||||
older.map((key) => ({ category: 'covers', key })),
|
||||
dbNow(new Date(Date.now() - DAY)),
|
||||
);
|
||||
await put('places', `${UUID}-p.jpg`);
|
||||
await release.releaseValues([`/uploads/places/${UUID}-p.jpg`]);
|
||||
const realDelete = storage.delete.bind(storage);
|
||||
vi.spyOn(storage, 'delete').mockImplementation(async (category, key) => {
|
||||
if (category === 'covers') throw new Error('covers bucket refuses deletes');
|
||||
await realDelete(category, key);
|
||||
});
|
||||
// The 230 older covers rows would fill a tick of 200 on their own, every tick.
|
||||
expect((await release.processDue(due())).deleted).toBe(1);
|
||||
expect(exists('places', `${UUID}-p.jpg`)).toBe(false);
|
||||
expect(await queued('places')).toBe(0);
|
||||
expect(await queued('covers')).toBe(230);
|
||||
});
|
||||
|
||||
it('RELS-024: rows dropped after thirty days and a full queue are logged at warn with their counts', async () => {
|
||||
await orm.repo(StorageReleaseQueue).enqueue(
|
||||
[
|
||||
{ category: 'covers', key: `${UUID}-old1.jpg` },
|
||||
{ category: 'places', key: `${UUID}-old2.jpg` },
|
||||
],
|
||||
dbNow(new Date(Date.now() - 31 * DAY)),
|
||||
);
|
||||
await release.processDue(new Date());
|
||||
expect(logWarn).toHaveBeenCalledWith(expect.stringMatching(/^\[storage\] 2 released object\(s\) waited 30 days/));
|
||||
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('only once deletion is on'));
|
||||
vi.mocked(logWarn).mockClear();
|
||||
vi.spyOn(StorageReleaseQueueRepository.prototype, 'size').mockResolvedValueOnce(50_000);
|
||||
expect(await release.releaseValues([`/uploads/covers/${UUID}-full.jpg`])).toBe(0);
|
||||
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('(50000 of 50000 waiting): 1 released object(s)'));
|
||||
expect(await countRows(orm, StorageReleaseQueue)).toBe(0);
|
||||
});
|
||||
|
||||
it('RELS-025: an allowed owner verdict holds for one window only, a refusal until the next night', async () => {
|
||||
const marker = storageOwnerMarkerKey('another-install-0002');
|
||||
const lists = vi.spyOn(storage, 'list');
|
||||
const start = due().getTime();
|
||||
await put('covers', `${UUID}-w1.jpg`);
|
||||
await release.releaseValues([`/uploads/covers/${UUID}-w1.jpg`]);
|
||||
expect((await release.processDue(new Date(start))).deleted).toBe(1);
|
||||
expect(lists).toHaveBeenCalledTimes(1);
|
||||
// Another installation's marker shows up: within the window the verdict stands, without a second listing.
|
||||
await put('covers', marker);
|
||||
try {
|
||||
await put('covers', `${UUID}-w2.jpg`);
|
||||
await release.releaseValues([`/uploads/covers/${UUID}-w2.jpg`]);
|
||||
expect((await release.processDue(new Date(start + 10 * 60_000))).deleted).toBe(1);
|
||||
expect(lists).toHaveBeenCalledTimes(1);
|
||||
// Past it the root is listed again and the category refused.
|
||||
await put('covers', `${UUID}-w3.jpg`);
|
||||
await release.releaseValues([`/uploads/covers/${UUID}-w3.jpg`]);
|
||||
expect((await release.processDue(new Date(start + 16 * 60_000))).deleted).toBe(0);
|
||||
expect(lists).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
await storage.delete('covers', marker);
|
||||
}
|
||||
// The refusal holds hours later, though the marker is gone, until the nightly pass judges again.
|
||||
expect((await release.processDue(new Date(start + 6 * 60 * 60_000))).deleted).toBe(0);
|
||||
expect(lists).toHaveBeenCalledTimes(2);
|
||||
expect(exists('covers', `${UUID}-w3.jpg`)).toBe(true);
|
||||
expect((await release.processDue(new Date(start + 27 * 60 * 60_000))).deleted).toBe(1);
|
||||
expect(exists('covers', `${UUID}-w3.jpg`)).toBe(false);
|
||||
});
|
||||
|
||||
it('RELS-026: the owner marker is looked for in the category root only, one shallow listing', async () => {
|
||||
const lists = vi.spyOn(storage, 'list');
|
||||
await put('covers', `${UUID}-root.jpg`);
|
||||
await release.releaseValues([`/uploads/covers/${UUID}-root.jpg`]);
|
||||
await release.processDue(due());
|
||||
expect(lists.mock.calls).toEqual([['covers', '', { shallow: true }]]);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user