fix(storage): keep the reclaim pass and a backup restore apart

A restore wipes and rehydrates the uploads under a swapped database, but it
never took the storage job slot, so a reclaim pass could start or carry on
beside it, and a delete already past the generation check could land on an
object the restore had just put back. BackupService now marks a restore as
running on StorageJobsService and waits for a pass in flight to let go; a
pass refuses to start while a restore runs and stops at its next check when
one begins. Backfills and migrations are not affected. The service test also
reads its fake source sets through a helper instead of non-null assertions.
This commit is contained in:
Maurice
2026-11-01 14:20:11 +01:00
parent 8fde643347
commit 19e4f0dadf
6 changed files with 150 additions and 13 deletions
+19 -2
View File
@@ -1,4 +1,5 @@
import { DATABASE_BACKUP, type DatabaseBackupStrategy } from '../database/database-backup.interface';
import { StorageJobsService } from '../storage/storage-jobs.service';
import { StorageService } from '../storage/storage.service';
import * as svc from './backup.impl';
import { Inject, Injectable } from '@nestjs/common';
@@ -23,6 +24,7 @@ export class BackupService {
constructor(
private readonly storage: StorageService,
@Inject(DATABASE_BACKUP) database: DatabaseBackupStrategy,
private readonly jobs: StorageJobsService,
) {
this.deps = { storage, database };
}
@@ -34,10 +36,10 @@ export class BackupService {
return svc.createBackup(this.deps, prefix);
}
restoreFromZip(zipPath: string) {
return svc.restoreFromZip(this.deps, zipPath);
return this.whileRestoring(() => svc.restoreFromZip(this.deps, zipPath));
}
restoreBackup(filename: string) {
return svc.restoreBackup(this.deps, filename);
return this.whileRestoring(() => svc.restoreBackup(this.deps, filename));
}
deleteBackup(filename: string) {
return svc.deleteBackup(this.storage, filename);
@@ -59,4 +61,19 @@ export class BackupService {
get rateWindow() {
return svc.BACKUP_RATE_WINDOW;
}
/**
* A restore wipes and rehydrates the uploads under a swapped database, so
* the storage reclaim pass must not run beside it: the restore marks itself
* running (a pass refuses to start and one in flight stops at its next
* check) and waits for a pass in flight to let go before it touches anything.
*/
private async whileRestoring<T>(restore: () => Promise<T>): Promise<T> {
await this.jobs.beginRestore();
try {
return await restore();
} finally {
this.jobs.endRestore();
}
}
}
@@ -43,6 +43,11 @@ export class StorageJobsService {
private ttlMs = 10 * 60_000;
/** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */
private exclusive: string | null = null;
/** Restores in progress: no exclusive pass starts while one runs, and one in flight stops. */
private restores = 0;
/** Settles once the exclusive pass holding the slot releases it; settled while none does. */
private exclusiveReleased: Promise<void> = Promise.resolve();
private onExclusiveReleased: (() => void) | null = null;
/** One storage job at a time, across backfills, category migrations and the reclaim pass. */
private anyJobRunning(): boolean {
@@ -60,17 +65,45 @@ export class StorageJobsService {
/**
* Takes the one-job slot for a pass that runs in the caller (the reclaim
* pass): false while a backfill, a migration or another such pass runs.
* pass): false while a backfill, a migration, another such pass or a
* restore runs.
* The caller releases it in a finally.
*/
claimExclusive(name: string): boolean {
if (this.anyJobRunning()) return false;
if (this.restores > 0 || this.anyJobRunning()) return false;
this.exclusive = name;
this.exclusiveReleased = new Promise((resolve) => {
this.onExclusiveReleased = resolve;
});
return true;
}
releaseExclusive(name: string): void {
if (this.exclusive === name) this.exclusive = null;
if (this.exclusive !== name) return;
this.exclusive = null;
this.onExclusiveReleased?.();
this.onExclusiveReleased = null;
}
/** True while a backup restore swaps the database and the uploads. */
get restoreRunning(): boolean {
return this.restores > 0;
}
/**
* Marks a restore as running and waits until an exclusive pass in flight
* (the reclaim pass, which stops at its next check of `restoreRunning`) has
* let go, so none of its deletes can land on the uploads the restore puts
* back. Backfills and migrations are not waited for, as before. The caller
* calls `endRestore` in a finally.
*/
async beginRestore(): Promise<void> {
this.restores += 1;
await this.exclusiveReleased;
}
endRestore(): void {
if (this.restores > 0) this.restores -= 1;
}
constructor(private readonly registry: StorageRegistryService) {}
@@ -61,7 +61,10 @@ interface ReclaimRunOptions {
now?: Date;
}
/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */
/**
* A restore began, or the database was swapped under the pass: nothing it
* learnt before holds any more. Reported as `database_reopened` either way.
*/
class ReclaimStopped extends Error {}
/**
@@ -78,8 +81,9 @@ class ReclaimStopped extends Error {}
* registered, whose objects share a location with another category's, whose
* listing or lookups fail, or where more than a fifth of the objects look
* unreferenced is skipped and its marks start over. A pass never runs beside
* a backfill or a category migration (the one-job slot of StorageJobsService),
* never on a demo instance, and stops when the database is reopened under it.
* a backfill, a category migration or a restore (the one-job slot of
* StorageJobsService, which a restore waits on), never on a demo instance,
* and stops when a restore begins or the database is reopened under it.
*/
@Injectable()
export class StorageReclaimService implements OnModuleInit {
@@ -377,7 +381,9 @@ export class StorageReclaimService implements OnModuleInit {
return true;
}
/** A restore that began, or a reopen that happened, since the pass started ends it. */
private assertSameDatabase(ctx: PassContext): void {
if (this.jobs.restoreRunning) throw new ReclaimStopped('a restore began during the pass');
if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass');
}
}
+3 -1
View File
@@ -37,6 +37,8 @@ import { Module } from '@nestjs/common';
* and follows database reopens (DatabaseLifecycleModule); the domains whose
* rows name stored objects register their reference sources with the exported
* StorageReclaimRegistryService, so storage never imports them.
* StorageJobsService is exported for the backup restore, which marks itself
* running there and waits for a reclaim pass in flight to let go.
*/
@Module({
imports: [
@@ -59,6 +61,6 @@ import { Module } from '@nestjs/common';
StorageReclaimService,
StorageReclaimJob,
],
exports: [StorageService, StorageEventsService, StorageReclaimRegistryService],
exports: [StorageService, StorageEventsService, StorageJobsService, StorageReclaimRegistryService],
})
export class StorageModule {}