mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 03:26:07 +00:00
fix(storage): keep the reclaim pass and a backup restore apart
A restore wipes and rehydrates the uploads under a swapped database, but it never took the storage job slot, so a reclaim pass could start or carry on beside it, and a delete already past the generation check could land on an object the restore had just put back. BackupService now marks a restore as running on StorageJobsService and waits for a pass in flight to let go; a pass refuses to start while a restore runs and stops at its next check when one begins. Backfills and migrations are not affected. The service test also reads its fake source sets through a helper instead of non-null assertions.
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import { DATABASE_BACKUP, type DatabaseBackupStrategy } from '../database/database-backup.interface';
|
||||
import { StorageJobsService } from '../storage/storage-jobs.service';
|
||||
import { StorageService } from '../storage/storage.service';
|
||||
import * as svc from './backup.impl';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
@@ -23,6 +24,7 @@ export class BackupService {
|
||||
constructor(
|
||||
private readonly storage: StorageService,
|
||||
@Inject(DATABASE_BACKUP) database: DatabaseBackupStrategy,
|
||||
private readonly jobs: StorageJobsService,
|
||||
) {
|
||||
this.deps = { storage, database };
|
||||
}
|
||||
@@ -34,10 +36,10 @@ export class BackupService {
|
||||
return svc.createBackup(this.deps, prefix);
|
||||
}
|
||||
restoreFromZip(zipPath: string) {
|
||||
return svc.restoreFromZip(this.deps, zipPath);
|
||||
return this.whileRestoring(() => svc.restoreFromZip(this.deps, zipPath));
|
||||
}
|
||||
restoreBackup(filename: string) {
|
||||
return svc.restoreBackup(this.deps, filename);
|
||||
return this.whileRestoring(() => svc.restoreBackup(this.deps, filename));
|
||||
}
|
||||
deleteBackup(filename: string) {
|
||||
return svc.deleteBackup(this.storage, filename);
|
||||
@@ -59,4 +61,19 @@ export class BackupService {
|
||||
get rateWindow() {
|
||||
return svc.BACKUP_RATE_WINDOW;
|
||||
}
|
||||
|
||||
/**
|
||||
* A restore wipes and rehydrates the uploads under a swapped database, so
|
||||
* the storage reclaim pass must not run beside it: the restore marks itself
|
||||
* running (a pass refuses to start and one in flight stops at its next
|
||||
* check) and waits for a pass in flight to let go before it touches anything.
|
||||
*/
|
||||
private async whileRestoring<T>(restore: () => Promise<T>): Promise<T> {
|
||||
await this.jobs.beginRestore();
|
||||
try {
|
||||
return await restore();
|
||||
} finally {
|
||||
this.jobs.endRestore();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,11 @@ export class StorageJobsService {
|
||||
private ttlMs = 10 * 60_000;
|
||||
/** A pass that holds the one-job slot without a status of its own (the reclaim pass), by name. */
|
||||
private exclusive: string | null = null;
|
||||
/** Restores in progress: no exclusive pass starts while one runs, and one in flight stops. */
|
||||
private restores = 0;
|
||||
/** Settles once the exclusive pass holding the slot releases it; settled while none does. */
|
||||
private exclusiveReleased: Promise<void> = Promise.resolve();
|
||||
private onExclusiveReleased: (() => void) | null = null;
|
||||
|
||||
/** One storage job at a time, across backfills, category migrations and the reclaim pass. */
|
||||
private anyJobRunning(): boolean {
|
||||
@@ -60,17 +65,45 @@ export class StorageJobsService {
|
||||
|
||||
/**
|
||||
* Takes the one-job slot for a pass that runs in the caller (the reclaim
|
||||
* pass): false while a backfill, a migration or another such pass runs.
|
||||
* pass): false while a backfill, a migration, another such pass or a
|
||||
* restore runs.
|
||||
* The caller releases it in a finally.
|
||||
*/
|
||||
claimExclusive(name: string): boolean {
|
||||
if (this.anyJobRunning()) return false;
|
||||
if (this.restores > 0 || this.anyJobRunning()) return false;
|
||||
this.exclusive = name;
|
||||
this.exclusiveReleased = new Promise((resolve) => {
|
||||
this.onExclusiveReleased = resolve;
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
releaseExclusive(name: string): void {
|
||||
if (this.exclusive === name) this.exclusive = null;
|
||||
if (this.exclusive !== name) return;
|
||||
this.exclusive = null;
|
||||
this.onExclusiveReleased?.();
|
||||
this.onExclusiveReleased = null;
|
||||
}
|
||||
|
||||
/** True while a backup restore swaps the database and the uploads. */
|
||||
get restoreRunning(): boolean {
|
||||
return this.restores > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a restore as running and waits until an exclusive pass in flight
|
||||
* (the reclaim pass, which stops at its next check of `restoreRunning`) has
|
||||
* let go, so none of its deletes can land on the uploads the restore puts
|
||||
* back. Backfills and migrations are not waited for, as before. The caller
|
||||
* calls `endRestore` in a finally.
|
||||
*/
|
||||
async beginRestore(): Promise<void> {
|
||||
this.restores += 1;
|
||||
await this.exclusiveReleased;
|
||||
}
|
||||
|
||||
endRestore(): void {
|
||||
if (this.restores > 0) this.restores -= 1;
|
||||
}
|
||||
|
||||
constructor(private readonly registry: StorageRegistryService) {}
|
||||
|
||||
@@ -61,7 +61,10 @@ interface ReclaimRunOptions {
|
||||
now?: Date;
|
||||
}
|
||||
|
||||
/** The database was swapped under the pass (a restore): nothing it learnt before holds any more. */
|
||||
/**
|
||||
* A restore began, or the database was swapped under the pass: nothing it
|
||||
* learnt before holds any more. Reported as `database_reopened` either way.
|
||||
*/
|
||||
class ReclaimStopped extends Error {}
|
||||
|
||||
/**
|
||||
@@ -78,8 +81,9 @@ class ReclaimStopped extends Error {}
|
||||
* registered, whose objects share a location with another category's, whose
|
||||
* listing or lookups fail, or where more than a fifth of the objects look
|
||||
* unreferenced is skipped and its marks start over. A pass never runs beside
|
||||
* a backfill or a category migration (the one-job slot of StorageJobsService),
|
||||
* never on a demo instance, and stops when the database is reopened under it.
|
||||
* a backfill, a category migration or a restore (the one-job slot of
|
||||
* StorageJobsService, which a restore waits on), never on a demo instance,
|
||||
* and stops when a restore begins or the database is reopened under it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StorageReclaimService implements OnModuleInit {
|
||||
@@ -377,7 +381,9 @@ export class StorageReclaimService implements OnModuleInit {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** A restore that began, or a reopen that happened, since the pass started ends it. */
|
||||
private assertSameDatabase(ctx: PassContext): void {
|
||||
if (this.jobs.restoreRunning) throw new ReclaimStopped('a restore began during the pass');
|
||||
if (this.generation !== ctx.generation) throw new ReclaimStopped('the database was reopened during the pass');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,8 @@ import { Module } from '@nestjs/common';
|
||||
* and follows database reopens (DatabaseLifecycleModule); the domains whose
|
||||
* rows name stored objects register their reference sources with the exported
|
||||
* StorageReclaimRegistryService, so storage never imports them.
|
||||
* StorageJobsService is exported for the backup restore, which marks itself
|
||||
* running there and waits for a reclaim pass in flight to let go.
|
||||
*/
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -59,6 +61,6 @@ import { Module } from '@nestjs/common';
|
||||
StorageReclaimService,
|
||||
StorageReclaimJob,
|
||||
],
|
||||
exports: [StorageService, StorageEventsService, StorageReclaimRegistryService],
|
||||
exports: [StorageService, StorageEventsService, StorageJobsService, StorageReclaimRegistryService],
|
||||
})
|
||||
export class StorageModule {}
|
||||
|
||||
Reference in New Issue
Block a user