feat(db): run TREK on PostgreSQL

TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before
otherwise: an engine port under src/db/engine, the dialect helpers under
src/db/dialect, a Postgres baseline for the migrations up to the cut,
pg_dump/pg_restore backups, a probe of every repository method against a
real server and a Postgres test lane in CI.
This commit is contained in:
Isaias Tavares
2026-11-05 01:15:21 -03:00
committed by Maurice
parent 17b47b38b9
commit 2e3762afe1
176 changed files with 6728 additions and 777 deletions
+40 -19
View File
@@ -366,6 +366,7 @@ export class AssistantService {
const decided = await this.claimDecisions(threadId, req.decisions);
// Taken but not run yet: whatever ends the turn before a change ran records it as failed.
const unrun = new Set(decided.filter(({ approved }) => approved).map(({ call }) => call.id));
const settle = () => this.failUnrun(threadId, decided, unrun);
try {
const stream = channel.open();
const w = new UiStreamWriter(stream);
@@ -387,20 +388,26 @@ export class AssistantService {
firstRowId: null,
renders: req.renders === true,
};
await this.converse(spec, stream, w, signal, (session) =>
this.applyDecisions(
(name, args) => this.runChange(session, { userId, scope, threadId }, name, args),
threadId,
decided,
{
w,
unrun,
scope,
},
),
await this.converse(
spec,
stream,
w,
signal,
(session) =>
this.applyDecisions(
(name, args) => this.runChange(session, { userId, scope, threadId }, name, args),
threadId,
decided,
{
w,
unrun,
scope,
},
),
settle,
);
} finally {
if (unrun.size > 0) await this.failUnrun(threadId, decided, unrun);
await settle();
}
});
}
@@ -440,8 +447,11 @@ export class AssistantService {
w.finish('error');
} finally {
await opened?.close();
w.close();
if (unrun.size > 0) await this.failUnrun(threadId, decided, unrun);
try {
await this.failUnrun(threadId, decided, unrun);
} finally {
w.close();
}
}
});
}
@@ -732,21 +742,26 @@ export class AssistantService {
/**
* The approved changes the turn ended before running (the tool session did
* not open, the stream broke): recorded as failed, so the thread shows them
* as not applied and nobody takes them for a change that ran.
* as not applied and nobody takes them for a change that ran. Once: the set
* is emptied first, so the stream's close and the caller's cleanup do not
* both record them.
*
* @txIndependent each outcome is written on its own, like the ones that ran.
*/
private async failUnrun(
threadId: number,
decided: Array<{ call: AssistantToolCallRow; approved: boolean }>,
unrun: ReadonlySet<string>,
unrun: Set<string>,
): Promise<void> {
if (unrun.size === 0) return;
const pending = new Set(unrun);
unrun.clear();
const outcomes = new Map<string, Outcome>();
for (const { call, approved } of decided) {
if (!approved) outcomes.set(call.id, { approved: false, result: DISCARDED, isError: false });
else if (unrun.has(call.id)) outcomes.set(call.id, { approved: true, result: NOT_RUN, isError: true });
else if (pending.has(call.id)) outcomes.set(call.id, { approved: true, result: NOT_RUN, isError: true });
}
const failed = decided.filter(({ call, approved }) => approved && unrun.has(call.id));
const failed = decided.filter(({ call, approved }) => approved && pending.has(call.id));
await Promise.all(
failed.map(({ call }) =>
this.inContext(() =>
@@ -810,6 +825,7 @@ export class AssistantService {
w: UiStreamWriter,
signal: AbortSignal,
prelude?: (session: AssistantToolSession) => Promise<void>,
settle?: () => Promise<void>,
): Promise<void> {
const stopKeepalive = armKeepalive(stream);
let opened: AssistantToolSession | null = null;
@@ -884,7 +900,12 @@ export class AssistantService {
} finally {
stopKeepalive();
await opened?.close();
w.close();
try {
// Before the close: the client reads the thread again once the stream ends.
await settle?.();
} finally {
w.close();
}
}
}
+10 -4
View File
@@ -4,7 +4,9 @@ import { UserSessions } from '../../db/entities/UserSessions.entity';
import type { CarriedUserSessionRow } from '../../db/repositories/UserSessions.repository';
import { dbNow } from '../../db/types';
import { logError, logWarn } from '../audit/audit-log.logger';
import type { DatabaseBackupStrategy } from '../database/database-backup.interface';
import { describeError } from '../common/describe-error';
import { DomainError } from '../common/domain-error';
import type { DatabaseBackupRefusal, DatabaseBackupStrategy } from '../database/database-backup.interface';
import { invalidatePermissionsCache } from '../permissions/permissions-cache';
import { snapshotAllPluginDataDbs } from '../plugins/host/plugin-data.service';
import { pluginsCodeRoot, pluginsDataRoot } from '../plugins/paths';
@@ -36,8 +38,6 @@ export interface BackupDeps {
database: DatabaseBackupStrategy;
}
const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err));
// The scratch and key directory. The database itself may live elsewhere
// (TREK_DB_FILE); the database port knows where.
const { dataDir, encryptionKeyFile, reclaimResetFile } = resolveDataPaths();
@@ -467,6 +467,12 @@ async function restoreCarriedSessions(rows: readonly CarriedUserSessionRow[]): P
}
}
/** The restore route's refusal for a `verify` that threw one. Any other failure is not a refusal, and keeps propagating. */
function refusalOf(err: unknown): DatabaseBackupRefusal {
if (err instanceof DomainError) return { error: err.publicMessage, status: err.getStatus() };
throw err;
}
export async function restoreFromZip({ storage, database }: BackupDeps, zipPath: string): Promise<RestoreResult> {
const extractDir = path.join(dataDir, `restore-${Date.now()}`);
let reinitFailed: unknown;
@@ -474,7 +480,7 @@ export async function restoreFromZip({ storage, database }: BackupDeps, zipPath:
const refused = await extractBackupArchive(zipPath, extractDir);
if (refused) return { success: false, ...refused };
const unreadable = database.verify(extractDir);
const unreadable = await database.verify(extractDir).then(() => null, refusalOf);
if (unreadable) {
fs.rmSync(extractDir, { recursive: true, force: true });
return { success: false, ...unreadable };
+39 -9
View File
@@ -1,5 +1,7 @@
import { pgRestore, pgTableNames } from '../../db/engine/postgres/pg-tools';
import { stageExtractedPluginTrees } from '../plugins/plugin-backup';
import { checkBackupDatabase, extractBackupArchive, markReclaimResetPending } from './backup-archive';
import { checkPostgresDump, PG_ARCHIVE_ENTRY } from './postgres-database-backup';
import fs from 'fs';
import path from 'path';
@@ -47,6 +49,11 @@ export interface BootRestorePlan {
dataDir: string;
/** `data/.trek-reclaim-reset` (data-paths.ts), left before the database is put in place. */
reclaimResetFile: string;
/**
* Set when TREK runs on Postgres (TREK_DB_URL): the archive's dump is loaded
* into the database, which must hold no table yet, instead of `dbFile`.
*/
postgres?: { url: string; binDir: string | undefined };
}
export type BootRestoreOutcome =
@@ -56,11 +63,38 @@ export type BootRestoreOutcome =
export class BootRestoreError extends Error {}
/** A database is there already: on a file, the file; on Postgres, any table in its schema. */
async function existingDatabase(plan: BootRestorePlan): Promise<string | null> {
if (!plan.postgres) return fs.existsSync(plan.dbFile) ? plan.dbFile : null;
const tables = await pgTableNames(plan.postgres.url, plan.postgres.binDir);
return tables.length > 0 ? `the Postgres database (${tables.length} table(s))` : null;
}
/** Why the unpacked archive's database cannot be restored here, or null. */
async function checkArchiveDatabase(plan: BootRestorePlan, staging: string): Promise<string | null> {
if (plan.postgres) return checkPostgresDump(staging, plan.postgres.binDir);
return checkBackupDatabase(staging)?.error ?? null;
}
/** Puts the archive's database in place: renamed in beside the file, or loaded into Postgres in one transaction. */
async function putDatabaseInPlace(plan: BootRestorePlan, staging: string): Promise<void> {
if (plan.postgres) {
await pgRestore(plan.postgres.url, path.join(staging, PG_ARCHIVE_ENTRY), plan.postgres.binDir);
return;
}
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
const tmp = `${plan.dbFile}.restore-tmp`;
fs.copyFileSync(path.join(staging, 'travel.db'), tmp);
fs.renameSync(tmp, plan.dbFile);
}
export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRestoreOutcome> {
if (!plan.archive) return { restored: false, reason: 'unset' };
if (fs.existsSync(plan.dbFile)) {
const existing = await existingDatabase(plan);
if (existing) {
console.warn(
`[restore] RESTORE_FROM_BACKUP is set, but ${plan.dbFile} already exists. The backup is only restored on a first start; remove the variable to silence this.`,
`[restore] RESTORE_FROM_BACKUP is set, but ${existing} already exists. The backup is only restored on a first start; remove the variable to silence this.`,
);
return { restored: false, reason: 'database-exists' };
}
@@ -73,20 +107,16 @@ export async function restoreOnFirstBoot(plan: BootRestorePlan): Promise<BootRes
const refused = await extractBackupArchive(plan.archive, staging);
if (refused) throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${refused.error}`);
const unreadable = checkBackupDatabase(staging);
const unreadable = await checkArchiveDatabase(plan, staging);
if (unreadable) {
fs.rmSync(staging, { recursive: true, force: true });
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable.error}`);
throw new BootRestoreError(`RESTORE_FROM_BACKUP: ${unreadable}`);
}
// Storage reclaim drops what the restored database knew about its storage on
// the start that follows, even when the backup is this installation's own.
markReclaimResetPending(plan.reclaimResetFile);
fs.mkdirSync(path.dirname(plan.dbFile), { recursive: true });
// Copied in beside the target and renamed, so a crash leaves either nothing or a whole file.
const tmp = `${plan.dbFile}.restore-tmp`;
fs.copyFileSync(path.join(staging, 'travel.db'), tmp);
fs.renameSync(tmp, plan.dbFile);
await putDatabaseInPlace(plan, staging);
// The key the archive's secrets were encrypted with. An ENCRYPTION_KEY in the
// environment still wins over the file, exactly as on any other start.
@@ -1,11 +1,15 @@
import { DATABASE_BACKUP } from '../database/database-backup.interface';
import type { DatabaseEngine } from '../../db/engine';
import { DATABASE_BACKUP, type DatabaseBackupStrategy } from '../database/database-backup.interface';
import { DatabaseEngineModule, DATABASE_ENGINE } from '../database/database-engine.module';
import { DatabaseLifecycleModule } from '../database/database-lifecycle.module';
import { MaintenanceModule } from '../database/maintenance.module';
import { PostgresDatabaseBackup } from './postgres-database-backup';
import { SqliteDatabaseBackup } from './sqlite-database-backup';
import { Global, Module } from '@nestjs/common';
/**
* Binds the database backup port (`DATABASE_BACKUP`) to the engine in use.
* Binds the database backup port (`DATABASE_BACKUP`) to the engine in use
* (`TREK_DB_URL`, the one `DATABASE_ENGINE` provides).
*
* Global because its consumers sit in two domains: the backup domain itself
* and the admin domain (the demo reset job and the baseline route). Admin
@@ -19,8 +23,20 @@ import { Global, Module } from '@nestjs/common';
*/
@Global()
@Module({
imports: [DatabaseLifecycleModule, MaintenanceModule],
providers: [SqliteDatabaseBackup, { provide: DATABASE_BACKUP, useExisting: SqliteDatabaseBackup }],
imports: [DatabaseEngineModule, DatabaseLifecycleModule, MaintenanceModule],
providers: [
SqliteDatabaseBackup,
PostgresDatabaseBackup,
{
provide: DATABASE_BACKUP,
useFactory: (
engine: DatabaseEngine,
sqlite: SqliteDatabaseBackup,
postgres: PostgresDatabaseBackup,
): DatabaseBackupStrategy => ({ sqlite, postgres })[engine.id],
inject: [DATABASE_ENGINE, SqliteDatabaseBackup, PostgresDatabaseBackup],
},
],
exports: [DATABASE_BACKUP],
})
export class DatabaseBackupModule {}
+58
View File
@@ -0,0 +1,58 @@
import { logInfo, logWarn } from '../audit/audit-log.logger';
import { describeError } from '../common/describe-error';
import { DatabaseConnectionLostError } from '../database/database-backup.interface';
/** Whether a step ran to the end, and what it threw when it did not. */
interface Outcome {
failed: boolean;
error: unknown;
}
/** Runs `step` and records that it threw even when what it threw is undefined. */
async function settle(step: () => void | Promise<void>): Promise<Outcome> {
try {
await step();
return { failed: false, error: null };
} catch (error) {
return { failed: true, error };
}
}
/**
* The restore step both backup ports share. `swap` puts the new database in
* place; `reopen` always runs after it, so the process is never left without a
* connection. A failed swap rethrows its error, or a DatabaseConnectionLostError
* carrying both when the reopen failed too. A swap that landed resolves with the
* reopen's error, which the caller reports as "restart required".
*/
export async function replaceThenReopen(
swap: () => void | Promise<void>,
reopen: () => Promise<void>,
): Promise<{ reopenError: unknown }> {
const swapped = await settle(swap);
const reopened = await settle(reopen);
if (swapped.failed) {
if (reopened.failed) throw new DatabaseConnectionLostError(swapped.error, reopened.error);
throw swapped.error;
}
return { reopenError: reopened.failed ? reopened.error : null };
}
/**
* Keeps a copy of the database a restore is about to replace, at `target`. Best
* effort: a database broken enough to need the restore may not copy, and that
* must not block the recovery. Resolves with where the copy went, or null.
*/
export async function keepRestoreCopy(
target: string,
snapshot: (target: string) => Promise<void>,
): Promise<string | null> {
try {
await snapshot(target);
logInfo(`Restore: the replaced database was kept as ${target}`);
return target;
} catch (err) {
logWarn(`Restore: could not keep a copy of the current database (${describeError(err)})`);
return null;
}
}
@@ -0,0 +1,127 @@
import type { DbSettings } from '../../app-config';
import { postgresTarget, type DatabaseEngine } from '../../db/engine';
import { pgDump, pgDumpMigrations, pgRestore } from '../../db/engine/postgres/pg-tools';
import { unknownMigrations } from '../../db/known-migrations';
import { DataPathsService } from '../app-config/data-paths.service';
import { logWarn } from '../audit/audit-log.logger';
import { describeError } from '../common/describe-error';
import { DomainError } from '../common/domain-error';
import type { DatabaseBackupStrategy } from '../database/database-backup.interface';
import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../database/database-engine.module';
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
import { keepRestoreCopy, replaceThenReopen } from './database-swap';
import { Inject, Injectable } from '@nestjs/common';
import fs from 'fs';
import path from 'path';
/** The name a Postgres database takes inside a backup archive: a pg_dump custom-format file. */
export const PG_ARCHIVE_ENTRY = 'travel.pgdump';
/**
* Why the database in an unpacked archive cannot be restored into a Postgres
* database, or null when it can: it must be a dump, carry TREK's migration
* history, and none this build does not know. Both restore paths ask it, the
* admin panel's and RESTORE_FROM_BACKUP's.
*/
export async function checkPostgresDump(extractDir: string, binDir: string | undefined): Promise<string | null> {
const dump = path.join(extractDir, PG_ARCHIVE_ENTRY);
if (!fs.existsSync(dump)) {
return fs.existsSync(path.join(extractDir, 'travel.db'))
? 'This backup holds a SQLite database, which cannot be restored into a Postgres database.'
: `Invalid backup: ${PG_ARCHIVE_ENTRY} not found`;
}
let executed: string[];
try {
executed = await pgDumpMigrations(dump, binDir);
} catch (err) {
logWarn(`Restore: the archive's database could not be read (${describeError(err)})`);
return 'Uploaded file is not a valid Postgres dump';
}
if (executed.length === 0) {
return 'The uploaded dump holds no TREK migration history. This does not appear to be a TREK backup.';
}
if (unknownMigrations(executed).length > 0) {
return 'This backup was made by a newer TREK version. Update TREK before restoring it.';
}
return null;
}
/**
* The backup port on a Postgres database: `pg_dump` and `pg_restore`
* (`db/engine/postgres/pg-tools.ts`), run by their absolute paths with the
* password in their environment.
*
* A snapshot is a custom-format dump, consistent on its own (pg_dump reads one
* snapshot of the database). A restore checks the dump first (it must be one,
* carry TREK's migration history, and none this build does not know), keeps a
* dump of the database it replaces, then clears the schema and loads the dump
* in one transaction (`pgRestore`), so a restore that fails leaves the database
* as it was and one that succeeds holds nothing a later migration added, and
* reopens through `DatabaseLifecycle`, which migrates the restored database
* forward. A backup of a SQLite install cannot be restored here.
*/
@Injectable()
export class PostgresDatabaseBackup implements DatabaseBackupStrategy {
readonly archiveEntry = PG_ARCHIVE_ENTRY;
/** The URL and the tools' directory; null on SQLite, where this provider is built but never called. */
private readonly target: ReturnType<typeof postgresTarget>;
constructor(
private readonly lifecycle: DatabaseLifecycle,
private readonly paths: DataPathsService,
@Inject(DATABASE_ENGINE) private readonly engine: DatabaseEngine,
@Inject(DATABASE_SETTINGS) settings: DbSettings,
) {
this.target = postgresTarget(settings);
}
private get connection(): { url: string; binDir: string | undefined } {
if (!this.target) throw new Error('The Postgres backup was used without TREK_DB_URL.');
return this.target;
}
location(): string {
return this.engine.describe();
}
/** A Postgres database is always there to copy; a missing pg_dump fails the snapshot, loudly. */
canSnapshot(): boolean {
return true;
}
async checkpoint(): Promise<void> {
// Postgres keeps nothing beside its store that a copy would need flushed.
}
/** pg_dump writes beside `target` and renames into place: a failed dump leaves nothing there. */
async snapshot(target: string): Promise<void> {
const { url, binDir } = this.connection;
await pgDump(url, target, binDir);
}
/** Throws a DomainError (400) with the refusal when the archive cannot be restored here, before anything is replaced. */
async verify(extractDir: string): Promise<void> {
const refusal = await checkPostgresDump(extractDir, this.connection.binDir);
if (refusal) throw new DomainError(400, refusal);
}
/** A dump of the database a restore is about to replace, in the data directory. */
keepCopyBeforeRestore(): Promise<string | null> {
const target = path.join(this.paths.dataDir, `pre-restore-${Date.now()}.pgdump`);
return keepRestoreCopy(target, (copy) => this.snapshot(copy));
}
replace(source: string): Promise<{ reopenError: unknown }> {
return replaceThenReopen(
async () => {
const { url, binDir } = this.connection;
// A rejection with no reason still has to fail the swap, so it is never swallowed.
await pgRestore(url, source, binDir).catch((err: unknown) => {
throw err ?? new Error('pg_restore failed');
});
},
() => this.lifecycle.reopen(),
);
}
}
@@ -1,29 +1,22 @@
import { MaintenanceRepository } from '../../db/repositories/MaintenanceRepository';
import { logInfo, logWarn } from '../audit/audit-log.logger';
import {
DatabaseConnectionLostError,
type DatabaseBackupRefusal,
type DatabaseBackupStrategy,
} from '../database/database-backup.interface';
import { SqliteMaintenance } from '../../db/engine/sqlite/sqlite-maintenance';
import { DomainError } from '../common/domain-error';
import type { DatabaseBackupStrategy } from '../database/database-backup.interface';
import { DatabaseLifecycle } from '../database/database-lifecycle.service';
import { checkBackupDatabase } from './backup-archive';
import { keepRestoreCopy, replaceThenReopen } from './database-swap';
import { Injectable } from '@nestjs/common';
import fs from 'fs';
import path from 'path';
const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err));
/**
* The backup port for the one engine TREK runs on: a SQLite file.
* The backup port on a SQLite file, the engine TREK runs on by default.
*
* Every step is what `backup.impl.ts` and `demo/demo-reset.ts` did inline
* before the port existed. A snapshot is `VACUUM INTO`, which takes a
* consistent copy even under concurrent writers. A restore closes the
* connection, swaps the file by copy and rename (a crash mid-swap leaves the
* old or the new file, never neither), drops the old `-wal`/`-shm` sidecars,
* and reopens through `DatabaseLifecycle`, which migrates the restored file
* forward.
* A snapshot is `VACUUM INTO`, which takes a consistent copy even under
* concurrent writers. A restore closes the connection, swaps the file by copy
* and rename (a crash mid-swap leaves the old or the new file, never neither),
* drops the old `-wal`/`-shm` sidecars, and reopens through `DatabaseLifecycle`,
* which migrates the restored file forward.
*
* The file is always `DatabaseLifecycle.file`, the path the connection was
* opened on, so `TREK_DB_FILE` moves the backup, the restore and the demo reset
@@ -35,7 +28,7 @@ export class SqliteDatabaseBackup implements DatabaseBackupStrategy {
constructor(
private readonly lifecycle: DatabaseLifecycle,
private readonly maintenance: MaintenanceRepository,
private readonly maintenance: SqliteMaintenance,
) {}
location(): string {
@@ -62,66 +55,42 @@ export class SqliteDatabaseBackup implements DatabaseBackupStrategy {
}
}
verify(extractDir: string): DatabaseBackupRefusal | null {
return checkBackupDatabase(extractDir);
/** Throws a DomainError carrying the refusal when the archive's database cannot be restored here. */
async verify(extractDir: string): Promise<void> {
const refusal = checkBackupDatabase(extractDir);
if (refusal) throw new DomainError(refusal.status, refusal.error);
}
/**
* A copy of the database a restore is about to replace, next to it. The swap
* deletes the current file, and a restore of the wrong archive used to leave
* nothing to go back to.
*/
async keepCopyBeforeRestore(): Promise<string | null> {
/** A copy of the database a restore is about to replace, next to it: the swap deletes the current file. */
keepCopyBeforeRestore(): Promise<string | null> {
const target = path.join(path.dirname(this.lifecycle.file), `pre-restore-${Date.now()}.db`);
try {
await this.snapshot(target);
logInfo(`Restore: the replaced database was kept as ${target}`);
return target;
} catch (err) {
logWarn(`Restore: could not keep a copy of the current database (${describeError(err)})`);
return null;
}
return keepRestoreCopy(target, (copy) => this.snapshot(copy));
}
async replace(source: string): Promise<{ reopenError: unknown }> {
const dest = this.lifecycle.file;
this.lifecycle.close();
let swapFailed = false;
let swapError: unknown = null;
try {
// Copy to a temp file on the SAME filesystem, drop the old sidecars (they
// belong to the database being replaced and would corrupt the new one),
// then rename into place. The rename is atomic.
const tmp = dest + '.restore-tmp';
fs.copyFileSync(source, tmp);
for (const ext of ['-wal', '-shm']) {
try {
fs.unlinkSync(dest + ext);
} catch {
/* no sidecar to drop */
}
}
fs.renameSync(tmp, dest);
} catch (err) {
swapFailed = true;
swapError = err;
}
// Reopening must always run, even when the swap threw, so the process is
// never left without a connection. After a swap that landed, a reopen
// failure is reported, not thrown: the files are in place and the caller
// has to say "restart". After a swap that failed, both errors go up
// together, because the swap's alone would hide that the connection is gone.
let reopenError: unknown = null;
try {
await this.lifecycle.reopen();
} catch (err) {
reopenError = err;
}
if (swapFailed) {
if (reopenError) throw new DatabaseConnectionLostError(swapError, reopenError);
throw swapError;
}
return { reopenError };
return replaceThenReopen(
() => swapFile(source, dest),
() => this.lifecycle.reopen(),
);
}
}
/**
* Puts `source` at `dest` by copying it to a temp file on the same filesystem
* and renaming it into place. The old sidecars go first: they belong to the
* database being replaced and would corrupt the new one.
*/
function swapFile(source: string, dest: string): void {
const tmp = dest + '.restore-tmp';
fs.copyFileSync(source, tmp);
for (const ext of ['-wal', '-shm']) {
try {
fs.unlinkSync(dest + ext);
} catch {
/* no sidecar to drop */
}
}
fs.renameSync(tmp, dest);
}
+2
View File
@@ -0,0 +1,2 @@
/** The message of an error, or the value itself as text when it is not an Error (a thrown string, say). */
export const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err));
@@ -1,14 +1,15 @@
import { describeError } from '../common/describe-error';
/**
* How a backup, a restore and the demo reset get at the core database, without
* knowing which engine holds it.
*
* The admin backup (`nest/backup/backup.impl.ts`), its restore, and the demo
* baseline save and hourly reset (`demo/demo-reset.ts`) all go through this
* port. The only implementation today is `SqliteDatabaseBackup`
* (`nest/backup/`), which does what those paths always did: `VACUUM INTO` for a
* consistent copy of the file, a file swap under a closed connection for a
* restore. An engine that is not one local file would plug in here with a
* logical dump and load instead.
* port. Each engine has an adapter in `nest/backup/`, chosen by the engine in
* use: SQLite copies the file (`VACUUM INTO`) and swaps it under a closed
* connection; Postgres dumps and restores it with pg_dump and pg_restore. The
* two adapters answer every method the same way, so the callers do not branch.
*
* Lives in the shared database kernel, not in the backup domain, so the admin
* domain (the demo reset job, the baseline route) can inject it without
@@ -42,10 +43,11 @@ export interface DatabaseBackupStrategy {
snapshot(target: string): Promise<void>;
/**
* Checks an unpacked archive's database before anything is replaced. Returns
* null when it can be restored, or the refusal in the restore route's words.
* Checks an unpacked archive's database before anything is replaced. Resolves
* when it can be restored; otherwise throws a DomainError (status 400) that
* carries the refusal in the restore route's words.
*/
verify(extractDir: string): DatabaseBackupRefusal | null;
verify(extractDir: string): Promise<void>;
/**
* Keeps a copy of the database a restore is about to replace. Best effort: a
@@ -67,7 +69,7 @@ export interface DatabaseBackupStrategy {
replace(source: string): Promise<{ reopenError: unknown }>;
}
/** A refusal in the restore route's own words and status. */
/** A refusal in the restore route's own words and status, as `verify` reports it. */
export interface DatabaseBackupRefusal {
error: string;
status: number;
@@ -76,8 +78,6 @@ export interface DatabaseBackupRefusal {
/** Injection token for the active `DatabaseBackupStrategy`. */
export const DATABASE_BACKUP = Symbol('DATABASE_BACKUP');
const describeError = (err: unknown): string => (err instanceof Error ? err.message : String(err));
/**
* Thrown by `replace` when the swap failed and the reopen that always follows
* failed too. The process is left without a database connection until a
@@ -0,0 +1,30 @@
import { readEnv, type DbSettings } from '../../app-config';
import { resolveEngine, type DatabaseEngine } from '../../db/engine';
import { Module } from '@nestjs/common';
/** `readEnv().db`, read once per container: what decides which engine runs and its settings. */
export const DATABASE_SETTINGS = Symbol('DATABASE_SETTINGS');
/**
* The one `DatabaseEngine` of this process. Every injector (the lifecycle, the
* unit of work, the backup port and its adapters) gets this instance, so the
* state an engine keeps (Postgres's reopen hook) is the same wherever it is used.
*/
export const DATABASE_ENGINE = Symbol('DATABASE_ENGINE');
/**
* Provides the engine and its settings. Not global: each module that needs
* the engine imports it, and Nest keeps one instance per container.
*/
@Module({
providers: [
{ provide: DATABASE_SETTINGS, useFactory: (): DbSettings => readEnv().db },
{
provide: DATABASE_ENGINE,
useFactory: (db: DbSettings): DatabaseEngine => resolveEngine(db),
inject: [DATABASE_SETTINGS],
},
],
exports: [DATABASE_SETTINGS, DATABASE_ENGINE],
})
export class DatabaseEngineModule {}
@@ -1,3 +1,4 @@
import { DatabaseEngineModule } from './database-engine.module';
import { DatabaseLifecycle } from './database-lifecycle.service';
import { Module } from '@nestjs/common';
@@ -6,5 +7,5 @@ import { Module } from '@nestjs/common';
* import it by name, so a partial test harness that never touches the
* connection lifecycle does not have to supply the `MikroORM` it injects.
*/
@Module({ providers: [DatabaseLifecycle], exports: [DatabaseLifecycle] })
@Module({ imports: [DatabaseEngineModule], providers: [DatabaseLifecycle], exports: [DatabaseLifecycle] })
export class DatabaseLifecycleModule {}
@@ -1,8 +1,8 @@
import { closeDb, getRawConnection, registerReinitializeHook, reinitialize } from '../../db/database';
import { resolveDbPath } from '../../db/db-path';
import { runSchemaBootstrap } from '../../db/orm';
import type { DatabaseEngine } from '../../db/engine';
import { DATABASE_ENGINE } from './database-engine.module';
import { MikroORM } from '@mikro-orm/core';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
/**
* The one owner of the core database connection's lifecycle: open, close and
@@ -13,18 +13,21 @@ import { Injectable } from '@nestjs/common';
* replace that module wholesale with `vi.mock`. What moved here is every
* decision about WHEN it opens and closes: `buildApp()` opens through this,
* a restore and the demo reset close and reopen through this (via the backup
* port), and the shutdown closes through this. Nothing opens the database at
* import any more; the first connect does, and this provider makes it explicit.
* port), and the shutdown closes through this. The ORM's first connect is what
* actually opens the file; this provider makes that step explicit.
*
* Only the SQLite file engine exists today. The engine specific parts (the file
* path, the swap) are confined to `file` and to the backup port's SQLite
* implementation, so another engine changes this provider and that one class.
* What differs by engine (a file to open or not, how a reopen reaches the ORM)
* is the engine's (`db/engine/`), the one `DATABASE_ENGINE` provides: this
* provider decides when, the engine decides how.
*/
@Injectable()
export class DatabaseLifecycle {
private readonly reopenListeners: Array<() => Promise<void>> = [];
constructor(private readonly orm: MikroORM) {}
constructor(
private readonly orm: MikroORM,
@Inject(DATABASE_ENGINE) private readonly engine: DatabaseEngine,
) {}
/**
* The database file this process runs on, resolved the one way every caller
@@ -36,31 +39,29 @@ export class DatabaseLifecycle {
}
/**
* Opens the connection, binds the ORM to its later swaps and brings the schema
* to head (legacy baseline, migrations, the deferred index, seeders, demo
* seed, in that order: see `db/orm.ts`). Called once by `buildApp()`, before
* anything reads the database.
* Opens the connection, points the engine's reopen at the ORM and brings the
* schema to head (on SQLite: legacy baseline, migrations, the deferred index,
* seeders, demo seed, in that order, see `db/orm.ts`; on Postgres: the
* baseline or the migrations after it, then the same seeding). Called once
* by `buildApp()`, before anything reads the database.
*/
async open(): Promise<void> {
// Normally a no-op: the ORM's first connect inside NestFactory.create()
// already opened it. Said here so the boot does not depend on that.
getRawConnection();
registerReinitializeHook(() => this.rebindOrm());
await runSchemaBootstrap(this.orm);
this.engine.open(this.orm, () => this.afterReopen());
await this.engine.schemaBootstrap(this.orm, { snapshot: true });
}
/** Checkpoints and closes the connection. A closed connection stays closed until `reopen()`. */
close(): void {
closeDb();
this.engine.close();
}
/**
* Reopens the connection after its file was replaced, rebuilds the ORM's
* cached client around the new handle and migrates the new file forward (a
* restored backup may predate this release).
* Reopens the connection after the database was replaced. The engine points
* the ORM at the new database first; then `afterReopen` migrates it forward
* and runs the listeners.
*/
async reopen(): Promise<void> {
await reinitialize();
await this.engine.reopen();
}
/**
@@ -78,19 +79,12 @@ export class DatabaseLifecycle {
}
/**
* Kysely caches whatever handle it was given, so without this the ORM would
* keep talking to the closed one. Closing and reconnecting sends the driver
* back through `createKyselyDialect()`, which picks up the new handle.
* What follows a reopen once the engine has pointed the ORM at the reopened
* database. No pre-migrate snapshot: the file was just unpacked from an
* archive that is still there to go back to.
*/
private async rebindOrm(): Promise<void> {
const connection = this.orm.em.getConnection();
// The handle is owned and closed by `db/database.ts`; the bound driver's
// `destroy()` is a no-op (see `orm-driver.ts`), so this close never reaches it.
await connection.close(true);
await connection.connect();
// No pre-migrate snapshot: the file was just unpacked from an archive that
// is still there to go back to.
await runSchemaBootstrap(this.orm, { snapshot: false });
private async afterReopen(): Promise<void> {
await this.engine.schemaBootstrap(this.orm, { snapshot: false });
for (const listener of this.reopenListeners) await listener();
}
}
+12 -5
View File
@@ -1,14 +1,16 @@
import { SqliteMaintenance } from '../../db/engine/sqlite/sqlite-maintenance';
import { MaintenanceRepository } from '../../db/repositories/MaintenanceRepository';
import { EntityManager } from '@mikro-orm/core';
import { Module } from '@nestjs/common';
/**
* Provides `MaintenanceRepository`, the home of the few whole-database
* statements no entity can express (the readiness ping, the WAL checkpoint,
* `VACUUM INTO`, the plugin-table erasure).
* statements no entity can express (the readiness ping, the plugin-table
* erasure), and `SqliteMaintenance`, the SQLite file's own (the WAL
* checkpoint, `VACUUM INTO`).
*
* A factory over the context-resolving global `EntityManager`, the same one
* Nest injects into every service, so the repository resolves the request or
* Factories over the context-resolving global `EntityManager`, the same one
* Nest injects into every service, so each resolves the request or
* transaction fork that is active when a method runs. The class itself stays
* free of Nest, like every other file under `db/`.
*/
@@ -19,7 +21,12 @@ import { Module } from '@nestjs/common';
useFactory: (em: EntityManager) => new MaintenanceRepository(em),
inject: [EntityManager],
},
{
provide: SqliteMaintenance,
useFactory: (em: EntityManager) => new SqliteMaintenance(em),
inject: [EntityManager],
},
],
exports: [MaintenanceRepository],
exports: [MaintenanceRepository, SqliteMaintenance],
})
export class MaintenanceModule {}
+6 -4
View File
@@ -1,3 +1,4 @@
import { DatabaseEngineModule } from './database-engine.module';
import { UnitOfWork } from './unit-of-work';
import { Global, Module } from '@nestjs/common';
@@ -10,10 +11,11 @@ import { Global, Module } from '@nestjs/common';
* (`Test.createTestingModule({ imports: [DatabaseModule, ...] })`)
* with no ORM in the container: a provider there that injects the core
* `EntityManager` fails every one of them at `compile()`, because Nest
* instantiates providers eagerly. Nothing is imported here — the
* `EntityManager` token comes from `MikroOrmModule.forRoot`, whose
* `MikroOrmCoreModule` is itself `@Global()`.
* instantiates providers eagerly. The `EntityManager` token comes from
* `MikroOrmModule.forRoot`, whose
* `MikroOrmCoreModule` is itself `@Global()`. It imports `DatabaseEngineModule`,
* the engine the unit of work serializes its writes through.
*/
@Global()
@Module({ providers: [UnitOfWork], exports: [UnitOfWork] })
@Module({ imports: [DatabaseEngineModule], providers: [UnitOfWork], exports: [UnitOfWork] })
export class OrmModule {}
+16 -3
View File
@@ -1,5 +1,7 @@
import type { DatabaseEngine } from '../../db/engine';
import { DATABASE_ENGINE } from './database-engine.module';
import { EntityManager } from '@mikro-orm/core';
import { Injectable } from '@nestjs/common';
import { Inject, Injectable } from '@nestjs/common';
/**
* The one way a service opens a transaction.
@@ -17,12 +19,23 @@ import { Injectable } from '@nestjs/common';
*
* Never awaits network I/O inside: the SQLite connection is one Kysely client
* with a mutex, and an open transaction holds it for every other request.
*
* The services read `MAX(...) + 1` inside these transactions on the strength
* of SQLite's single writer. On Postgres a pool of connections would let two
* of them read the same MAX, so every transaction first takes the engine's
* write lock (`DatabaseEngine.serializeWrites`), which keeps that guarantee.
*/
@Injectable()
export class UnitOfWork {
constructor(private readonly em: EntityManager) {}
constructor(
private readonly em: EntityManager,
@Inject(DATABASE_ENGINE) private readonly engine: DatabaseEngine,
) {}
transactional<T>(fn: () => Promise<T>): Promise<T> {
return this.em.transactional(() => fn());
return this.em.transactional(async (em) => {
await this.engine.serializeWrites(em);
return fn();
});
}
}
+17
View File
@@ -1,9 +1,26 @@
import type { PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository';
import { haversineMetres } from '../common/geo';
import { toRowId } from '../common/row-id';
import type { KmlImportSummary } from '../place-import/place-import.types';
import type { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
import { externalIdsOf, normalizePlaceName, placeMatchStrategies, type PlaceMatchCandidate } from '@trek/shared';
/**
* The category and tag ids a place list filters by, from their query-string
* values. An absent or empty filter is left out; an id that is not canonical
* (`toRowId`) matches no place, so the list is `null` and the answer is empty
* without a query. The repository binds numbers only.
*/
export function placeListIdFilters(filters: {
category?: string;
tag?: string;
}): { category?: number; tag?: number } | null {
const category = filters.category ? toRowId(filters.category) : undefined;
const tag = filters.tag ? toRowId(filters.tag) : undefined;
if (category === null || tag === null) return null;
return { category, tag };
}
/**
* Pure helpers and module-scope constants of the places domain, moved verbatim
* out of the legacy services/placeService.ts when it went DI-native. Same
+7 -9
View File
@@ -34,7 +34,7 @@ import { TripMembers } from '../../db/entities/TripMembers.entity';
import { DayAssignments } from '../../db/entities/DayAssignments.entity';
import { Categories } from '../../db/entities/Categories.entity';
import { Trips } from '../../db/entities/Trips.entity';
import type { PlacesRepository, PlaceListRow, PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository';
import type { PlacesRepository, PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository';
import type { TagsRepository } from '../../db/repositories/Tags.repository';
import type { PlaceRatingsRepository } from '../../db/repositories/PlaceRatings.repository';
import type { TripMembersRepository } from '../../db/repositories/TripMembers.repository';
@@ -73,6 +73,7 @@ import {
type ListImportOptions,
type ListImportResult,
type PlaceImportResult,
placeListIdFilters,
} from './places.helpers';
type Trip = TripAccess;
@@ -247,16 +248,13 @@ export class PlacesService {
filters: { search?: string; category?: string; tag?: string; assignment?: 'all' | 'unassigned' | 'assigned' },
) {
// ESCAPE so a `%` or `_` the user typed matches literally instead of
// acting as a LIKE wildcard (a bare '%' used to return the whole trip) —
// the coercion stays here (the service), the repository binds the
// acting as a LIKE wildcard (a bare `%` would match every place in the
// trip). The coercion stays here (the service); the repository binds the
// already-built pattern verbatim (PL3).
const searchPattern = filters.search ? `%${escapeLikePattern(filters.search)}%` : undefined;
const places: PlaceListRow[] = await this.placesRepo.listForTrip(tripId, {
searchPattern,
category: filters.category,
tag: filters.tag,
assignment: filters.assignment,
});
const ids = placeListIdFilters(filters);
if (!ids) return [];
const places = await this.placesRepo.listForTrip(tripId, { searchPattern, ...ids, assignment: filters.assignment });
const placeIds = places.map(p => p.id);
const tagsByPlaceId = await this.queryHelpers.loadTagsByPlaceIds(placeIds);