feat(db): copy the database before a boot migrates it

Migrations only go forward, and since #2610 an older image refuses a
database a newer one migrated. Going back after a bad update therefore
needs a copy from before it, and the only copies were backups the admin
takes by hand or auto-backup, which is off by default. The first boot
on 4.4 migrates every 4.3.x install through the legacy baseline and the
new migrations without either.

Before applying pending migrations to a file database that holds
anything, the boot now writes a VACUUM INTO copy next to it, named
pre-migrate-<schema>-<time>.db (legacy-<schema_version> for an install
the positional runner left, otherwise the last recorded migration), and
keeps the newest TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP (3). The copy is
written under a temporary name and renamed, so a crash never leaves a
truncated file that looks good. If it cannot be written the boot refuses
before anything is migrated; TREK_DB_PRE_MIGRATE_SNAPSHOT=false is the
documented way past that. The newer-database refusal names the copy
taken right before the newer release migrated past this one.

A fresh install, a restart with nothing pending, an in-memory database
and a restore's re-bootstrap (the archive is the copy) take none, and
NODE_ENV=test leaves it off unless a suite sets the switch.

server/src/app-config/README.md

@@ -85,6 +85,8 @@ TREK_MANAGED, PLACES_API_BASE, PLACES_API_KEY, AMAP_API_BASE, AMAP_API_KEY, AMAP
 TREK_PLUGINS_ENABLED / _DEV_LINK / _IGNORE_TREK_RANGE / _DIR / _DATA_DIR / TREK_PLUGIN_PERMISSIONS,
 OIDC_*, SMTP_*, FORCE_HTTPS, COOKIE_SECURE, HSTS_INCLUDE_SUBDOMAINS,
 ALLOWED_ORIGINS, UNSPLASH_ACCESS_KEY, WEBAUTHN_*, TZ, ADMIN_EMAIL,
+TREK_DB_PRE_MIGRATE_SNAPSHOT(_KEEP) (read when a migration run starts; the
+legacy-upgrade suites set it per file),
 ADMIN_PASSWORD, IDEMPOTENCY_TTL_SECONDS, MCP_RATE_LIMIT (request-path check),
 LLM_TIMEOUT_MS, NOMINATIM_URL, VAPID_* (resolved per use by the Web Push key
 service, which the notification suites build without the container).

server/src/app-config/derive.ts

@@ -335,6 +335,14 @@ export function deriveDb(raw: RawEnv) {
     synchronous: durability.synchronous,
     /** Complaints about unusable values; derivation stays side-effect free, so db/durability.ts logs them when it opens the file. */
     durabilityWarnings: durability.warnings,
+    /**
+     * Snapshot the database before a boot migrates it. Undefined when unset: the
+     * caller then decides (on, except under NODE_ENV=test). false is the escape
+     * hatch for a data dir that cannot hold a second copy.
+     */
+    preMigrateSnapshot: parseBool(raw.TREK_DB_PRE_MIGRATE_SNAPSHOT),
+    /** How many pre-migration snapshots stay in the data dir, newest first. */
+    preMigrateSnapshotKeep: Math.min(positiveIntOr(raw.TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP, 3), 100),
   };
 }
 

server/src/app-config/env.schema.ts

@@ -158,6 +158,12 @@ export const envSchema = z.object({
   // this schema at all. Resolution lives in parsers.resolveDurability().
   TREK_DB_JOURNAL_MODE: anyString,
   TREK_DB_SYNCHRONOUS: anyString,
+  // The copy of the database taken before a boot applies pending migrations.
+  // Unlike the two pragmas a typo here refuses the boot: the switch exists to
+  // turn a safety net off, and a misspelt "off" must not leave it on silently
+  // or a misspelt "on" turn it off.
+  TREK_DB_PRE_MIGRATE_SNAPSHOT: boolStr,
+  TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP: integer(1, 100),
   TREK_WIKI_DIR: anyString,
   TREK_PLACE_PHOTO_DIR: anyString,
   BACKUP_UPLOAD_LIMIT_MB: positiveNumber,

server/src/db/legacy-baseline.ts

@@ -38,7 +38,39 @@ import { knownMigrationNames, unknownMigrations } from './known-migrations';
  */
 const STEP_MARKER = /Legacy migration step (\d+)\b/g;
 
-const REFUSAL = '[DB] Refusing to boot:';
+export const REFUSAL = '[DB] Refusing to boot:';
+
+/**
+ * What the boot does around a migration run (`pre-migrate-snapshot.ts` in
+ * production, nothing in the unit tests and on a restore).
+ */
+export interface MigrationSafetyNet {
+  /**
+   * Runs after the plan is known and before anything is written, only when
+   * something is pending. `from` labels the schema the database holds: the
+   * last recorded migration's timestamp, `legacy-<schema_version>`, or null
+   * when nothing is recorded at all. Throwing refuses the boot.
+   */
+  beforeMigrate(from: string | null): Promise<void>;
+  /** A sentence pointing at a copy this release can run, for the newer-database refusal. */
+  restoreHint(thisRelease: string): Promise<string | null>;
+}
+
+export const NO_SAFETY_NET: MigrationSafetyNet = {
+  beforeMigrate: async () => {},
+  restoreHint: async () => null,
+};
+
+/** The part of a migration name that orders it: what a schema state is labelled by. */
+export function migrationLabel(name: string): string {
+  return /^Migration(\d+)_/.exec(name)?.[1] ?? name;
+}
+
+function latest(names: Iterable<string>): string | null {
+  let last: string | null = null;
+  for (const name of names) if (last === null || name > last) last = name;
+  return last;
+}
 
 export interface LegacyStepMap {
   /** The migrations standing for `createTables()`: everything that sorts ahead of step 1. */
@@ -184,12 +216,23 @@ export async function migrateToHead(
   migrator: Migrator,
   readSource: ReadSource = readFromDisk,
   known: Set<string> = knownMigrationNames(),
+  safetyNet: MigrationSafetyNet = NO_SAFETY_NET,
 ): Promise<void> {
   const baselined = await planLegacyBaseline(connection, migrator, readSource);
-  refuseNewerDatabase((await migrator.getExecuted()).map((row) => row.name), known);
+  const executed = (await migrator.getExecuted()).map((row) => row.name);
+  if (unknownMigrations(executed, known).length > 0) {
+    const thisRelease = latest(known);
+    refuseNewerDatabase(executed, known, thisRelease ? await safetyNet.restoreHint(migrationLabel(thisRelease)) : null);
+  }
   const already = new Set(baselined);
   const pending = (await migrator.getPending()).filter((migration) => !already.has(migration.name));
   if (pending.length === 0 && baselined.length === 0) return;
+
+  const lastRecorded = latest(executed);
+  const legacy = lastRecorded === null ? await legacyVersion(connection, migrator) : null;
+  await safetyNet.beforeMigrate(
+    lastRecorded !== null ? migrationLabel(lastRecorded) : legacy !== null ? `legacy-${legacy}` : null,
+  );
   console.log(`[DB] Applying ${pending.length} pending migration(s)`);
   if (baselined.length === 0) {
     await migrator.up();
@@ -210,11 +253,12 @@ export async function migrateToHead(
  * older code against a schema it does not know and fail later, far from the
  * cause. Refused before anything is written.
  */
-export function refuseNewerDatabase(executed: string[], known: Set<string>): void {
+export function refuseNewerDatabase(executed: string[], known: Set<string>, restoreHint: string | null = null): void {
   const unknown = unknownMigrations(executed, known);
   if (unknown.length === 0) return;
   throw new Error(
     `${REFUSAL} the database was migrated by a newer TREK (${unknown.length} unknown migration(s), latest ${unknown[unknown.length - 1]}). ` +
-      'Run that version again, or restore a backup taken with this one.',
+      'Run that version again, or restore a backup taken with this one.' +
+      (restoreHint ? ` ${restoreHint}` : ''),
   );
 }

server/src/db/orm.ts

@@ -1,6 +1,7 @@
 import mikroOrmConfig from '../mikro-orm.config';
 import { registerReinitializeHook, runDemoSeed } from './database';
-import { migrateToHead } from './legacy-baseline';
+import { migrateToHead, NO_SAFETY_NET } from './legacy-baseline';
+import { preMigrateSnapshot } from './pre-migrate-snapshot';
 import { withRequestContext } from '../nest/database/request-context';
 import type { AnyEntity, EntityClass, EntityManager, EntitySchema, IDatabaseDriver, MikroORM } from '@mikro-orm/core';
 import type { Migrator } from '@mikro-orm/migrations';
@@ -29,14 +30,21 @@ type AnyOrm = MikroORM<
   readonly (string | EntityClass<AnyEntity> | EntitySchema)[]
 >;
 
-/** Migrate to head, then seed. Safe to call again — both halves are idempotent. */
-export async function runSchemaBootstrap(orm: AnyOrm): Promise<void> {
+/**
+ * Migrate to head, then seed. Safe to call again, both halves are idempotent.
+ *
+ * `snapshot: false` skips the pre-migration copy (`pre-migrate-snapshot.ts`).
+ * Only the restore hook passes it: the file it migrates was just unpacked from
+ * a backup archive, which is still there to go back to.
+ */
+export async function runSchemaBootstrap(orm: AnyOrm, { snapshot = true }: { snapshot?: boolean } = {}): Promise<void> {
   const migrator = orm.config.getExtension('@mikro-orm/migrator') as Migrator;
+  const connection = orm.em.getConnection();
   // A database the retired positional runner migrated has no migrator rows yet;
   // what it already has is recorded in the same transaction as the run, or the
   // run would replay the whole history over it (see legacy-baseline.ts). Every
   // other database just gets its pending migrations.
-  await migrateToHead(orm.em.getConnection(), migrator);
+  await migrateToHead(connection, migrator, undefined, undefined, snapshot ? preMigrateSnapshot(connection) : NO_SAFETY_NET);
 
   const defaultSeeder = orm.config.get('seeder').defaultSeeder;
   if (defaultSeeder) await orm.seeder.seedString(defaultSeeder);
@@ -77,7 +85,7 @@ export function attachOrm(orm: AnyOrm): void {
     // `destroy()` is a no-op (see `orm-driver.ts`), so this close never reaches it.
     await connection.close(true);
     await connection.connect();
-    await runSchemaBootstrap(orm);
+    await runSchemaBootstrap(orm, { snapshot: false });
   });
 }
 

server/src/db/pre-migrate-snapshot.ts

@@ -0,0 +1,147 @@
+import { readEnv, type AppEnv } from '../app-config';
+import { REFUSAL, type MigrationSafetyNet } from './legacy-baseline';
+import type { Connection } from '@mikro-orm/core';
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * A copy of the database taken right before a boot migrates it.
+ *
+ * Migrations only go forward, and a database a newer TREK migrated is refused
+ * by the older one (`refuseNewerDatabase`). Without a copy from before the
+ * upgrade, going back to the previous image means restoring a backup, and
+ * automatic backups are off by default. So the boot takes one itself: a
+ * `VACUUM INTO` snapshot next to the database file, named after the schema it
+ * holds, e.g. `pre-migrate-legacy-244-20261008T091500Z.db` for a v4.3.3
+ * install or `pre-migrate-20200101042300-….db` for one the migrator already
+ * owns. The newest `TREK_DB_PRE_MIGRATE_SNAPSHOT_KEEP` stay.
+ *
+ * Only when something is pending and the database holds anything: a restart
+ * with nothing to apply, a fresh install and an in-memory database take none.
+ * A snapshot that cannot be written refuses the boot before anything is
+ * migrated; `TREK_DB_PRE_MIGRATE_SNAPSHOT=false` is the way past that.
+ */
+
+const PREFIX = 'pre-migrate-';
+const SNAPSHOT_FILE = /^pre-migrate-.+\.db$/;
+
+/** `20261008T091500Z`: sortable, and free of the colons Windows refuses in a file name. */
+function stamp(now: Date): string {
+  return now
+    .toISOString()
+    .replace(/\.\d+Z$/, 'Z')
+    .replace(/[-:]/g, '');
+}
+
+/** The file behind the connection, or null for `:memory:` and temp databases. */
+async function databaseFile(connection: Connection): Promise<string | null> {
+  const rows: Array<{ name: string; file: string }> = await connection.execute('PRAGMA database_list');
+  const main = rows.find((row) => row.name === 'main');
+  return main?.file ? main.file : null;
+}
+
+/** A fresh install: nothing but the migrator's own bookkeeping table yet. */
+async function holdsAnything(connection: Connection): Promise<boolean> {
+  const rows: unknown[] = await connection.execute(
+    `SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name <> 'mikro_orm_migrations' LIMIT 1`,
+  );
+  return rows.length > 0;
+}
+
+/** The snapshots in `dir`, newest first. */
+export function listSnapshots(dir: string): string[] {
+  let entries: string[];
+  try {
+    entries = fs.readdirSync(dir).filter((entry) => SNAPSHOT_FILE.test(entry));
+  } catch {
+    return [];
+  }
+  return entries
+    .map((entry) => ({ file: path.join(dir, entry), mtime: fs.statSync(path.join(dir, entry)).mtimeMs }))
+    .sort((a, b) => b.mtime - a.mtime || b.file.localeCompare(a.file))
+    .map((entry) => entry.file);
+}
+
+/** Deletes all but the newest `keep`. A failure only warns: the new snapshot is already safe. */
+export function pruneSnapshots(dir: string, keep: number): void {
+  for (const file of listSnapshots(dir).slice(keep)) {
+    try {
+      fs.rmSync(file, { force: true });
+    } catch (err) {
+      console.warn(
+        `[DB] Could not delete the old pre-migration copy ${file}: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+}
+
+/**
+ * Writes the copy under a temporary name and renames it into place, so a crash
+ * halfway never leaves a truncated file that looks like a good snapshot.
+ */
+async function writeSnapshot(connection: Connection, file: string, from: string, now: Date): Promise<string> {
+  const dir = path.dirname(file);
+  const target = path.join(dir, `${PREFIX}${from}-${stamp(now)}.db`);
+  const partial = `${target}.partial`;
+  try {
+    fs.rmSync(partial, { force: true });
+    await connection.execute(`VACUUM INTO '${partial.replaceAll("'", "''")}'`);
+    fs.renameSync(partial, target);
+  } catch (err) {
+    try {
+      fs.rmSync(partial, { force: true });
+    } catch {
+      // The refusal below is what matters; a stray .partial is ignored by listSnapshots.
+    }
+    throw new Error(
+      `${REFUSAL} could not write a copy of the database to ${target} before migrating it ` +
+        `(${err instanceof Error ? err.message : String(err)}). Nothing was migrated. Free some space in ${dir} ` +
+        'or fix its permissions, or set TREK_DB_PRE_MIGRATE_SNAPSHOT=false to migrate without a copy.',
+      { cause: err },
+    );
+  }
+  return target;
+}
+
+/** The boot's safety net: snapshot before migrating, and point at the right copy when refusing. */
+export function preMigrateSnapshot(
+  connection: Connection,
+  env: () => AppEnv = readEnv,
+  now: () => Date = () => new Date(),
+): MigrationSafetyNet {
+  return {
+    async beforeMigrate(from) {
+      const { db, app } = env();
+      // Unset means on, except under NODE_ENV=test, where a suite asks for it explicitly.
+      if (db.preMigrateSnapshot === undefined && app.isTest) return;
+      const file = await databaseFile(connection);
+      if (!file) return;
+      if (from === null && !(await holdsAnything(connection))) return;
+      if (db.preMigrateSnapshot === false) {
+        console.warn(`[DB] TREK_DB_PRE_MIGRATE_SNAPSHOT is off: migrating ${file} without a copy to go back to`);
+        return;
+      }
+      const target = await writeSnapshot(connection, file, from ?? 'unversioned', now());
+      console.log(`[DB] Copy of the database before migrating: ${target}`);
+      pruneSnapshots(path.dirname(file), db.preMigrateSnapshotKeep);
+    },
+
+    async restoreHint(thisRelease) {
+      const file = await databaseFile(connection);
+      if (!file) return null;
+      const snapshots = listSnapshots(path.dirname(file));
+      const match = snapshots.find((snapshot) => path.basename(snapshot).startsWith(`${PREFIX}${thisRelease}-`));
+      if (match) {
+        return (
+          `The copy taken right before the newer TREK migrated it is ${match}: stop TREK, delete ${file}-wal and ` +
+          `${file}-shm, and copy it over ${file}.`
+        );
+      }
+      if (snapshots.length > 0) {
+        return `Copies taken before earlier migrations are in ${path.dirname(file)} (${PREFIX}*.db), none of them from this release.`;
+      }
+      return null;
+    },
+  };
+}