From e62dceb4bbd6402422f564fef3bb40b794041301 Mon Sep 17 00:00:00 2001 From: Maurice Date: Wed, 29 Jul 2026 16:42:47 +0200 Subject: [PATCH] fix(backup): snapshot the database instead of archiving it live (#1675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scheduled backup archived travel.db directly. The archiver reads its entries during finalize(), so a WAL checkpoint writing pages back mid-stream tore the copy — and the -wal that would make it recoverable isn't in the zip. The manual path was moved to VACUUM INTO for exactly this reason; the scheduler never followed. It calls the same createBackup() now, which also gets it the encryption key and plugin data it was missing, so a scheduled backup is finally restorable onto another install. The pre-rotation copy in migrate-encryption.ts had the same problem: a plain file copy of a live WAL database, and it is the operator's only rollback if a key rotation goes wrong. Auto-backups now exclude the re-derivable photo caches, matching manual ones. --- server/scripts/migrate-encryption.ts | 42 ++++- server/src/scheduler.ts | 33 ++-- server/src/services/backupService.ts | 19 ++- server/tests/unit/auto-backup.test.ts | 156 ++++++++++++++++++ .../tests/unit/services/backupService.test.ts | 28 ++++ wiki/Backups.md | 4 +- 6 files changed, 250 insertions(+), 32 deletions(-) create mode 100644 server/tests/unit/auto-backup.test.ts diff --git a/server/scripts/migrate-encryption.ts b/server/scripts/migrate-encryption.ts index 78e19e1a5..ae8bfdf03 100644 --- a/server/scripts/migrate-encryption.ts +++ b/server/scripts/migrate-encryption.ts @@ -181,11 +181,47 @@ async function main() { // Backup const backupPath = `${dbPath}.backup-${Date.now()}`; - fs.copyFileSync(dbPath, backupPath); - console.log(`\nBackup created: ${backupPath}`); const db = new Database(dbPath); - db.pragma('journal_mode = WAL'); + db.pragma('busy_timeout = 5000'); + + // The rollback copy has to be a consistent snapshot, not a plain file copy: + // this script is documented as running against a live server, so in WAL mode + // the committed tail sits in the -wal sidecar a copy would leave behind. + // VACUUM INTO snapshots even while the server writes; the checkpoint plus + // copy is only the fallback for when it cannot (disk, lock). + try { + db.exec(`VACUUM INTO '${backupPath.replace(/'/g, "''")}'`); + } catch { + db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + fs.copyFileSync(dbPath, backupPath); + } + console.log(`\nBackup created: ${backupPath}`); + + // journal_mode lives in the file header, so hardcoding WAL here would quietly + // undo a deliberate DELETE/TRUNCATE setup (network storage) on the first key + // rotation. This script ships without src/ in the image and cannot import + // src/app-config, so it reads the same variables directly — keep the defaults + // in step with parsers.resolveDurability(). + const journalModes = ['DELETE', 'TRUNCATE', 'PERSIST', 'MEMORY', 'WAL', 'OFF']; + const syncLevels = ['OFF', 'NORMAL', 'FULL', 'EXTRA']; + + const wantedJournal = (process.env.TREK_DB_JOURNAL_MODE ?? '').trim().toUpperCase(); + const journalMode = journalModes.includes(wantedJournal) ? wantedJournal : 'WAL'; + if (wantedJournal && journalMode !== wantedJournal) { + console.warn(`TREK_DB_JOURNAL_MODE="${process.env.TREK_DB_JOURNAL_MODE}" is not a SQLite journal mode — using ${journalMode}.`); + } + + const wantedSync = (process.env.TREK_DB_SYNCHRONOUS ?? '').trim().toUpperCase(); + const defaultSync = journalMode === 'WAL' ? 'NORMAL' : 'FULL'; + const synchronous = syncLevels.includes(wantedSync) ? wantedSync : defaultSync; + if (wantedSync && synchronous !== wantedSync) { + console.warn(`TREK_DB_SYNCHRONOUS="${process.env.TREK_DB_SYNCHRONOUS}" is not a SQLite synchronous level — using ${synchronous}.`); + } + + db.pragma(`journal_mode = ${journalMode}`); + db.pragma(`synchronous = ${synchronous}`); + console.log(`Journal mode: ${db.pragma('journal_mode', { simple: true })}\n`); const result: MigrationResult = { migrated: 0, alreadyMigrated: 0, skipped: 0, errors: [] }; diff --git a/server/src/scheduler.ts b/server/src/scheduler.ts index 5d815b85d..f53758b4e 100644 --- a/server/src/scheduler.ts +++ b/server/src/scheduler.ts @@ -1,5 +1,4 @@ import cron, { type ScheduledTask } from 'node-cron'; -import archiver from 'archiver'; import { readEnv } from './app-config'; import path from 'node:path'; import fs from 'node:fs'; @@ -7,7 +6,6 @@ import { logInfo, logError } from './nest/audit/audit-log.logger'; const dataDir = path.join(__dirname, '../data'); const backupsDir = path.join(dataDir, 'backups'); -const uploadsDir = path.join(__dirname, '../uploads'); const settingsFile = path.join(dataDir, 'backup-settings.json'); const VALID_INTERVALS = ['hourly', 'daily', 'weekly', 'monthly']; @@ -60,31 +58,20 @@ function saveSettings(settings: BackupSettings): void { } async function runBackup(): Promise { - if (!fs.existsSync(backupsDir)) fs.mkdirSync(backupsDir, { recursive: true }); - - const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); - const filename = `auto-backup-${timestamp}.zip`; - const outputPath = path.join(backupsDir, filename); - try { - // Flush WAL to main DB file before archiving - try { const { db } = require('./db/database'); db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) {} - - await new Promise((resolve, reject) => { - const output = fs.createWriteStream(outputPath); - const archive = archiver('zip', { zlib: { level: 9 } }); - output.on('close', resolve); - archive.on('error', reject); - archive.pipe(output); - const dbPath = path.join(dataDir, 'travel.db'); - if (fs.existsSync(dbPath)) archive.file(dbPath, { name: 'travel.db' }); - if (fs.existsSync(uploadsDir)) archive.directory(uploadsDir, 'uploads'); - archive.finalize(); - }); + // Same archive the admin panel builds, only under the auto-backup name: it + // snapshots travel.db with VACUUM INTO instead of archiving the live file + // (the archiver reads its entries during finalize, so a WAL auto-checkpoint + // landing in between tears the copy), and it carries .encryption_key and the + // plugin trees — without the key a scheduled backup cannot be decrypted on + // another install. Imported lazily because backupService imports this module + // for the schedule settings. + const { createBackup } = await import('./services/backupService'); + const { filename } = await createBackup('auto-backup'); logInfo(`Auto-Backup created: ${filename}`); } catch (err: unknown) { + // createBackup removes its own half-written zip before rethrowing. logError(`Auto-Backup: ${err instanceof Error ? err.message : err}`); - if (fs.existsSync(outputPath)) fs.unlinkSync(outputPath); return; } diff --git a/server/src/services/backupService.ts b/server/src/services/backupService.ts index 6c0c66db6..79dd84a3c 100644 --- a/server/src/services/backupService.ts +++ b/server/src/services/backupService.ts @@ -143,14 +143,25 @@ export function listBackups(): BackupInfo[] { // Create backup // --------------------------------------------------------------------------- -export async function createBackup(): Promise { +/** + * Writes a full backup zip and returns its BackupInfo. + * + * `prefix` picks the filename scheme. The scheduler passes 'auto-backup' because + * everything downstream tells the two apart by name: cleanupOldBackups() prunes + * only auto-backup-*.zip, and the admin panel badges them as automatic. Manual + * backups keep the default. + */ +export async function createBackup(prefix: 'backup' | 'auto-backup' = 'backup'): Promise { ensureBackupsDir(); const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19); - const filename = `backup-${timestamp}.zip`; + const filename = `${prefix}-${timestamp}.zip`; const outputPath = path.join(backupsDir, filename); - const pdataSnap = path.join(backupsDir, `.plugins-snap-${timestamp}`); - const dbSnap = path.join(backupsDir, `.travel-snap-${timestamp}.db`); + // The scratch names carry the prefix too: a scheduled run and a manual one that + // start in the same second would otherwise share a snapshot path, and the first + // to finish would delete the other's staging copy mid-archive. + const pdataSnap = path.join(backupsDir, `.plugins-snap-${prefix}-${timestamp}`); + const dbSnap = path.join(backupsDir, `.travel-snap-${prefix}-${timestamp}.db`); try { try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch (e) {} diff --git a/server/tests/unit/auto-backup.test.ts b/server/tests/unit/auto-backup.test.ts new file mode 100644 index 000000000..cdf2c03b2 --- /dev/null +++ b/server/tests/unit/auto-backup.test.ts @@ -0,0 +1,156 @@ +/** + * Scheduled backup run. + * + * The auto-backup used to archive the live travel.db: the archiver reads its + * entries while streaming, so a WAL auto-checkpoint writing pages back mid-run + * left a torn copy in the zip — without the -wal that would make it recoverable. + * It also shipped without .encryption_key, so the archive could not be decrypted + * on another install. The run now goes through backupService.createBackup(), + * which snapshots the DB with VACUUM INTO and bundles the key. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +const fsMock = vi.hoisted(() => ({ + existsSync: vi.fn(() => false), + mkdirSync: vi.fn(), + readFileSync: vi.fn(() => '{}'), + writeFileSync: vi.fn(), + readdirSync: vi.fn(() => [] as unknown as string[]), + statSync: vi.fn(() => ({ size: 0, birthtime: new Date(0), mtimeMs: 0 })), + unlinkSync: vi.fn(), + createWriteStream: vi.fn(), + createReadStream: vi.fn(), + rmSync: vi.fn(), + copyFileSync: vi.fn(), + renameSync: vi.fn(), + cpSync: vi.fn(), + realpathSync: vi.fn((p: string) => p), +})); + +const archiveMock = vi.hoisted(() => ({ + pipe: vi.fn(), + file: vi.fn(), + directory: vi.fn(), + glob: vi.fn(), + finalize: vi.fn(), + on: vi.fn(), +})); + +const archiverMock = vi.hoisted(() => vi.fn()); + +const dbMock = vi.hoisted(() => ({ + db: { exec: vi.fn(), prepare: vi.fn() }, + closeDb: vi.fn(), + reinitialize: vi.fn(), +})); + +const logMock = vi.hoisted(() => ({ logInfo: vi.fn(), logError: vi.fn() })); + +const cronMock = vi.hoisted(() => ({ schedule: vi.fn(), validate: vi.fn(() => true) })); + +vi.mock('node-cron', () => ({ default: cronMock, ...cronMock })); +vi.mock('fs', () => ({ default: fsMock, ...fsMock })); +vi.mock('node:fs', () => ({ default: fsMock, ...fsMock })); +vi.mock('archiver', () => ({ default: archiverMock })); +vi.mock('unzipper', () => ({ default: { Extract: vi.fn(), Open: { file: vi.fn() } } })); +vi.mock('../../src/db/database', () => dbMock); +vi.mock('../../src/nest/audit/audit-log.logger', () => logMock); +vi.mock('../../src/config', () => ({ + JWT_SECRET: 'test-secret', + ENCRYPTION_KEY: 'a'.repeat(64), + updateJwtSecret: () => {}, +})); + +import path from 'node:path'; +import { start } from '../../src/scheduler'; + +const liveDb = path.join(__dirname, '../../data', 'travel.db'); + +/** Wires up the archiver so finalize() resolves the run, and returns its events. */ +function stubArchiver(): Record void> { + const outputEvents: Record void> = {}; + const archiveEvents: Record void> = {}; + fsMock.createWriteStream.mockReturnValue({ + on: vi.fn((event: string, cb: () => void) => { outputEvents[event] = cb; }), + } as never); + archiveMock.on.mockImplementation((event: string, cb: (arg?: unknown) => void) => { archiveEvents[event] = cb; }); + archiveMock.finalize.mockImplementation(() => { outputEvents['close']?.(); }); + archiverMock.mockReturnValue(archiveMock); + return archiveEvents; +} + +/** Starts the scheduler with auto-backup enabled and hands back the cron callback. */ +function scheduledRun(): () => Promise { + fsMock.readFileSync.mockReturnValue(JSON.stringify({ enabled: true, interval: 'daily', keep_days: 7 })); + start(); + return cronMock.schedule.mock.calls.at(-1)?.[1] as () => Promise; +} + +describe('auto-backup run', () => { + const envKey = process.env.ENCRYPTION_KEY; + + beforeEach(() => { + vi.clearAllMocks(); + // No ENCRYPTION_KEY in the env means the key file is the source of truth and + // has to travel with the backup. + delete process.env.ENCRYPTION_KEY; + fsMock.statSync.mockReturnValue({ size: 4096, birthtime: new Date('2026-04-27T02:00:00Z'), mtimeMs: Date.now() } as never); + }); + + afterEach(() => { + process.env.ENCRYPTION_KEY = envKey; + }); + + it('archives a VACUUM INTO snapshot, never the live travel.db', async () => { + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('travel.db') || String(p).endsWith('backup-settings.json')); + stubArchiver(); + + await scheduledRun()(); + + expect(dbMock.db.exec).toHaveBeenCalledWith(expect.stringContaining('VACUUM INTO')); + const dbEntry = archiveMock.file.mock.calls.find(([, opts]) => opts?.name === 'travel.db'); + expect(dbEntry).toBeDefined(); + expect(dbEntry?.[0]).toContain('.travel-snap-auto-backup-'); + expect(dbEntry?.[0]).not.toBe(liveDb); + }); + + it('bundles the at-rest encryption key', async () => { + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('.encryption_key') || String(p).endsWith('backup-settings.json')); + stubArchiver(); + + await scheduledRun()(); + + expect(archiveMock.file).toHaveBeenCalledWith( + expect.stringContaining('.encryption_key'), + { name: '.encryption_key' }, + ); + }); + + it('keeps the auto-backup-*.zip naming so retention still prunes it', async () => { + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('backup-settings.json')); + fsMock.readdirSync.mockReturnValue(['auto-backup-2020-01-01T02-00-00.zip'] as unknown as string[]); + stubArchiver(); + + await scheduledRun()(); + + expect(logMock.logInfo).toHaveBeenCalledWith(expect.stringMatching(/^Auto-Backup created: auto-backup-[\dT-]+\.zip$/)); + // cleanupOldBackups(keep_days) still runs after the archive and only sees + // files it can match by prefix + expect(fsMock.unlinkSync).toHaveBeenCalledWith(expect.stringContaining('auto-backup-2020-01-01T02-00-00.zip')); + }); + + it('logs the failure, drops the partial zip and skips retention', async () => { + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('.zip') || String(p).endsWith('backup-settings.json')); + const archiveEvents = stubArchiver(); + archiveMock.finalize.mockImplementation(() => { archiveEvents['error']?.(new Error('disk full')); }); + + await scheduledRun()(); + + expect(logMock.logError).toHaveBeenCalledWith('Auto-Backup: disk full'); + expect(logMock.logInfo).not.toHaveBeenCalledWith(expect.stringContaining('Auto-Backup created')); + expect(fsMock.unlinkSync).toHaveBeenCalledWith(expect.stringContaining('auto-backup-')); + // the snapshot scratch file is cleaned up too, and retention never runs + expect(fsMock.rmSync).toHaveBeenCalledWith(expect.stringContaining('.travel-snap-auto-backup-'), { force: true }); + expect(fsMock.readdirSync).not.toHaveBeenCalled(); + }); +}); diff --git a/server/tests/unit/services/backupService.test.ts b/server/tests/unit/services/backupService.test.ts index ec9ab4836..de02c0c4d 100644 --- a/server/tests/unit/services/backupService.test.ts +++ b/server/tests/unit/services/backupService.test.ts @@ -609,6 +609,34 @@ describe('BACKUP-036 createBackup', () => { expect.anything(), ); }); + + it('BACKUP-036i — the auto-backup prefix names both the zip and its scratch snapshots', async () => { + // The scheduler passes 'auto-backup' so retention and the admin panel can + // still tell scheduled archives apart by filename. + fsMock.existsSync.mockImplementation((p: string) => String(p).endsWith('travel.db')); + fsMock.mkdirSync.mockReturnValue(undefined); + + const writableEvents: Record = {}; + fsMock.createWriteStream.mockReturnValue({ + on: vi.fn((event: string, cb: Function) => { writableEvents[event] = cb; }), + } as never); + archiverInstanceMock.on.mockImplementation((_e: string, _cb: Function) => {}); + archiverInstanceMock.pipe.mockReturnValue(undefined); + archiverInstanceMock.finalize.mockImplementation(() => { + if (writableEvents['close']) writableEvents['close'](); + }); + archiverMock.mockReturnValue(archiverInstanceMock); + + fsMock.statSync.mockReturnValue({ size: 1024, birthtime: new Date('2026-04-06T12:00:00Z') }); + + const result = await createBackup('auto-backup'); + + expect(result.filename).toMatch(/^auto-backup-.*\.zip$/); + expect(archiverInstanceMock.file).toHaveBeenCalledWith( + expect.stringContaining('.travel-snap-auto-backup-'), + { name: 'travel.db' }, + ); + }); }); // --------------------------------------------------------------------------- diff --git a/wiki/Backups.md b/wiki/Backups.md index 76beb7c01..71c368f06 100644 --- a/wiki/Backups.md +++ b/wiki/Backups.md @@ -19,7 +19,7 @@ A backup is a ZIP archive with these entries: | `plugins-data/` | Each installed plugin's own database + files (present only if plugins are installed) | | `plugins-code/` | The installed plugin code, so a restore is self-contained (dev-linked plugins are skipped) | -**Not included:** the encryption key. Store your `ENCRYPTION_KEY` separately from the backup ZIP — for example, in a password manager. See [Encryption-Key-Rotation](Encryption-Key-Rotation). +**Also included:** the at-rest encryption key, unless you supply it through the `ENCRYPTION_KEY` environment variable — in that case the file is not the source of truth and is left out. Bundling it is what makes a backup restorable onto a different install, and it makes the ZIP as sensitive as the key itself: store and transfer it accordingly. See [Encryption-Key-Rotation](Encryption-Key-Rotation). ## Manual backup @@ -64,7 +64,7 @@ Enable scheduled backups in the **Auto-Backup** section of the Backup tab. Auto-backup files are named `auto-backup-.zip` (manual backups use `backup-.zip`). -After each auto-backup run, **all** backup files (manual and auto) older than `keep_days` are pruned. Set `keep_days` to `0` to disable pruning entirely. +After each auto-backup run, **auto-backup files** older than `keep_days` are pruned. Manual backups are never pruned — delete those yourself when you no longer need them. Set `keep_days` to `0` to disable pruning entirely. ## Before updating TREK