Files
TREK/server/src/nest/plugins/host/plugin-data.service.ts
T
Maurice 2b14fe1a2a fix(plugins): bind a plugin's own SQLite arguments exactly again, NUL included
5ff5e7660 stripped U+0000 from every argument a plugin bound to its own
SQLite file, but left the rows it had stored alone. A key written with a
NUL (through a bound arg or char(0) in the plugin's SQL) could no longer be
found by the same key, upserts added duplicates, and keys that differ only
in where a NUL sits ('a\0bc', 'ab\0c') collapsed into one and collided.

A plugin's data is the plugin's, so its file binds the arguments exactly as
sent again, as before that commit. TREK's own data still holds no NUL; the
Postgres store, whose text cannot hold the character, refuses it instead.
2026-11-17 22:28:26 +01:00

383 lines
18 KiB
TypeScript

import { openDatabase } from '../../../db/connection';
import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data';
import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths';
import { splitAfterSemicolons } from './sql-script';
import type { Database as Db, Statement } from 'better-sqlite3';
import fs from 'node:fs';
import path from 'node:path';
// The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs).
export type { PluginDataStore };
/**
* A plugin's own sqlite database (#plugins, db:own): the SQLite side of the
* `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a
* schema per plugin, `db/engine/postgres/plugin-data-store.ts`). The HOST owns the handle;
* the plugin child never gets a path or a connection — it can only reach this
* through RPC (db.exec / db.query / db.migrate). Because it is a SEPARATE FILE,
* containment is a filesystem fact: the plugin physically cannot read trek.db,
* and we don't have to police table-name prefixes in its SQL.
*
* A thin guard still rejects statements that would let a plugin escape its file
* (ATTACH another db, VACUUM INTO elsewhere, PRAGMA fiddling) or DoS via
* oversize SQL, every read is held to a row cap, and query(), tx() and an exec()
* script are held to a time budget (below).
*
* Bound arguments reach SQLite exactly as the plugin sent them, U+0000 (NUL)
* included: the data is the plugin's own, so a key it stored with a NUL is
* found again by the same key, and `'a\0bc'` and `'ab\0c'` stay two keys. TREK
* drops NUL only from its own data (`db/nul-free.ts`); the Postgres store,
* whose text cannot hold the character, refuses such an argument instead.
*/
const MAX_SQL_LENGTH = PLUGIN_DATA_CAPS.maxSqlLength;
// RECURSIVE is the one construct that generates unbounded rows/CPU independent of
// the (capped) data size — a `WITH RECURSIVE …` can spin the synchronous host
// forever even with an empty database, which neither the size quota nor the
// result-row cap can stop (an aggregate over it never yields a first row). Refuse
// it outright; the row/size caps below bound everything else.
// load_extension is included as defense-in-depth: better-sqlite3 disables
// extension loading by default (so it's inert today), but banning it in the guard
// means a future connection-option slip can't turn it into an arbitrary-.so RCE.
const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i;
// Transaction-control keywords, matched only at statement start (so CASE…END and
// identifiers are unaffected). Refused inside db.tx() so a plugin can't COMMIT the
// batch's earlier writes and then have the wrapper report failure — breaking atomicity.
const TX_CONTROL = /^\s*(BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|END)\b/i;
// Per-plugin on-disk quota. better-sqlite3 is synchronous and runs in the HOST
// process, so an unbounded plugin DB is both a disk-exhaustion DoS on the shared
// trek.db volume and (via a huge scan) an event-loop stall. max_page_count caps
// the file (writes past it fail SQLITE_FULL, contained to the plugin) and bounds
// the worst-case scan cost. Result sets are additionally row-capped below so a
// recursive CTE / cartesian product can't materialize an unbounded array.
const QUOTA_BYTES = PLUGIN_DATA_CAPS.quotaBytes;
const MAX_ROWS = PLUGIN_DATA_CAPS.maxRows;
// Cap statements per atomic batch so a single tx() can't monopolise the synchronous
// host — generous for real write batches, far below anything abusive.
const MAX_TX_OPS = PLUGIN_DATA_CAPS.maxTxOps;
// Wall-clock budget for one query(), one whole tx() batch or one exec() script.
// better-sqlite3 has no interrupt or progress handler, so the budget is checked
// between the rows a statement yields and between the statements of a batch or a
// script: a scan that keeps producing rows, a select whose every row is expensive, or
// a script of many statements stops at the first row or statement past the budget
// instead of holding the host loop. One statement on its own still runs to the end
// (a write such as `INSERT … SELECT` over a cartesian product) or to its first row
// (an aggregate over one); only moving plugin SQL off the host thread can bound that,
// and the plugin-sdk README says so. migrate() is left unbudgeted on purpose: a
// migration is stopped part of the way only to fail the same way on every later
// start, which leaves a plugin that cannot update its schema at all.
const TIME_BUDGET_MS = PLUGIN_DATA_CAPS.timeBudgetMs;
// What better-sqlite3 throws from prepare() for a statement cut off before its end:
// in an exec() script, a trigger body whose semicolon split it.
const INCOMPLETE_INPUT = 'incomplete input';
// And for a piece that holds no statement at all, only whitespace or comments.
const NO_STATEMENT = 'The supplied SQL string contains no statements';
/** The caps a PluginDataDb enforces. Overridable for tests; production uses the defaults. */
export interface PluginDataLimits {
maxRows: number;
timeBudgetMs: number;
/** The clock the time budget reads. */
now: () => number;
}
const DEFAULT_LIMITS: PluginDataLimits = { maxRows: MAX_ROWS, timeBudgetMs: TIME_BUDGET_MS, now: () => Date.now() };
// Every live per-plugin handle, so a backup can WAL-checkpoint them before archiving
// (the host keeps these open, so their .db files would otherwise be copied with recent
// commits still stranded in the -wal sidecar → a stale/torn snapshot in the backup).
const openDbs = new Set<PluginDataDb>();
/** Fold the WAL back into each open plugin.db so a subsequent file copy is a complete,
* consistent snapshot — mirrors the wal_checkpoint the core backup runs on travel.db.
* Best-effort per handle; never throws. */
export function checkpointAllPluginDataDbs(): void {
for (const d of openDbs) {
try {
d.checkpoint();
} catch {
/* a busy/closed handle is skipped, not fatal */
}
}
}
export class PluginDataDb implements PluginDataStore {
private db: Db;
readonly pluginId: string;
readonly engine = 'sqlite' as const;
private readonly limits: PluginDataLimits;
constructor(pluginId: string, limits: Partial<PluginDataLimits> = {}) {
this.pluginId = pluginId;
this.limits = { ...DEFAULT_LIMITS, ...limits };
fs.mkdirSync(pluginDataDir(pluginId), { recursive: true });
this.db = openDatabase(pluginDbFile(pluginId));
this.db.pragma('journal_mode = WAL');
this.db.pragma('foreign_keys = ON');
openDbs.add(this);
// Cap the file size (per-connection; not persisted, so set on every open).
const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096;
this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(QUOTA_BYTES / pageSize))}`);
// Track applied migrations so db.migrate is idempotent per (plugin, id).
this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`);
}
private guard(sql: string): void {
if (typeof sql !== 'string') throw new Error('sql must be a string');
if (sql.length > MAX_SQL_LENGTH) throw new Error('sql too long');
if (FORBIDDEN.test(sql)) throw new Error('statement type not allowed for plugin databases');
}
/** A deadline for one call: throws once the call has run past the time budget. */
private startBudget(what: string): () => void {
const { now, timeBudgetMs } = this.limits;
const startedAt = now();
return () => {
if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`);
};
}
/** Read query: returns rows up to the row cap, within the time budget. Single statement only. */
query(sql: string, args: unknown[] = []): unknown[] {
this.guard(sql);
const { maxRows } = this.limits;
const checkTime = this.startBudget('query');
// iterate() pulls one row at a time, so a recursive CTE that would yield
// unboundedly is halted at the cap instead of materializing via all(), and the
// clock is read between rows. It is read only once another row has come back, so
// a result that has been read to its end is returned, however long that took.
const rows: unknown[] = [];
for (const row of this.db.prepare(sql).iterate(...(args as never[]))) {
if (rows.length > 0) checkTime();
rows.push(row);
if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`);
}
return rows;
}
/**
* Write statement(s). With bound args, exec() runs one statement. Without, it runs a
* script of any number (e.g. a small setup script), one statement at a time and
* within the time budget, checked before each statement after the first.
*/
exec(sql: string, args: unknown[] = []): { changes: number } {
this.guard(sql);
if (args.length > 0) {
const info = this.db.prepare(sql).run(...(args as never[]));
return { changes: info.changes };
}
this.runScript(sql, this.startBudget('exec'));
return { changes: 0 };
}
/**
* Runs a script the way `Database#exec` did (statement after statement, each in
* autocommit unless the script opens a transaction), with `checkTime` read before
* every statement but the first. The clock is never read after the last one, so a
* script whose statements have all run (and committed) is reported as done. A
* trigger body is gathered until SQLite stops calling it incomplete. A script that
* runs out of time inside a transaction it opened itself has that transaction rolled
* back, so the connection is not left holding it.
*/
private runScript(sql: string, checkTime: () => void): void {
const openedNoTransaction = !this.db.inTransaction;
let pending = '';
let ranOne = false;
for (const piece of splitAfterSemicolons(sql)) {
pending += piece;
const stmt = this.prepareComplete(pending);
if (stmt === 'incomplete') continue;
pending = '';
if (stmt === 'empty') continue;
if (ranOne) {
try {
checkTime();
} catch (e) {
if (openedNoTransaction && this.db.inTransaction) this.db.prepare('ROLLBACK').run();
throw e;
}
}
stmt.run();
ranOne = true;
}
// A statement still open at the end of the script: prepare it once more for
// SQLite's own error, as Database#exec reports it.
if (pending !== '') this.db.prepare(pending);
}
/** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */
private prepareComplete(sql: string): Statement | 'incomplete' | 'empty' {
try {
return this.db.prepare(sql);
} catch (e) {
const message = e instanceof Error ? e.message : '';
if (message === INCOMPLETE_INPUT) return 'incomplete';
if (message === NO_STATEMENT) return 'empty';
throw e;
}
}
/**
* Atomic batch on the plugin's OWN db: every op runs in a single transaction, so
* they all commit or all roll back — the primitive a plugin needs for a consistent
* multi-write (e.g. move an item between two tables). Each op is ONE statement;
* a read (SELECT/RETURNING) yields `{ rows }`, a write yields `{ changes }`, and
* reads within the batch see the batch's own earlier writes (read-modify-write).
*/
tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } {
if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }');
if (ops.length === 0) return { results: [] };
if (ops.length > MAX_TX_OPS) throw new Error(`tx allows at most ${MAX_TX_OPS} statements`);
for (const op of ops) {
this.guard(op?.sql);
// Reject transaction-control statements: a raw COMMIT/ROLLBACK inside the batch
// would break atomicity — it commits the earlier writes even though the wrapper
// then reports the tx as failed. Strip any LEADING comments/whitespace first so a
// `/* */COMMIT` or `-- x\nCOMMIT` can't slip past the start-anchored check; these
// keywords are only valid at statement start, so CASE ... END is unaffected.
const head = String(op?.sql ?? '').replace(/^(?:\s|--[^\n]*\n?|\/\*[\s\S]*?\*\/)*/, '');
if (TX_CONTROL.test(head)) throw new Error('transaction-control statements are not allowed inside tx()');
}
const { maxRows } = this.limits;
let batchRows = 0; // one row budget for the WHOLE batch, not per statement
// One time budget for the whole batch too. A throw inside the transaction rolls
// every statement of the batch back, so running out of time is all-or-nothing.
const checkTime = this.startBudget('tx');
const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => {
const results: Array<{ changes?: number; rows?: unknown[] }> = [];
for (const op of batch) {
const stmt = this.db.prepare(op.sql);
const args = (op.args ?? []) as never[];
if (stmt.reader) {
const rows: unknown[] = [];
for (const row of stmt.iterate(...args)) {
rows.push(row);
if (++batchRows > maxRows) throw new Error(`tx returned more than ${maxRows} rows in total`);
checkTime();
}
results.push({ rows });
} else {
results.push({ changes: stmt.run(...args).changes });
}
checkTime();
}
return results;
});
return { results: run(ops) };
}
/** Run a migration once, keyed by id. Re-running with the same id is a no-op. */
migrate(id: string, sql: string): { applied: boolean } {
this.guard(sql);
const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id);
if (seen) return { applied: false };
this.db.transaction(() => {
this.db.exec(sql);
this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now());
})();
return { applied: true };
}
/** Whether the underlying sqlite handle is still open (better-sqlite3 `.open`).
* The host uses this to detect a handle closed by a terminal-failure dispose that
* left the instance cached, so it can recreate it instead of throwing on reuse. */
isOpen(): boolean {
return this.db.open;
}
/** Fold the WAL back into the main db file (checkpoint TRUNCATE) so a file-level copy
* is a complete snapshot. No-op on a closed handle. */
checkpoint(): void {
if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)');
}
/** Write a fully-consistent copy of this DB to `destPath` via VACUUM INTO. Unlike a
* file copy it folds in the WAL and reads a point-in-time snapshot, so the result is
* correct even while the plugin is writing — no torn page, no separate -wal to keep in
* sync. This is a host op on the host's own handle, not plugin SQL, so it bypasses the
* FORBIDDEN guard by design. */
snapshotInto(destPath: string): void {
fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target already exists
this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`);
}
close(): void {
try {
openDbs.delete(this);
this.db.close();
} catch {
/* already closed */
}
}
}
/** Delete a plugin's data directory (uninstall "delete data"). */
export function removePluginData(pluginId: string): void {
fs.rmSync(pluginDataDir(pluginId), { recursive: true, force: true });
}
/**
* Copy every plugin's data dir into `destRoot` as a CONSISTENT snapshot, for a backup to
* archive instead of the live tree. An open plugin.db is captured with VACUUM INTO (safe
* under concurrent writes); a plugin with no live handle is copied as-is (no writer). The
* -wal/-shm sidecars are never copied — the snapshot folds them in, and copying them out
* of step with the .db is exactly what produced torn/corrupt restores when the archiver
* read the live files lazily while a plugin kept writing. Blobs and any other files a
* plugin wrote to its dir are copied verbatim. Best-effort per file; never throws.
*/
export function snapshotAllPluginDataDbs(destRoot: string): void {
const root = pluginsDataRoot();
if (!fs.existsSync(root)) return;
const openById = new Map<string, PluginDataDb>();
for (const d of openDbs) if (d.isOpen()) openById.set(d.pluginId, d);
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const srcDir = path.join(root, entry.name);
const destDir = path.join(destRoot, entry.name);
fs.mkdirSync(destDir, { recursive: true });
const open = openById.get(entry.name);
// Handle the live db up front so we know whether its WAL got folded in. If both
// the VACUUM INTO snapshot and the checkpoint fail, the -wal/-shm are NOT folded,
// so they must be copied alongside the .db — a .db stripped of an un-checkpointed
// WAL loses committed transactions, whereas the .db + its WAL is a recoverable set.
let foldedIn = false;
if (open) {
try {
open.snapshotInto(path.join(destDir, 'plugin.db'));
foldedIn = true;
} catch {
try {
open.checkpoint();
foldedIn = true;
} catch {
/* WAL not folded — keep sidecars */
}
try {
fs.copyFileSync(path.join(srcDir, 'plugin.db'), path.join(destDir, 'plugin.db'));
} catch {
/* unreadable live db — best effort */
}
}
}
for (const f of fs.readdirSync(srcDir, { withFileTypes: true })) {
if (f.name === 'plugin.db' && open) continue; // already snapshotted above
// Skip the .db sidecars only when the live handle's WAL was folded in — VACUUM
// INTO / checkpoint absorbs them, and copying them out of step with a live writer
// is what produced torn restores. For a plugin with NO open handle there is no
// writer, so the -wal/-shm are a consistent set with the .db; copy them too, or an
// unclean shutdown's committed-but-uncheckpointed transactions (still sitting in
// the WAL) would be lost from the backup.
if ((f.name.endsWith('-wal') || f.name.endsWith('-shm')) && foldedIn) continue;
const src = path.join(srcDir, f.name);
const dest = path.join(destDir, f.name);
try {
if (f.isDirectory()) fs.cpSync(src, dest, { recursive: true });
else fs.copyFileSync(src, dest);
} catch {
/* skip an unreadable entry rather than fail the whole backup */
}
}
}
}