mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
2b14fe1a2a
5ff5e7660 stripped U+0000 from every argument a plugin bound to its own
SQLite file, but left the rows it had stored alone. A key written with a
NUL (through a bound arg or char(0) in the plugin's SQL) could no longer be
found by the same key, upserts added duplicates, and keys that differ only
in where a NUL sits ('a\0bc', 'ab\0c') collapsed into one and collided.
A plugin's data is the plugin's, so its file binds the arguments exactly as
sent again, as before that commit. TREK's own data still holds no NUL; the
Postgres store, whose text cannot hold the character, refuses it instead.
383 lines
18 KiB
TypeScript
383 lines
18 KiB
TypeScript
import { openDatabase } from '../../../db/connection';
|
|
import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data';
|
|
import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths';
|
|
import { splitAfterSemicolons } from './sql-script';
|
|
|
|
import type { Database as Db, Statement } from 'better-sqlite3';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
|
|
// The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs).
|
|
export type { PluginDataStore };
|
|
|
|
/**
|
|
* A plugin's own sqlite database (#plugins, db:own): the SQLite side of the
|
|
* `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a
|
|
* schema per plugin, `db/engine/postgres/plugin-data-store.ts`). The HOST owns the handle;
|
|
* the plugin child never gets a path or a connection — it can only reach this
|
|
* through RPC (db.exec / db.query / db.migrate). Because it is a SEPARATE FILE,
|
|
* containment is a filesystem fact: the plugin physically cannot read trek.db,
|
|
* and we don't have to police table-name prefixes in its SQL.
|
|
*
|
|
* A thin guard still rejects statements that would let a plugin escape its file
|
|
* (ATTACH another db, VACUUM INTO elsewhere, PRAGMA fiddling) or DoS via
|
|
* oversize SQL, every read is held to a row cap, and query(), tx() and an exec()
|
|
* script are held to a time budget (below).
|
|
*
|
|
* Bound arguments reach SQLite exactly as the plugin sent them, U+0000 (NUL)
|
|
* included: the data is the plugin's own, so a key it stored with a NUL is
|
|
* found again by the same key, and `'a\0bc'` and `'ab\0c'` stay two keys. TREK
|
|
* drops NUL only from its own data (`db/nul-free.ts`); the Postgres store,
|
|
* whose text cannot hold the character, refuses such an argument instead.
|
|
*/
|
|
|
|
const MAX_SQL_LENGTH = PLUGIN_DATA_CAPS.maxSqlLength;
|
|
// RECURSIVE is the one construct that generates unbounded rows/CPU independent of
|
|
// the (capped) data size — a `WITH RECURSIVE …` can spin the synchronous host
|
|
// forever even with an empty database, which neither the size quota nor the
|
|
// result-row cap can stop (an aggregate over it never yields a first row). Refuse
|
|
// it outright; the row/size caps below bound everything else.
|
|
// load_extension is included as defense-in-depth: better-sqlite3 disables
|
|
// extension loading by default (so it's inert today), but banning it in the guard
|
|
// means a future connection-option slip can't turn it into an arbitrary-.so RCE.
|
|
const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i;
|
|
// Transaction-control keywords, matched only at statement start (so CASE…END and
|
|
// identifiers are unaffected). Refused inside db.tx() so a plugin can't COMMIT the
|
|
// batch's earlier writes and then have the wrapper report failure — breaking atomicity.
|
|
const TX_CONTROL = /^\s*(BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|END)\b/i;
|
|
// Per-plugin on-disk quota. better-sqlite3 is synchronous and runs in the HOST
|
|
// process, so an unbounded plugin DB is both a disk-exhaustion DoS on the shared
|
|
// trek.db volume and (via a huge scan) an event-loop stall. max_page_count caps
|
|
// the file (writes past it fail SQLITE_FULL, contained to the plugin) and bounds
|
|
// the worst-case scan cost. Result sets are additionally row-capped below so a
|
|
// recursive CTE / cartesian product can't materialize an unbounded array.
|
|
const QUOTA_BYTES = PLUGIN_DATA_CAPS.quotaBytes;
|
|
const MAX_ROWS = PLUGIN_DATA_CAPS.maxRows;
|
|
// Cap statements per atomic batch so a single tx() can't monopolise the synchronous
|
|
// host — generous for real write batches, far below anything abusive.
|
|
const MAX_TX_OPS = PLUGIN_DATA_CAPS.maxTxOps;
|
|
// Wall-clock budget for one query(), one whole tx() batch or one exec() script.
|
|
// better-sqlite3 has no interrupt or progress handler, so the budget is checked
|
|
// between the rows a statement yields and between the statements of a batch or a
|
|
// script: a scan that keeps producing rows, a select whose every row is expensive, or
|
|
// a script of many statements stops at the first row or statement past the budget
|
|
// instead of holding the host loop. One statement on its own still runs to the end
|
|
// (a write such as `INSERT … SELECT` over a cartesian product) or to its first row
|
|
// (an aggregate over one); only moving plugin SQL off the host thread can bound that,
|
|
// and the plugin-sdk README says so. migrate() is left unbudgeted on purpose: a
|
|
// migration is stopped part of the way only to fail the same way on every later
|
|
// start, which leaves a plugin that cannot update its schema at all.
|
|
const TIME_BUDGET_MS = PLUGIN_DATA_CAPS.timeBudgetMs;
|
|
// What better-sqlite3 throws from prepare() for a statement cut off before its end:
|
|
// in an exec() script, a trigger body whose semicolon split it.
|
|
const INCOMPLETE_INPUT = 'incomplete input';
|
|
// And for a piece that holds no statement at all, only whitespace or comments.
|
|
const NO_STATEMENT = 'The supplied SQL string contains no statements';
|
|
|
|
/** The caps a PluginDataDb enforces. Overridable for tests; production uses the defaults. */
|
|
export interface PluginDataLimits {
|
|
maxRows: number;
|
|
timeBudgetMs: number;
|
|
/** The clock the time budget reads. */
|
|
now: () => number;
|
|
}
|
|
|
|
const DEFAULT_LIMITS: PluginDataLimits = { maxRows: MAX_ROWS, timeBudgetMs: TIME_BUDGET_MS, now: () => Date.now() };
|
|
|
|
// Every live per-plugin handle, so a backup can WAL-checkpoint them before archiving
|
|
// (the host keeps these open, so their .db files would otherwise be copied with recent
|
|
// commits still stranded in the -wal sidecar → a stale/torn snapshot in the backup).
|
|
const openDbs = new Set<PluginDataDb>();
|
|
|
|
/** Fold the WAL back into each open plugin.db so a subsequent file copy is a complete,
|
|
* consistent snapshot — mirrors the wal_checkpoint the core backup runs on travel.db.
|
|
* Best-effort per handle; never throws. */
|
|
export function checkpointAllPluginDataDbs(): void {
|
|
for (const d of openDbs) {
|
|
try {
|
|
d.checkpoint();
|
|
} catch {
|
|
/* a busy/closed handle is skipped, not fatal */
|
|
}
|
|
}
|
|
}
|
|
|
|
export class PluginDataDb implements PluginDataStore {
|
|
private db: Db;
|
|
readonly pluginId: string;
|
|
readonly engine = 'sqlite' as const;
|
|
private readonly limits: PluginDataLimits;
|
|
|
|
constructor(pluginId: string, limits: Partial<PluginDataLimits> = {}) {
|
|
this.pluginId = pluginId;
|
|
this.limits = { ...DEFAULT_LIMITS, ...limits };
|
|
fs.mkdirSync(pluginDataDir(pluginId), { recursive: true });
|
|
this.db = openDatabase(pluginDbFile(pluginId));
|
|
this.db.pragma('journal_mode = WAL');
|
|
this.db.pragma('foreign_keys = ON');
|
|
openDbs.add(this);
|
|
// Cap the file size (per-connection; not persisted, so set on every open).
|
|
const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096;
|
|
this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(QUOTA_BYTES / pageSize))}`);
|
|
// Track applied migrations so db.migrate is idempotent per (plugin, id).
|
|
this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`);
|
|
}
|
|
|
|
private guard(sql: string): void {
|
|
if (typeof sql !== 'string') throw new Error('sql must be a string');
|
|
if (sql.length > MAX_SQL_LENGTH) throw new Error('sql too long');
|
|
if (FORBIDDEN.test(sql)) throw new Error('statement type not allowed for plugin databases');
|
|
}
|
|
|
|
/** A deadline for one call: throws once the call has run past the time budget. */
|
|
private startBudget(what: string): () => void {
|
|
const { now, timeBudgetMs } = this.limits;
|
|
const startedAt = now();
|
|
return () => {
|
|
if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`);
|
|
};
|
|
}
|
|
|
|
/** Read query: returns rows up to the row cap, within the time budget. Single statement only. */
|
|
query(sql: string, args: unknown[] = []): unknown[] {
|
|
this.guard(sql);
|
|
const { maxRows } = this.limits;
|
|
const checkTime = this.startBudget('query');
|
|
// iterate() pulls one row at a time, so a recursive CTE that would yield
|
|
// unboundedly is halted at the cap instead of materializing via all(), and the
|
|
// clock is read between rows. It is read only once another row has come back, so
|
|
// a result that has been read to its end is returned, however long that took.
|
|
const rows: unknown[] = [];
|
|
for (const row of this.db.prepare(sql).iterate(...(args as never[]))) {
|
|
if (rows.length > 0) checkTime();
|
|
rows.push(row);
|
|
if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`);
|
|
}
|
|
return rows;
|
|
}
|
|
|
|
/**
|
|
* Write statement(s). With bound args, exec() runs one statement. Without, it runs a
|
|
* script of any number (e.g. a small setup script), one statement at a time and
|
|
* within the time budget, checked before each statement after the first.
|
|
*/
|
|
exec(sql: string, args: unknown[] = []): { changes: number } {
|
|
this.guard(sql);
|
|
if (args.length > 0) {
|
|
const info = this.db.prepare(sql).run(...(args as never[]));
|
|
return { changes: info.changes };
|
|
}
|
|
this.runScript(sql, this.startBudget('exec'));
|
|
return { changes: 0 };
|
|
}
|
|
|
|
/**
|
|
* Runs a script the way `Database#exec` did (statement after statement, each in
|
|
* autocommit unless the script opens a transaction), with `checkTime` read before
|
|
* every statement but the first. The clock is never read after the last one, so a
|
|
* script whose statements have all run (and committed) is reported as done. A
|
|
* trigger body is gathered until SQLite stops calling it incomplete. A script that
|
|
* runs out of time inside a transaction it opened itself has that transaction rolled
|
|
* back, so the connection is not left holding it.
|
|
*/
|
|
private runScript(sql: string, checkTime: () => void): void {
|
|
const openedNoTransaction = !this.db.inTransaction;
|
|
let pending = '';
|
|
let ranOne = false;
|
|
for (const piece of splitAfterSemicolons(sql)) {
|
|
pending += piece;
|
|
const stmt = this.prepareComplete(pending);
|
|
if (stmt === 'incomplete') continue;
|
|
pending = '';
|
|
if (stmt === 'empty') continue;
|
|
if (ranOne) {
|
|
try {
|
|
checkTime();
|
|
} catch (e) {
|
|
if (openedNoTransaction && this.db.inTransaction) this.db.prepare('ROLLBACK').run();
|
|
throw e;
|
|
}
|
|
}
|
|
stmt.run();
|
|
ranOne = true;
|
|
}
|
|
// A statement still open at the end of the script: prepare it once more for
|
|
// SQLite's own error, as Database#exec reports it.
|
|
if (pending !== '') this.db.prepare(pending);
|
|
}
|
|
|
|
/** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */
|
|
private prepareComplete(sql: string): Statement | 'incomplete' | 'empty' {
|
|
try {
|
|
return this.db.prepare(sql);
|
|
} catch (e) {
|
|
const message = e instanceof Error ? e.message : '';
|
|
if (message === INCOMPLETE_INPUT) return 'incomplete';
|
|
if (message === NO_STATEMENT) return 'empty';
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Atomic batch on the plugin's OWN db: every op runs in a single transaction, so
|
|
* they all commit or all roll back — the primitive a plugin needs for a consistent
|
|
* multi-write (e.g. move an item between two tables). Each op is ONE statement;
|
|
* a read (SELECT/RETURNING) yields `{ rows }`, a write yields `{ changes }`, and
|
|
* reads within the batch see the batch's own earlier writes (read-modify-write).
|
|
*/
|
|
tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } {
|
|
if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }');
|
|
if (ops.length === 0) return { results: [] };
|
|
if (ops.length > MAX_TX_OPS) throw new Error(`tx allows at most ${MAX_TX_OPS} statements`);
|
|
for (const op of ops) {
|
|
this.guard(op?.sql);
|
|
// Reject transaction-control statements: a raw COMMIT/ROLLBACK inside the batch
|
|
// would break atomicity — it commits the earlier writes even though the wrapper
|
|
// then reports the tx as failed. Strip any LEADING comments/whitespace first so a
|
|
// `/* */COMMIT` or `-- x\nCOMMIT` can't slip past the start-anchored check; these
|
|
// keywords are only valid at statement start, so CASE ... END is unaffected.
|
|
const head = String(op?.sql ?? '').replace(/^(?:\s|--[^\n]*\n?|\/\*[\s\S]*?\*\/)*/, '');
|
|
if (TX_CONTROL.test(head)) throw new Error('transaction-control statements are not allowed inside tx()');
|
|
}
|
|
const { maxRows } = this.limits;
|
|
let batchRows = 0; // one row budget for the WHOLE batch, not per statement
|
|
// One time budget for the whole batch too. A throw inside the transaction rolls
|
|
// every statement of the batch back, so running out of time is all-or-nothing.
|
|
const checkTime = this.startBudget('tx');
|
|
const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => {
|
|
const results: Array<{ changes?: number; rows?: unknown[] }> = [];
|
|
for (const op of batch) {
|
|
const stmt = this.db.prepare(op.sql);
|
|
const args = (op.args ?? []) as never[];
|
|
if (stmt.reader) {
|
|
const rows: unknown[] = [];
|
|
for (const row of stmt.iterate(...args)) {
|
|
rows.push(row);
|
|
if (++batchRows > maxRows) throw new Error(`tx returned more than ${maxRows} rows in total`);
|
|
checkTime();
|
|
}
|
|
results.push({ rows });
|
|
} else {
|
|
results.push({ changes: stmt.run(...args).changes });
|
|
}
|
|
checkTime();
|
|
}
|
|
return results;
|
|
});
|
|
return { results: run(ops) };
|
|
}
|
|
|
|
/** Run a migration once, keyed by id. Re-running with the same id is a no-op. */
|
|
migrate(id: string, sql: string): { applied: boolean } {
|
|
this.guard(sql);
|
|
const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id);
|
|
if (seen) return { applied: false };
|
|
this.db.transaction(() => {
|
|
this.db.exec(sql);
|
|
this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now());
|
|
})();
|
|
return { applied: true };
|
|
}
|
|
|
|
/** Whether the underlying sqlite handle is still open (better-sqlite3 `.open`).
|
|
* The host uses this to detect a handle closed by a terminal-failure dispose that
|
|
* left the instance cached, so it can recreate it instead of throwing on reuse. */
|
|
isOpen(): boolean {
|
|
return this.db.open;
|
|
}
|
|
|
|
/** Fold the WAL back into the main db file (checkpoint TRUNCATE) so a file-level copy
|
|
* is a complete snapshot. No-op on a closed handle. */
|
|
checkpoint(): void {
|
|
if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
}
|
|
|
|
/** Write a fully-consistent copy of this DB to `destPath` via VACUUM INTO. Unlike a
|
|
* file copy it folds in the WAL and reads a point-in-time snapshot, so the result is
|
|
* correct even while the plugin is writing — no torn page, no separate -wal to keep in
|
|
* sync. This is a host op on the host's own handle, not plugin SQL, so it bypasses the
|
|
* FORBIDDEN guard by design. */
|
|
snapshotInto(destPath: string): void {
|
|
fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target already exists
|
|
this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`);
|
|
}
|
|
|
|
close(): void {
|
|
try {
|
|
openDbs.delete(this);
|
|
this.db.close();
|
|
} catch {
|
|
/* already closed */
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Delete a plugin's data directory (uninstall "delete data"). */
|
|
export function removePluginData(pluginId: string): void {
|
|
fs.rmSync(pluginDataDir(pluginId), { recursive: true, force: true });
|
|
}
|
|
|
|
/**
|
|
* Copy every plugin's data dir into `destRoot` as a CONSISTENT snapshot, for a backup to
|
|
* archive instead of the live tree. An open plugin.db is captured with VACUUM INTO (safe
|
|
* under concurrent writes); a plugin with no live handle is copied as-is (no writer). The
|
|
* -wal/-shm sidecars are never copied — the snapshot folds them in, and copying them out
|
|
* of step with the .db is exactly what produced torn/corrupt restores when the archiver
|
|
* read the live files lazily while a plugin kept writing. Blobs and any other files a
|
|
* plugin wrote to its dir are copied verbatim. Best-effort per file; never throws.
|
|
*/
|
|
export function snapshotAllPluginDataDbs(destRoot: string): void {
|
|
const root = pluginsDataRoot();
|
|
if (!fs.existsSync(root)) return;
|
|
const openById = new Map<string, PluginDataDb>();
|
|
for (const d of openDbs) if (d.isOpen()) openById.set(d.pluginId, d);
|
|
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
|
if (!entry.isDirectory()) continue;
|
|
const srcDir = path.join(root, entry.name);
|
|
const destDir = path.join(destRoot, entry.name);
|
|
fs.mkdirSync(destDir, { recursive: true });
|
|
const open = openById.get(entry.name);
|
|
// Handle the live db up front so we know whether its WAL got folded in. If both
|
|
// the VACUUM INTO snapshot and the checkpoint fail, the -wal/-shm are NOT folded,
|
|
// so they must be copied alongside the .db — a .db stripped of an un-checkpointed
|
|
// WAL loses committed transactions, whereas the .db + its WAL is a recoverable set.
|
|
let foldedIn = false;
|
|
if (open) {
|
|
try {
|
|
open.snapshotInto(path.join(destDir, 'plugin.db'));
|
|
foldedIn = true;
|
|
} catch {
|
|
try {
|
|
open.checkpoint();
|
|
foldedIn = true;
|
|
} catch {
|
|
/* WAL not folded — keep sidecars */
|
|
}
|
|
try {
|
|
fs.copyFileSync(path.join(srcDir, 'plugin.db'), path.join(destDir, 'plugin.db'));
|
|
} catch {
|
|
/* unreadable live db — best effort */
|
|
}
|
|
}
|
|
}
|
|
for (const f of fs.readdirSync(srcDir, { withFileTypes: true })) {
|
|
if (f.name === 'plugin.db' && open) continue; // already snapshotted above
|
|
// Skip the .db sidecars only when the live handle's WAL was folded in — VACUUM
|
|
// INTO / checkpoint absorbs them, and copying them out of step with a live writer
|
|
// is what produced torn restores. For a plugin with NO open handle there is no
|
|
// writer, so the -wal/-shm are a consistent set with the .db; copy them too, or an
|
|
// unclean shutdown's committed-but-uncheckpointed transactions (still sitting in
|
|
// the WAL) would be lost from the backup.
|
|
if ((f.name.endsWith('-wal') || f.name.endsWith('-shm')) && foldedIn) continue;
|
|
const src = path.join(srcDir, f.name);
|
|
const dest = path.join(destDir, f.name);
|
|
try {
|
|
if (f.isDirectory()) fs.cpSync(src, dest, { recursive: true });
|
|
else fs.copyFileSync(src, dest);
|
|
} catch {
|
|
/* skip an unreadable entry rather than fail the whole backup */
|
|
}
|
|
}
|
|
}
|
|
}
|