mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fix(plugins): hold plugin data on Postgres to its own schema and to one contract on both engines
The plugin SQL guard now refuses what still reached outside a plugin's schema or depended on the server: - reads of the system columns (xmin, xmax, cmin, cmax, ctid, tableoid), which show transaction ids and storage layout and do not exist on SQLite; a value of the plugin's own may still be named xmin with AS - the xid and xid8 types wherever the word stands for the type, since age() of an xid is the cluster's transaction counter - the text search functions that coerce a text argument to regconfig, a catalog lookup by name, which leave the allowlist - collations and access methods outside the set every server has, which could tell whether a locale or an extension exists Every plugin transaction pins, with SET LOCAL, the settings that decide how values print and parse (TimeZone UTC, DateStyle, the C locales, float, bytea and xml output, the text search configuration), so a plugin's output neither depends on nor reveals the server's configuration. Both stores now keep one contract. Its texts and caps live once, in plugin-data-contract.ts, and both stores throw only those: NUL in SQL, an argument or a migration id is refused on SQLite as on Postgres, transaction control is refused in every call on both engines (tx() is how a batch becomes atomic), a closed store refuses every call, and the Postgres store answers the cases better-sqlite3 names with its words. The SQLite store moves beside the Postgres one as SqlitePluginDataStore in db/engine/sqlite, takes its file from the backend so src/db never imports src/nest, and no longer needs the driver exemption. A contract test runs every violation on both stores and fails when they answer differently. The Postgres lane holds the allowlist, the pinned settings and the SQL plugins commonly write against a real database.
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
/**
|
||||
* The contract of a plugin's own database (`db:own`) that holds on every
|
||||
* engine: the caps, the words a call that breaks it is refused with, and the
|
||||
* refusals of U+0000 (NUL) and of transaction control, so a plugin answers the
|
||||
* same whichever engine TREK runs on.
|
||||
*
|
||||
* TREK's two stores (`server/src/db/engine/sqlite/plugin-data-store.ts` and
|
||||
* `server/src/db/engine/postgres/plugin-data-store.ts`) throw these texts;
|
||||
* `server/tests/unit/db/engine/plugin-data-contract.test.ts` runs every case
|
||||
* on both and fails on a store that answers one differently.
|
||||
*
|
||||
* The plugin SDK keeps a byte-identical copy (`plugin-sdk/src/plugin-data-contract.ts`)
|
||||
* so `trek-plugin dev` and the mock host refuse what TREK refuses, with the
|
||||
* same words; its `test/permissions-parity.test.ts` compares the two files.
|
||||
* Change both together. No imports: the SDK ships standalone.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The limits every plugin database is held to, whatever the engine. The
|
||||
* plugin-sdk README's "Runtime limits" table documents them.
|
||||
*/
|
||||
export const PLUGIN_DATA_CAPS = {
|
||||
/** The longest SQL text one call may carry. */
|
||||
maxSqlLength: 100_000,
|
||||
/** The most rows a query, or a whole tx batch, returns. */
|
||||
maxRows: 100_000,
|
||||
/** The most statements one tx batch holds. */
|
||||
maxTxOps: 100,
|
||||
/** Wall-clock time for one query, one tx batch or one exec script. */
|
||||
timeBudgetMs: 2_000,
|
||||
/** The most a plugin's data may take on disk. */
|
||||
quotaBytes: 256 * 1024 * 1024,
|
||||
} as const;
|
||||
|
||||
/** The calls a time budget holds. */
|
||||
export type PluginDataBudgetedCall = 'query' | 'exec' | 'tx';
|
||||
|
||||
/** The start of every refusal of a statement a plugin may not run. */
|
||||
const STATEMENT_NOT_ALLOWED = 'statement type not allowed for plugin databases';
|
||||
|
||||
/**
|
||||
* What a plugin is told when a call breaks the contract. Both stores throw
|
||||
* these and nothing else for these cases.
|
||||
*
|
||||
* `noStatement`, `moreThanOneStatement` and `closed` are better-sqlite3's own
|
||||
* messages: the SQLite store passes them on as the driver throws them, and
|
||||
* the Postgres store says the same. `statementNotAllowed` starts every
|
||||
* refusal of a statement the store does not run for a plugin; the Postgres
|
||||
* guard adds the reason after a colon. `txControl` is such a refusal with its
|
||||
* reason, in the same words on both engines ({@link refuseTransactionControl}).
|
||||
* `parameterBeforeParenthesis` is the refusal of a spelling no engine TREK
|
||||
* runs accepts, which the scanner behind `refuseTransactionControl` cannot
|
||||
* read the way every SQLite build does.
|
||||
*/
|
||||
export const PLUGIN_DATA_ERRORS = {
|
||||
sqlNotString: 'sql must be a string',
|
||||
sqlTooLong: 'sql too long',
|
||||
statementNotAllowed: STATEMENT_NOT_ALLOWED,
|
||||
txControl:
|
||||
`${STATEMENT_NOT_ALLOWED}: transaction-control statements are not allowed: ` +
|
||||
'each call runs in its own transaction, and tx() makes a batch atomic',
|
||||
parameterBeforeParenthesis:
|
||||
`${STATEMENT_NOT_ALLOWED}: a parameter name ($name, :name, @name) directly followed by "(" is not allowed: ` +
|
||||
'no engine TREK runs accepts it, and the SQLite of trek-plugin dev would read the parentheses as part of the name',
|
||||
noStatement: 'The supplied SQL string contains no statements',
|
||||
moreThanOneStatement: 'The supplied SQL string contains more than one statement',
|
||||
closed: 'The database connection is not open',
|
||||
txNotArray: 'tx requires an array of { sql, args }',
|
||||
txTooManyOps: (max: number): string => `tx allows at most ${max} statements`,
|
||||
queryTooManyRows: (max: number): string => `query returned more than ${max} rows`,
|
||||
txTooManyRows: (max: number): string => `tx returned more than ${max} rows in total`,
|
||||
overBudget: (call: PluginDataBudgetedCall, ms: number): string => `${call} exceeded its ${ms} ms time budget`,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Why a plugin's SQL holding NUL is refused: Postgres ends a statement's text
|
||||
* at a NUL and its text cannot hold one. On SQLite a NUL in a string literal
|
||||
* would store a value the same plugin could never store on Postgres.
|
||||
*/
|
||||
export const PLUGIN_SQL_NUL_REFUSAL =
|
||||
'the SQL holds the character U+0000 (NUL), which plugin databases refuse on every engine: ' +
|
||||
'Postgres cannot read it in a statement or store it in text. Remove it from the SQL';
|
||||
|
||||
/**
|
||||
* Why a plugin's bound value or migration id holding NUL is refused: TREK
|
||||
* will not change what a plugin asked to store or compare (`'a\0b'` and
|
||||
* `'ab'` are different keys). SQLite could store it, but then the plugin
|
||||
* would answer differently on Postgres and its data could not move there.
|
||||
*/
|
||||
export const PLUGIN_VALUE_NUL_REFUSAL =
|
||||
'a plugin argument holds the character U+0000 (NUL), which plugin databases refuse on every engine: ' +
|
||||
'Postgres text cannot store it, and TREK refuses it instead of changing the value. ' +
|
||||
'Remove it, or bind binary data as a Buffer (a BLOB on SQLite, bytea on Postgres)';
|
||||
|
||||
/** Refuses plugin SQL holding NUL ({@link PLUGIN_SQL_NUL_REFUSAL}). */
|
||||
export function refuseNulSql(sql: string): void {
|
||||
if (sql.includes('\u0000')) throw new Error(PLUGIN_SQL_NUL_REFUSAL);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses plugin values holding NUL ({@link PLUGIN_VALUE_NUL_REFUSAL}): a
|
||||
* string, or a string anywhere in an array or a plain object (SQLite binds an
|
||||
* object's fields as named parameters, `pg` sends one as JSON). A Buffer is
|
||||
* binary data and holds any byte.
|
||||
*/
|
||||
export function refuseNulValues(values: readonly unknown[]): void {
|
||||
if (holdsNul(values, new Set())) throw new Error(PLUGIN_VALUE_NUL_REFUSAL);
|
||||
}
|
||||
|
||||
function holdsNul(value: unknown, seen: Set<object>): boolean {
|
||||
if (typeof value === 'string') return value.includes('\u0000');
|
||||
if (typeof value !== 'object' || value === null || seen.has(value)) return false;
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) return value.some((item) => holdsNul(item, seen));
|
||||
const proto: unknown = Object.getPrototypeOf(value);
|
||||
if (proto !== Object.prototype && proto !== null) return false;
|
||||
return Object.values(value).some((item) => holdsNul(item, seen));
|
||||
}
|
||||
|
||||
/**
|
||||
* The verbs of transaction control, in lower case. TREK runs every call in a
|
||||
* transaction of its own and `tx()` makes a batch atomic, so a plugin never
|
||||
* opens, ends or nests one: a COMMIT in the middle would keep writes the call
|
||||
* then reports as failed. SQLite knows no START or ABORT, but they are refused
|
||||
* there too, so a plugin is told the same on both engines.
|
||||
*/
|
||||
export const PLUGIN_TRANSACTION_CONTROL: ReadonlySet<string> = new Set([
|
||||
'begin',
|
||||
'start',
|
||||
'commit',
|
||||
'end',
|
||||
'rollback',
|
||||
'abort',
|
||||
'savepoint',
|
||||
'release',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Refuses SQL in which any statement starts with transaction control
|
||||
* ({@link PLUGIN_DATA_ERRORS.txControl}), in a query, a tx op, a migration or
|
||||
* any statement of an exec script, before anything runs. The SQLite store and
|
||||
* the SDK harnesses call it; the Postgres guard finds the same verbs with its
|
||||
* own lexer.
|
||||
*
|
||||
* A named parameter directly followed by `(` is refused as well
|
||||
* ({@link PLUGIN_DATA_ERRORS.parameterBeforeParenthesis}). A SQLite built with
|
||||
* Tcl variables, as node:sqlite is under `trek-plugin dev`, reads `$a(...)`
|
||||
* up to the closing parenthesis as one token, quotes and semicolons
|
||||
* included, so `$a(')); COMMIT; --')` would hide a COMMIT from this scan.
|
||||
* better-sqlite3 and Postgres refuse the spelling anyway.
|
||||
*
|
||||
* `dialect` says whose spelling `sql` is. The stores and `trek-plugin dev`
|
||||
* run SQLite and pass nothing. The SDK's mock host emulating Postgres passes
|
||||
* `'postgres'`, so `$1`, dollar quotes (`$$...$$`, `$tag$...$tag$`) and
|
||||
* escape strings (`E'...'`) are read as Postgres reads them, `:`, `@` and `#`
|
||||
* are operators there, and a trigger has no body of statements.
|
||||
*/
|
||||
export function refuseTransactionControl(sql: string, dialect: PluginSqlDialect = 'sqlite'): void {
|
||||
if (statementVerbs(sql, dialect).some((verb) => PLUGIN_TRANSACTION_CONTROL.has(verb))) {
|
||||
throw new Error(PLUGIN_DATA_ERRORS.txControl);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whose spelling a plugin's SQL is in. */
|
||||
export type PluginSqlDialect = 'sqlite' | 'postgres';
|
||||
|
||||
/** A name or keyword as SQLite reads one, unquoted. */
|
||||
const WORD = /[A-Za-z_\u0080-][A-Za-z0-9_$\u0080-]*/y;
|
||||
|
||||
/** A character SQLite reads as part of a name. */
|
||||
const NAME_CHAR = /[A-Za-z0-9_$\u0080-]/;
|
||||
|
||||
/** The characters a named parameter starts with in SQLite. */
|
||||
const PARAMETER_PREFIXES = new Set(['$', ':', '@', '#']);
|
||||
|
||||
/**
|
||||
* How many leading words of a statement are kept: enough for
|
||||
* EXPLAIN QUERY PLAN CREATE TEMP TRIGGER.
|
||||
*/
|
||||
const LEADING_WORDS = 6;
|
||||
|
||||
/**
|
||||
* The first word of every statement in `sql`, in lower case, read the way
|
||||
* SQLite splits a script: quoted text ('...', "...", `...`, [...]) and
|
||||
* comments are skipped, and a semicolon ends a statement, except inside
|
||||
* the body of a CREATE TRIGGER, which only `; END ;` ends (as in SQLite's
|
||||
* sqlite3_complete). The statements of a trigger body are not counted, so
|
||||
* its closing END is not taken for transaction control. A statement that
|
||||
* starts with a quoted name or a parenthesis has no verb here. A named
|
||||
* parameter is read as SQLite reads it ({@link afterParameter}).
|
||||
*/
|
||||
function statementVerbs(sql: string, dialect: PluginSqlDialect): string[] {
|
||||
const verbs: string[] = [];
|
||||
let words: string[] = [];
|
||||
let tokens = 0;
|
||||
let inTrigger = false;
|
||||
// In a trigger body: whether the last tokens were `;`, or `;` and END.
|
||||
let afterSemicolon = false;
|
||||
let afterEnd = false;
|
||||
let i = 0;
|
||||
while (i < sql.length) {
|
||||
const c = sql[i];
|
||||
const next = sql[i + 1];
|
||||
if (/\s/.test(c)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '-' && next === '-') {
|
||||
i = indexAfter(sql, '\n', i + 2);
|
||||
continue;
|
||||
}
|
||||
if (c === '/' && next === '*') {
|
||||
i = indexAfter(sql, '*/', i + 2);
|
||||
continue;
|
||||
}
|
||||
if (c === ';') {
|
||||
if (!inTrigger || afterEnd) {
|
||||
words = [];
|
||||
tokens = 0;
|
||||
inTrigger = false;
|
||||
}
|
||||
afterSemicolon = inTrigger;
|
||||
afterEnd = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
WORD.lastIndex = i;
|
||||
const word = WORD.exec(sql)?.[0];
|
||||
tokens++;
|
||||
if (word === undefined) {
|
||||
i = afterOtherToken(sql, i, dialect);
|
||||
afterSemicolon = false;
|
||||
afterEnd = false;
|
||||
continue;
|
||||
}
|
||||
i += word.length;
|
||||
const lower = word.toLowerCase();
|
||||
if (tokens === 1) verbs.push(lower);
|
||||
if (tokens === words.length + 1 && words.length < LEADING_WORDS) words.push(lower);
|
||||
if (!inTrigger && dialect === 'sqlite') inTrigger = opensTrigger(words);
|
||||
afterEnd = afterSemicolon && lower === 'end';
|
||||
afterSemicolon = false;
|
||||
}
|
||||
return verbs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a statement starting with `words` is
|
||||
* [EXPLAIN [QUERY PLAN]] CREATE [TEMP | TEMPORARY] TRIGGER.
|
||||
*/
|
||||
function opensTrigger(words: readonly string[]): boolean {
|
||||
let at = 0;
|
||||
if (words[0] === 'explain') at = words[1] === 'query' && words[2] === 'plan' ? 3 : 1;
|
||||
if (words[at] !== 'create') return false;
|
||||
at += words[at + 1] === 'temp' || words[at + 1] === 'temporary' ? 2 : 1;
|
||||
return words[at] === 'trigger';
|
||||
}
|
||||
|
||||
/** The index after the token at `start` that is no word: quoted text, a parameter, or one character. */
|
||||
function afterOtherToken(sql: string, start: number, dialect: PluginSqlDialect): number {
|
||||
if (dialect === 'postgres') return afterPostgresToken(sql, start);
|
||||
const c = sql[start];
|
||||
if (c === '[') return indexAfter(sql, ']', start + 1);
|
||||
if (c === "'" || c === '"' || c === '`') return afterQuoted(sql, start);
|
||||
if (PARAMETER_PREFIXES.has(c)) return afterParameter(sql, start);
|
||||
return start + 1;
|
||||
}
|
||||
|
||||
/** A dollar quote's opening tag, `$$` or `$tag$`, as Postgres reads one. */
|
||||
const DOLLAR_TAG = /\$(?:[A-Za-z_\u0080-][A-Za-z0-9_\u0080-]*)?\$/y;
|
||||
|
||||
/**
|
||||
* The index after the token at `start` that is no word, read as Postgres reads
|
||||
* it: quoted text and quoted names, a dollar-quoted string up to its closing
|
||||
* tag, an escape string (`E'...'`, where a backslash escapes the next
|
||||
* character), or one character (`$1` is `$` and a number, `::` two colons).
|
||||
*/
|
||||
function afterPostgresToken(sql: string, start: number): number {
|
||||
const c = sql[start];
|
||||
if (c === '$') {
|
||||
DOLLAR_TAG.lastIndex = start;
|
||||
const tag = DOLLAR_TAG.exec(sql)?.[0];
|
||||
if (tag !== undefined) return indexAfter(sql, tag, start + tag.length);
|
||||
return start + 1;
|
||||
}
|
||||
if (c === "'" && isEscapeStringPrefix(sql, start - 1)) return afterEscapeString(sql, start);
|
||||
if (c === "'" || c === '"') return afterQuoted(sql, start);
|
||||
return start + 1;
|
||||
}
|
||||
|
||||
/** Whether the character at `at` is a lone `E`/`e` that makes the string after it an escape string. */
|
||||
function isEscapeStringPrefix(sql: string, at: number): boolean {
|
||||
return (sql[at] === 'E' || sql[at] === 'e') && (at === 0 || !NAME_CHAR.test(sql[at - 1]));
|
||||
}
|
||||
|
||||
/** The index after the escape string whose quote is at `start`. */
|
||||
function afterEscapeString(sql: string, start: number): number {
|
||||
let i = start + 1;
|
||||
while (i < sql.length) {
|
||||
if (sql[i] === '\\') i += 2;
|
||||
else if (sql[i] !== "'") i++;
|
||||
else if (sql[i + 1] === "'") i += 2;
|
||||
else return i + 1;
|
||||
}
|
||||
return sql.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* The index after the named parameter starting at `start`, read as SQLite's
|
||||
* tokenizer reads one: name characters, with `::` allowed inside. Refuses a
|
||||
* name directly followed by `(` ({@link PLUGIN_DATA_ERRORS.parameterBeforeParenthesis}).
|
||||
*
|
||||
* A prefix without a name is no parameter, and SQLite refuses the statement
|
||||
* holding it. Its colons are skipped together, so the scan does not read a
|
||||
* Postgres cast (`x::numeric(10)`, which the mock host may be handed) as a
|
||||
* parameter.
|
||||
*/
|
||||
function afterParameter(sql: string, start: number): number {
|
||||
let i = start + 1;
|
||||
let named = false;
|
||||
while (i < sql.length) {
|
||||
if (NAME_CHAR.test(sql[i])) {
|
||||
named = true;
|
||||
i++;
|
||||
} else if (sql[i] === ':' && sql[i + 1] === ':') {
|
||||
i += 2;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!named) {
|
||||
while (sql[i] === ':') i++;
|
||||
return i;
|
||||
}
|
||||
if (sql[i] === '(') throw new Error(PLUGIN_DATA_ERRORS.parameterBeforeParenthesis);
|
||||
return i;
|
||||
}
|
||||
|
||||
/** The index after the first `token` from `from`, or the end of `sql`. */
|
||||
function indexAfter(sql: string, token: string, from: number): number {
|
||||
const at = sql.indexOf(token, from);
|
||||
return at < 0 ? sql.length : at + token.length;
|
||||
}
|
||||
|
||||
/** The index after the quoted text starting at `start`; a doubled quote stands for itself. */
|
||||
function afterQuoted(sql: string, start: number): number {
|
||||
const quote = sql[start];
|
||||
let i = start + 1;
|
||||
while (i < sql.length) {
|
||||
if (sql[i] !== quote) i++;
|
||||
else if (sql[i + 1] === quote) i += 2;
|
||||
else return i + 1;
|
||||
}
|
||||
return sql.length;
|
||||
}
|
||||
@@ -2,11 +2,14 @@ import type { EngineId } from '../../app-config/db-settings';
|
||||
|
||||
/**
|
||||
* A plugin's own database (`db:own`), as the host's RPC handlers reach it.
|
||||
* One implementation per engine: on SQLite a file of the plugin's own
|
||||
* (`nest/plugins/host/plugin-data.service.ts`), on Postgres a schema of the
|
||||
* plugin's own (`postgres/plugin-data-store.ts`). The plugin's SQL is its
|
||||
* own, written for the engine `engine` names; neither implementation
|
||||
* translates it.
|
||||
* One implementation per engine, each in its engine's folder: on SQLite a
|
||||
* file of the plugin's own (`SqlitePluginDataStore`,
|
||||
* `sqlite/plugin-data-store.ts`), on Postgres a schema of the plugin's own
|
||||
* (`PostgresPluginDataStore`, `postgres/plugin-data-store.ts`). The plugin's
|
||||
* SQL is its own, written for the engine `engine` names; neither
|
||||
* implementation translates it. Both hold a plugin to the caps and refuse a
|
||||
* call that breaks the contract with the same words, from
|
||||
* `plugin-data-contract.ts`.
|
||||
*
|
||||
* The SQLite handle answers synchronously, the Postgres one with a promise,
|
||||
* so every method may return either. The caller awaits the result.
|
||||
@@ -40,20 +43,3 @@ export interface PluginTxOp {
|
||||
export interface PluginTxResult {
|
||||
results: Array<{ changes?: number; rows?: unknown[] }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The limits every plugin database is held to, whatever the engine. The
|
||||
* plugin-sdk README's "Runtime limits" table documents them.
|
||||
*/
|
||||
export const PLUGIN_DATA_CAPS = {
|
||||
/** The longest SQL text one call may carry. */
|
||||
maxSqlLength: 100_000,
|
||||
/** The most rows a query, or a whole tx batch, returns. */
|
||||
maxRows: 100_000,
|
||||
/** The most statements one tx batch holds. */
|
||||
maxTxOps: 100,
|
||||
/** Wall-clock time for one query, one tx batch or one exec script. */
|
||||
timeBudgetMs: 2_000,
|
||||
/** The most a plugin's data may take on disk. */
|
||||
quotaBytes: 256 * 1024 * 1024,
|
||||
} as const;
|
||||
|
||||
@@ -138,24 +138,3 @@ export async function schemaUsage(client: PluginSqlClient, schema: string): Prom
|
||||
const [row] = result.rows as Array<{ bytes?: number | string; objects?: number | string }>;
|
||||
return { bytes: Number(row?.bytes ?? 0), objects: Number(row?.objects ?? 0) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a plugin's bound value that holds U+0000 (NUL): Postgres text
|
||||
* cannot store the character, and TREK will not change what a plugin asked
|
||||
* to store or compare (`'a\0b'` and `'ab'` are different keys on SQLite).
|
||||
* A string, or a string anywhere in an array, is checked; a Buffer goes to
|
||||
* `bytea`, which holds any byte.
|
||||
*/
|
||||
export function refuseNulValues(values: readonly unknown[]): void {
|
||||
if (values.some(holdsNul)) {
|
||||
throw new Error(
|
||||
'a plugin argument holds the character U+0000 (NUL), which Postgres text cannot store; ' +
|
||||
'TREK refuses it instead of changing the value. Remove it, or bind binary data as a Buffer (bytea)',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function holdsNul(value: unknown): boolean {
|
||||
if (typeof value === 'string') return value.includes('\u0000');
|
||||
return Array.isArray(value) && value.some(holdsNul);
|
||||
}
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data';
|
||||
import type { PluginDataStore, PluginTxOp, PluginTxResult } from '../plugin-data';
|
||||
import {
|
||||
PLUGIN_DATA_CAPS,
|
||||
PLUGIN_DATA_ERRORS,
|
||||
refuseNulSql,
|
||||
refuseNulValues,
|
||||
type PluginDataBudgetedCall,
|
||||
} from '../plugin-data-contract';
|
||||
import {
|
||||
PLUGIN_SCHEMA_MAX_OBJECTS,
|
||||
PrivilegedRoleGate,
|
||||
refuseForeignTargets,
|
||||
refuseNulValues,
|
||||
schemaUsage,
|
||||
type PluginRoleGate,
|
||||
type SchemaUsage,
|
||||
} from './plugin-data-checks';
|
||||
import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema';
|
||||
import { pluginSessionSql } from './plugin-session';
|
||||
import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard';
|
||||
|
||||
import { CompiledQuery, type Kysely } from 'kysely';
|
||||
@@ -25,8 +32,11 @@ import Cursor from 'pg-cursor';
|
||||
* once per connection). The transaction first makes sure the schema and its
|
||||
* migration table exist, then puts only that schema on the `search_path`,
|
||||
* with `pg_temp` explicitly last so a temporary table is never found before
|
||||
* the plugin's own, and sets `statement_timeout` to the call's time budget;
|
||||
* every setting is `local` and ends with the transaction. The plugin's SQL is
|
||||
* the plugin's own, sets `statement_timeout` to the call's time budget, and
|
||||
* pins the settings that decide how values print and parse
|
||||
* (`plugin-session.ts`: the time zone, the locale, ...), so what a plugin
|
||||
* reads does not depend on the server's configuration; every setting is
|
||||
* `local` and ends with the transaction. The plugin's SQL is
|
||||
* checked by `plugin-sql-guard.ts` before it runs, so it cannot name another
|
||||
* schema; right before each statement, the existing objects it changes are
|
||||
* resolved by Postgres and must lie in the plugin's schema
|
||||
@@ -34,8 +44,9 @@ import Cursor from 'pg-cursor';
|
||||
* over the extended query protocol, which runs one statement and no more: a
|
||||
* write as a query with `queryMode: 'extended'`, which `pg` honours even
|
||||
* without values, and a read through a `pg-cursor`, which always uses that
|
||||
* protocol. A bound value or a migration id holding U+0000 is refused before
|
||||
* anything runs, since Postgres text cannot hold it and changing it would
|
||||
* protocol. SQL, a bound value or a migration id holding U+0000 is refused
|
||||
* before anything runs, on this engine as on SQLite (`refuseNulValues` in
|
||||
* `../plugin-data-contract.ts`): Postgres text cannot hold it, and changing it would
|
||||
* change the plugin's data. Reads stop at the row cap. A call that may write
|
||||
* measures the schema before and after (`schemaUsage`: the bytes of every
|
||||
* relation with its TOAST table, and the catalog objects), and rolls back one
|
||||
@@ -50,12 +61,15 @@ import Cursor from 'pg-cursor';
|
||||
* (`pluginDataSlots`), so a busy plugin never takes the connections TREK's
|
||||
* own requests need.
|
||||
*
|
||||
* The SQLite store's semantics carry over: `query` and `tx` are held to the
|
||||
* row cap, `query`, a `tx` batch and an `exec` script to the time budget, a
|
||||
* script's statements each commit on their own, and `migrate` runs once per
|
||||
* id with no time budget. What Postgres adds is that the budget also stops a
|
||||
* single statement (`statement_timeout`), which SQLite cannot, and that an
|
||||
* `exec` with bound args is held to it as well.
|
||||
* The SQLite store's semantics (`../sqlite/plugin-data-store.ts`) carry
|
||||
* over: `query` and `tx` are held to the row cap, `query`, a `tx` batch and
|
||||
* an `exec` script to the time budget, a script's statements each commit on
|
||||
* their own, and `migrate` runs once per id with no time budget. What
|
||||
* Postgres adds is that the budget also stops a single statement
|
||||
* (`statement_timeout`), which SQLite cannot, and that an `exec` with bound
|
||||
* args is held to it as well. A call that breaks the contract is refused
|
||||
* with the words the SQLite store uses (`PLUGIN_DATA_ERRORS`),
|
||||
* better-sqlite3's own among them.
|
||||
*/
|
||||
|
||||
/** The limits one store enforces. Overridable for tests; production uses `PLUGIN_DATA_CAPS`. */
|
||||
@@ -81,8 +95,6 @@ const DEFAULT_LIMITS: PostgresPluginDataLimits = {
|
||||
const QUERY_CANCELED = '57014';
|
||||
/** Rows a cursor fetches at once. */
|
||||
const CHUNK_ROWS = 500;
|
||||
/** What better-sqlite3 says about a closed handle, which the host recognizes. */
|
||||
const CLOSED = 'The database connection is not open';
|
||||
|
||||
/**
|
||||
* What a client runs before it goes back to the pool, outside any
|
||||
@@ -206,7 +218,7 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
const { maxRows } = this.limits;
|
||||
return this.call('query', (budget) =>
|
||||
this.transaction(budget.ms, [statement], (client) => {
|
||||
const rows = { count: 0, max: maxRows, message: `query returned more than ${maxRows} rows` };
|
||||
const rows = { count: 0, max: maxRows, message: PLUGIN_DATA_ERRORS.queryTooManyRows(maxRows) };
|
||||
return this.stream(client, statement, args, rows, budget.check);
|
||||
}),
|
||||
);
|
||||
@@ -222,6 +234,7 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
);
|
||||
}
|
||||
const statements = this.parse(sql);
|
||||
this.refuseIfClosed();
|
||||
return this.call('exec', async (budget) => {
|
||||
// A script runs as SQLite runs one: each statement commits on its own, and
|
||||
// the clock is read before every statement but the first.
|
||||
@@ -234,16 +247,19 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
}
|
||||
|
||||
async tx(ops: PluginTxOp[]): Promise<PluginTxResult> {
|
||||
if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }');
|
||||
if (ops.length === 0) return { results: [] };
|
||||
if (!Array.isArray(ops)) throw new Error(PLUGIN_DATA_ERRORS.txNotArray);
|
||||
if (ops.length === 0) {
|
||||
this.refuseIfClosed();
|
||||
return { results: [] };
|
||||
}
|
||||
if (ops.length > PLUGIN_DATA_CAPS.maxTxOps) {
|
||||
throw new Error(`tx allows at most ${PLUGIN_DATA_CAPS.maxTxOps} statements`);
|
||||
throw new Error(PLUGIN_DATA_ERRORS.txTooManyOps(PLUGIN_DATA_CAPS.maxTxOps));
|
||||
}
|
||||
const statements = ops.map((op) => this.single(op?.sql, op?.args ?? []));
|
||||
const { maxRows } = this.limits;
|
||||
return this.call('tx', (budget) =>
|
||||
this.transaction(budget.ms, statements, async (client) => {
|
||||
const rows = { count: 0, max: maxRows, message: `tx returned more than ${maxRows} rows in total` };
|
||||
const rows = { count: 0, max: maxRows, message: PLUGIN_DATA_ERRORS.txTooManyRows(maxRows) };
|
||||
const results = await inOrder(statements, async (statement, index) => {
|
||||
const args = ops[index].args ?? [];
|
||||
// One budget for the whole batch: each statement after the first gets what is left of it.
|
||||
@@ -263,6 +279,7 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
async migrate(id: string, sql: string): Promise<{ applied: boolean }> {
|
||||
const statements = this.parse(sql);
|
||||
refuseNulValues([id]);
|
||||
this.refuseIfClosed();
|
||||
return this.enqueue(() =>
|
||||
this.transaction(null, statements, async (client) => {
|
||||
const seen = await client.query({
|
||||
@@ -280,23 +297,37 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
);
|
||||
}
|
||||
|
||||
/** `sql` as exactly one statement, checked against `args`. */
|
||||
/**
|
||||
* `sql` as exactly one statement, checked against `args`. The checks run in
|
||||
* the order the SQLite store meets them (the text, the args, then what
|
||||
* better-sqlite3's prepare says: a closed handle, no statement, more than
|
||||
* one), so a call that breaks several rules is told the same on both.
|
||||
*/
|
||||
private single(sql: unknown, args: readonly unknown[]): PluginStatement {
|
||||
const statements = this.parse(sql);
|
||||
if (statements.length !== 1) throw new Error('sql must hold exactly one statement here; use exec() for a script');
|
||||
checkPlaceholders(statements[0], args);
|
||||
refuseNulValues(args);
|
||||
this.refuseIfClosed();
|
||||
if (statements.length === 0) throw new Error(PLUGIN_DATA_ERRORS.noStatement);
|
||||
if (statements.length > 1) throw new Error(PLUGIN_DATA_ERRORS.moreThanOneStatement);
|
||||
checkPlaceholders(statements[0], args);
|
||||
return statements[0];
|
||||
}
|
||||
|
||||
/** The statements of `sql`, refused when the guard does not allow one or none is there. */
|
||||
/**
|
||||
* The statements of `sql`, refused when it is no string, too long, holds NUL
|
||||
* or holds a statement the guard does not allow. A script may hold none,
|
||||
* as on SQLite, where an empty script runs nothing.
|
||||
*/
|
||||
private parse(sql: unknown): PluginStatement[] {
|
||||
if (!this.open) throw new Error(CLOSED);
|
||||
if (typeof sql !== 'string') throw new Error('sql must be a string');
|
||||
if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error('sql too long');
|
||||
const statements = parsePluginSql(sql);
|
||||
if (statements.length === 0) throw new Error('The supplied SQL string contains no statements');
|
||||
return statements;
|
||||
if (typeof sql !== 'string') throw new Error(PLUGIN_DATA_ERRORS.sqlNotString);
|
||||
if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error(PLUGIN_DATA_ERRORS.sqlTooLong);
|
||||
refuseNulSql(sql);
|
||||
return parsePluginSql(sql);
|
||||
}
|
||||
|
||||
/** Refuses a call once {@link close} ran, with better-sqlite3's words for a closed handle. */
|
||||
private refuseIfClosed(): void {
|
||||
if (!this.open) throw new Error(PLUGIN_DATA_ERRORS.closed);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -304,7 +335,7 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
* turn comes, and reports a statement Postgres cancelled at
|
||||
* `statement_timeout` as the budget running out.
|
||||
*/
|
||||
private call<T>(what: string, work: (budget: Budget) => Promise<T>): Promise<T> {
|
||||
private call<T>(what: PluginDataBudgetedCall, work: (budget: Budget) => Promise<T>): Promise<T> {
|
||||
const { now, timeBudgetMs } = this.limits;
|
||||
return this.enqueue(async () => {
|
||||
const startedAt = now();
|
||||
@@ -312,14 +343,14 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
ms: timeBudgetMs,
|
||||
remaining: () => Math.max(1, timeBudgetMs - (now() - startedAt)),
|
||||
check: () => {
|
||||
if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`);
|
||||
if (now() - startedAt > timeBudgetMs) throw new Error(PLUGIN_DATA_ERRORS.overBudget(what, timeBudgetMs));
|
||||
},
|
||||
};
|
||||
try {
|
||||
return await work(budget);
|
||||
} catch (err) {
|
||||
if (sqlState(err) === QUERY_CANCELED) {
|
||||
throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`, { cause: err });
|
||||
throw new Error(PLUGIN_DATA_ERRORS.overBudget(what, timeBudgetMs), { cause: err });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
@@ -382,7 +413,7 @@ export class PostgresPluginDataStore implements PluginDataStore {
|
||||
`create schema if not exists ${this.quoted}`,
|
||||
`create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`,
|
||||
`set local search_path to ${this.quoted}, pg_temp`,
|
||||
'set local standard_conforming_strings to on',
|
||||
pluginSessionSql(),
|
||||
];
|
||||
if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`);
|
||||
await client.query(setup.join(';\n'));
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* The settings every plugin transaction on Postgres runs with, so that what
|
||||
* a plugin reads never depends on how the server is configured.
|
||||
*
|
||||
* Postgres formats and parses values by settings the server, the database or
|
||||
* TREK's role may set: `TimeZone` decides how a `timestamptz` prints and
|
||||
* where `date_trunc('day', now())` cuts, `lc_monetary` how `money` prints,
|
||||
* `lc_numeric` and `lc_time` what `to_char` writes, `DateStyle` how
|
||||
* `'01/02/2020'` reads, `default_text_search_config` what `text @@ text`
|
||||
* matches. Left to the server, a plugin would answer differently on two
|
||||
* installs, and its output would tell it how the server is set up (its time
|
||||
* zone, its locale). SQLite has no such settings: its dates are UTC and its
|
||||
* numbers plain.
|
||||
*
|
||||
* So the store sets each of these with `SET LOCAL` at the start of every
|
||||
* plugin transaction (`plugin-data-store.ts`), to Postgres's own built-in
|
||||
* default (what a server with an untouched configuration gives), with the
|
||||
* time zone UTC as on SQLite. `SET LOCAL` ends with the transaction, so
|
||||
* nothing reaches TREK's own use of the pool connection. A plugin cannot
|
||||
* change them: the guard refuses `SET`, `set_config` and `current_setting`.
|
||||
*
|
||||
* The `search_path` and `statement_timeout` are set by the store per call,
|
||||
* since they depend on the plugin and its time budget. The Postgres lane
|
||||
* (PGPLUG-023) lists every setting of the server a client may change in the
|
||||
* categories that cover output and statement behaviour, and fails on one
|
||||
* that is neither pinned here nor named there as one that cannot change what
|
||||
* a plugin reads, so a setting a new Postgres adds is decided on, not missed.
|
||||
*/
|
||||
export const PLUGIN_SESSION_SETTINGS: Readonly<Record<string, string>> = {
|
||||
// How dates, times and intervals print and parse.
|
||||
TimeZone: 'UTC',
|
||||
DateStyle: 'ISO, MDY',
|
||||
IntervalStyle: 'postgres',
|
||||
timezone_abbreviations: 'Default',
|
||||
// How money, numbers and month and day names print (`money`, `to_char`, `to_number`).
|
||||
lc_monetary: 'C',
|
||||
lc_numeric: 'C',
|
||||
lc_time: 'C',
|
||||
// How floating point numbers, bytea and xml print and parse.
|
||||
extra_float_digits: '1',
|
||||
bytea_output: 'hex',
|
||||
xmlbinary: 'base64',
|
||||
xmloption: 'content',
|
||||
// The configuration `text @@ text` and the other text search defaults use.
|
||||
default_text_search_config: 'pg_catalog.simple',
|
||||
// How SQL reads: string literals, NULL in array input, `x = NULL`, `quote_ident`.
|
||||
standard_conforming_strings: 'on',
|
||||
backslash_quote: 'safe_encoding',
|
||||
array_nulls: 'on',
|
||||
transform_null_equals: 'off',
|
||||
quote_all_identifiers: 'off',
|
||||
// How many rows a GIN index search may return; 0 is all of them.
|
||||
gin_fuzzy_search_limit: '0',
|
||||
};
|
||||
|
||||
/** The `SET LOCAL` statements for {@link PLUGIN_SESSION_SETTINGS}, in one string. */
|
||||
export function pluginSessionSql(): string {
|
||||
return Object.entries(PLUGIN_SESSION_SETTINGS)
|
||||
.map(([name, value]) => `set local ${name} to '${value}'`)
|
||||
.join(';\n');
|
||||
}
|
||||
@@ -19,9 +19,18 @@ import path from 'node:path';
|
||||
* refused unless its name is in {@link PLUGIN_FUNCTIONS_FILE}, the string, math,
|
||||
* date and time, JSON, array, range, aggregate, window, UUID and text search
|
||||
* functions a data plugin plausibly needs, each of which works only on its
|
||||
* arguments. A name counts as called when `(` follows it (`version()`,
|
||||
* `"version" ()`) or when it follows a dot, Postgres's column notation for
|
||||
* a call (`(16386::oid).obj_description`, `u.obj_description` over `FROM
|
||||
* arguments. None of them takes or returns a type that looks an object up by
|
||||
* a name (`regconfig`, `regclass`, ..., `aclitem`): such a parameter turns a
|
||||
* text argument into a catalog lookup, so `to_tsvector('schema.cfg', ...)`
|
||||
* would tell whether a schema exists. None returns a transaction id type
|
||||
* either, so the only ways to an `xid` value, which `age(xid)` turns into
|
||||
* the server's transaction counter, are the system columns and the type's
|
||||
* name, which the guard refuses. PGPLUG-022 checks every overload of every
|
||||
* listed name on the lane's server.
|
||||
*
|
||||
* A name counts as called when `(` follows it (`version()`, `"version" ()`)
|
||||
* or when it follows a dot, Postgres's column notation for a call
|
||||
* (`(16386::oid).obj_description`, `u.obj_description` over `FROM
|
||||
* unnest(...) u`, `$1.obj_description`), unless it is one of
|
||||
* {@link COLUMN_NAMES}. A plugin's own table or type may still carry such a
|
||||
* name where it cannot be a call: after INTO, TABLE, VIEW, REFERENCES or IF
|
||||
@@ -63,12 +72,13 @@ export const SESSION_KEYWORDS: ReadonlySet<string> = new Set([
|
||||
* returns a single value (`FROM unnest(...) u`, `FROM generate_series(...)
|
||||
* g`) or a parameter (`$1.f`), that value is the argument. None of these
|
||||
* reads anything called that way: `version` takes no argument, so the
|
||||
* notation never calls it, and `money`, `name`, `oid`, `xid` and `xml` only
|
||||
* notation never calls it, and `money`, `name`, `oid` and `xml` only
|
||||
* convert their argument to the type of that name. Every other refused name
|
||||
* is refused after a dot too. PGPLUG-021 holds each name here to that on the
|
||||
* lane's server.
|
||||
* is refused after a dot too. `xid` is not here: `u.xid` casts `u` to the
|
||||
* transaction id type, whose `age()` reads the server's transaction counter.
|
||||
* PGPLUG-021 holds each name here to that on the lane's server.
|
||||
*/
|
||||
export const COLUMN_NAMES: ReadonlySet<string> = new Set(['money', 'name', 'oid', 'version', 'xid', 'xml']);
|
||||
export const COLUMN_NAMES: ReadonlySet<string> = new Set(['money', 'name', 'oid', 'version', 'xml']);
|
||||
|
||||
/** The function names of `pg_catalog` on every supported major, one per line, sorted. */
|
||||
export const CATALOG_FUNCTIONS_FILE = path.join(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { PLUGIN_DATA_ERRORS, PLUGIN_TRANSACTION_CONTROL, refuseNulSql } from '../plugin-data-contract';
|
||||
import { PLUGIN_SCHEMA_PREFIX } from './plugin-schema';
|
||||
import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plugin-sql-functions';
|
||||
|
||||
@@ -83,6 +84,24 @@ import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plu
|
||||
* `regcollation`, and their array types `_regclass`, ...), in every
|
||||
* spelling: `'x'::regtype`, `CAST('x' AS regtype)`, `regtype 'x'`, quoted;
|
||||
* and `aclitem`, whose input and output name roles;
|
||||
* - a read of a system column every Postgres table has (`xmin`, `xmax`,
|
||||
* `cmin`, `cmax`, `ctid`, `tableoid`), quoted or not: they show the
|
||||
* server's transaction ids, where a row sits on disk and the table's OID,
|
||||
* and SQLite has none of them. Nothing of TREK's reads them either; a
|
||||
* plugin that wants optimistic locking keeps a version column of its own.
|
||||
* The name may still be given to a value of the plugin's own, after AS or
|
||||
* in the column list of a WITH query or an alias (`min(lng) AS xmin`);
|
||||
* - the transaction id types `xid` and `xid8` (and `_xid`, `_xid8`) where
|
||||
* the word stands for the type: `age('3'::xid)` is how many transactions
|
||||
* the cluster ran since. That is after `::`, before a string, as a call
|
||||
* or after a dot (`xid(x)` and `x.xid` are casts when `x` has no such
|
||||
* column), after AS in a CAST, and after any word but the keywords that
|
||||
* precede a column (`c xid` in a column definition, `TYPE xid`). A column
|
||||
* of the plugin's own may be called `xid` (`SELECT xid FROM payments`);
|
||||
* - a collation other than `"C"`, `"POSIX"`, `"default"` and `ucs_basic`,
|
||||
* and an access method other than the built-in ones (`USING bloom`): which
|
||||
* others exist depends on the server's locales, ICU and extensions, so
|
||||
* naming one would tell a plugin about the server;
|
||||
* - a temporary object: CREATE TEMP or TEMPORARY (also after OR REPLACE,
|
||||
* LOCAL or GLOBAL), and SELECT ... INTO TEMP; a temporary table outlives
|
||||
* the call on a pooled connection. Elsewhere `temp` is an ordinary name;
|
||||
@@ -104,11 +123,23 @@ import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plu
|
||||
/** Why a plugin's SQL is refused. The message is the plugin's to read, so it says what to change. */
|
||||
export class PluginSqlRefused extends Error {
|
||||
constructor(reason: string) {
|
||||
super(`statement type not allowed for plugin databases: ${reason}`);
|
||||
super(`${PLUGIN_DATA_ERRORS.statementNotAllowed}: ${reason}`);
|
||||
this.name = 'PluginSqlRefused';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transaction control, refused with the contract's whole sentence
|
||||
* (`PLUGIN_DATA_ERRORS.txControl`), which the SQLite store and the SDK throw
|
||||
* too, so a plugin reads the same words on both engines.
|
||||
*/
|
||||
class TransactionControlRefused extends PluginSqlRefused {
|
||||
constructor() {
|
||||
super('');
|
||||
this.message = PLUGIN_DATA_ERRORS.txControl;
|
||||
}
|
||||
}
|
||||
|
||||
/** One token of plugin SQL. A word is a name or a keyword: `value` is folded to lower case unless it was quoted. */
|
||||
type Token = (
|
||||
| { kind: 'word'; value: string; quoted: boolean }
|
||||
@@ -177,9 +208,6 @@ const STATEMENTS = new Set([
|
||||
'drop',
|
||||
]);
|
||||
|
||||
/** Transaction control: every call already runs in a transaction of the host's. */
|
||||
const TRANSACTION_CONTROL = new Set(['begin', 'start', 'commit', 'end', 'rollback', 'abort', 'savepoint', 'release']);
|
||||
|
||||
/** What a CREATE, ALTER or DROP may act on. */
|
||||
const OBJECTS = new Set(['table', 'index', 'view', 'materialized', 'sequence', 'type']);
|
||||
|
||||
@@ -217,6 +245,48 @@ const NAME_LOOKUP_TYPES = [
|
||||
*/
|
||||
const ROLE_LOOKUP_TYPES = ['aclitem', '_aclitem'];
|
||||
|
||||
/** The system columns of every Postgres table, refused where they are read ({@link checkSystemColumn}). */
|
||||
const SYSTEM_COLUMNS = new Set(['xmin', 'xmax', 'cmin', 'cmax', 'ctid', 'tableoid']);
|
||||
|
||||
/**
|
||||
* The transaction id types and their array types, refused where the word
|
||||
* stands for the type ({@link checkXidType}): an `xid` value is all
|
||||
* `age(xid)` needs to show the server's transaction counter.
|
||||
*/
|
||||
const XID_TYPES = new Set(['xid', '_xid', 'xid8', '_xid8']);
|
||||
|
||||
/**
|
||||
* The keywords after which a word names a column, a table or an alias and
|
||||
* never a type: `SELECT xid`, `WHERE xid = $1`, `ORDER BY xid`, `ADD xid
|
||||
* text`, `INSERT INTO xid`. After AS and RETURNING it depends on what
|
||||
* encloses them ({@link namesType}); after any other word the word is a type
|
||||
* (`c xid` in a column definition, `TYPE xid`) or a bare alias, which a
|
||||
* plugin then writes with AS.
|
||||
*/
|
||||
const NAME_AFTER = new Set(
|
||||
(
|
||||
'select distinct where and or not by set on then else when case between like ilike having from join into ' +
|
||||
'update table column add alter drop rename to with index view exists only lateral over window constraint ' +
|
||||
'references using'
|
||||
).split(' '),
|
||||
);
|
||||
|
||||
/** The calls in which AS is followed by a type: `CAST(x AS xid)`. */
|
||||
const CAST_CALLS = new Set(['cast', 'treat', 'xmlserialize']);
|
||||
|
||||
/** The words a WITH query's body starts with, after `AS (`. */
|
||||
const QUERY_START = new Set(['select', 'values', 'table', 'insert', 'update', 'delete', 'merge', 'with']);
|
||||
|
||||
/**
|
||||
* The collations every Postgres server has. Which others exist depends on
|
||||
* the server's locales and on ICU, so naming one tells a plugin whether the
|
||||
* server has it. `"C"` sorts by code point, as SQLite's BINARY does.
|
||||
*/
|
||||
const COLLATIONS = new Set(['C', 'POSIX', 'default', 'ucs_basic']);
|
||||
|
||||
/** The built-in access methods. Another one comes from an extension, whose presence it would tell. */
|
||||
const ACCESS_METHODS = new Set(['btree', 'hash', 'gist', 'gin', 'spgist', 'brin', 'heap']);
|
||||
|
||||
/** Functions and types refused under any spelling, quoted or not. */
|
||||
const REFUSED_NAMES = new Set([
|
||||
'set_config',
|
||||
@@ -317,12 +387,10 @@ function truncateName(name: string): string {
|
||||
|
||||
/** Splits `sql` into tokens, or refuses what this guard does not read the way Postgres would. */
|
||||
export function tokenize(sql: string): Token[] {
|
||||
if (sql.includes('\u0000')) {
|
||||
// The protocol ends a statement's text at a NUL, so what Postgres would read is not what was checked.
|
||||
throw new PluginSqlRefused(
|
||||
'the SQL holds the character U+0000 (NUL), which Postgres cannot read in a statement or store in text',
|
||||
);
|
||||
}
|
||||
// The protocol ends a statement's text at a NUL, so what Postgres would read
|
||||
// is not what was checked. The store refuses it before the guard runs, with
|
||||
// the words every engine uses; the guard refuses it too, so it holds on its own.
|
||||
refuseNulSql(sql);
|
||||
const tokens: Token[] = [];
|
||||
let i = 0;
|
||||
const n = sql.length;
|
||||
@@ -466,11 +534,8 @@ function describe(text: string, tokens: Token[]): PluginStatement {
|
||||
const first = tokens[0];
|
||||
const opensWithParen = first.kind === 'punct' && first.value === '(';
|
||||
const verb = opensWithParen ? 'select' : keyword(first);
|
||||
if (verb !== null && TRANSACTION_CONTROL.has(verb)) {
|
||||
throw new PluginSqlRefused(
|
||||
'transaction-control statements are not allowed: each call runs in its own transaction, and tx() makes a batch atomic',
|
||||
);
|
||||
}
|
||||
// Transaction control: every call already runs in a transaction of the host's.
|
||||
if (verb !== null && PLUGIN_TRANSACTION_CONTROL.has(verb)) throw new TransactionControlRefused();
|
||||
if (verb === null || !STATEMENTS.has(verb)) {
|
||||
throw new PluginSqlRefused(`a statement may not start with ${describeToken(first)}`);
|
||||
}
|
||||
@@ -490,11 +555,14 @@ function describe(text: string, tokens: Token[]): PluginStatement {
|
||||
if (token.kind === 'punct' && token.value === ')') depth--;
|
||||
if (token.kind !== 'word') continue;
|
||||
checkName(token.value);
|
||||
checkSystemColumn(tokens, index);
|
||||
checkXidType(tokens, index);
|
||||
checkCall(tokens, index);
|
||||
const after = tokens[index + 1];
|
||||
if (after?.kind === 'punct' && after.value === '.') qualifies = true;
|
||||
if (token.quoted) continue;
|
||||
checkKeyword(token.value);
|
||||
checkServerNames(verb, tokens, index);
|
||||
if (token.value === 'into') checkInto(tokens, index, mainAt);
|
||||
if (verb === 'alter' && (token.value === 'schema' || token.value === 'owner')) {
|
||||
throw new PluginSqlRefused(`an ALTER may not change the ${token.value}`);
|
||||
@@ -720,6 +788,132 @@ function depthAt(tokens: readonly Token[], index: number): number {
|
||||
return depth;
|
||||
}
|
||||
|
||||
/** The token before the `(` that encloses the token at `index`, the name of a call: `cast` in `CAST(x AS t)`. */
|
||||
function opener(tokens: readonly Token[], index: number): Token | undefined {
|
||||
let depth = 0;
|
||||
for (let i = index - 1; i >= 0; i--) {
|
||||
if (isPunct(tokens[i], ')')) depth++;
|
||||
if (isPunct(tokens[i], '(')) {
|
||||
if (depth === 0) return tokens[i - 1];
|
||||
depth--;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a read of a system column ({@link SYSTEM_COLUMNS}): the word
|
||||
* anywhere but where it gives a name ({@link givesName}). `t.xmin`, `xmin`
|
||||
* in WHERE or RETURNING and `"xmin"` read the column of the table in FROM.
|
||||
*/
|
||||
function checkSystemColumn(tokens: readonly Token[], index: number): void {
|
||||
const token = tokens[index];
|
||||
if (token.kind !== 'word' || !SYSTEM_COLUMNS.has(token.value) || givesName(tokens, index)) return;
|
||||
throw new PluginSqlRefused(
|
||||
`"${token.value}" reads a Postgres system column, which a plugin may not read: xmin, xmax, cmin, cmax, ` +
|
||||
"ctid and tableoid show the server's transaction ids and where rows are stored, and SQLite has none of " +
|
||||
'them. Keep a column of your own (a version number, an updated_at) for optimistic locking. A value of ' +
|
||||
`your own may carry the name where you give it (min(lng) AS ${token.value}) and be read from the result, ` +
|
||||
'but the SQL may not refer to it by that name',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the word at `index` gives a name rather than reads one: right
|
||||
* after AS (`min(lng) AS xmin`, `FROM t AS xmin`), or in the column list of
|
||||
* an alias (`AS b(xmin, xmax)`) or of a WITH query (`WITH b(xmin) AS
|
||||
* (SELECT ...)`). A word read wrongly here as a reference is only refused; a
|
||||
* reference read wrongly as a name would let the column through, so a list
|
||||
* counts only when it holds names alone and stands where a list of names
|
||||
* does. `f(xmin) AS (a int)` is a call with a column definition list.
|
||||
*/
|
||||
function givesName(tokens: readonly Token[], index: number): boolean {
|
||||
if (keyword(tokens[index - 1]) === 'as') return true;
|
||||
let first = index;
|
||||
while (isPunct(tokens[first - 1], ',') && tokens[first - 2]?.kind === 'word') first -= 2;
|
||||
let last = index;
|
||||
while (isPunct(tokens[last + 1], ',') && tokens[last + 2]?.kind === 'word') last += 2;
|
||||
if (!isPunct(tokens[first - 1], '(') || !isPunct(tokens[last + 1], ')') || tokens[first - 2]?.kind !== 'word') {
|
||||
return false;
|
||||
}
|
||||
if (keyword(tokens[first - 3]) === 'as') return true;
|
||||
if (keyword(tokens[first - 3]) !== 'with' && !isPunct(tokens[first - 3], ',')) return false;
|
||||
const as = last + 2;
|
||||
if (keyword(tokens[as]) !== 'as') return false;
|
||||
const body = skip(tokens, skip(tokens, as + 1, 'not'), 'materialized');
|
||||
return (
|
||||
isPunct(tokens[body], '(') && (isPunct(tokens[body + 1], '(') || QUERY_START.has(keyword(tokens[body + 1]) ?? ''))
|
||||
);
|
||||
}
|
||||
|
||||
/** Refuses a transaction id type ({@link XID_TYPES}) where the word stands for the type ({@link namesType}). */
|
||||
function checkXidType(tokens: readonly Token[], index: number): void {
|
||||
const token = tokens[index];
|
||||
if (token.kind !== 'word' || !XID_TYPES.has(token.value) || !namesType(tokens, index)) return;
|
||||
throw new PluginSqlRefused(
|
||||
`"${token.value}" stands for a transaction id type here, which a plugin may not use: age() of one shows ` +
|
||||
`how many transactions the server ran. A column of your own may be called ${token.value}, but only ` +
|
||||
`without its table: Postgres reads x.${token.value} as the cast ${token.value}(x) when x has no such ` +
|
||||
`column, so where you need x.${token.value} (excluded.${token.value} in an upsert) rename the column`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the word at `index` stands for a type: after `::`, before a string
|
||||
* (`xid '3'`), as a call or after a dot (both casts: `xid(x)`, `x.xid`),
|
||||
* after `subtype =` (a range type), after AS in a CAST, after RETURNING in
|
||||
* a JSON function, and after any word but {@link NAME_AFTER} (`c xid`,
|
||||
* `TYPE xid`). Anywhere else, after a comma, a parenthesis or an operator,
|
||||
* it is a column.
|
||||
*/
|
||||
function namesType(tokens: readonly Token[], index: number): boolean {
|
||||
const before = tokens[index - 1];
|
||||
const after = tokens[index + 1];
|
||||
if (isPunct(before, '::') || isPunct(before, '.') || after?.kind === 'string') return true;
|
||||
if (isPunct(after, '(')) return !namesObject(tokens, index);
|
||||
if (isPunct(before, '=')) return keyword(tokens[index - 2]) === 'subtype';
|
||||
if (before?.kind !== 'word') return false;
|
||||
const word = keyword(before);
|
||||
if (word === 'as') return CAST_CALLS.has(keyword(opener(tokens, index)) ?? '');
|
||||
if (word === 'returning') return (keyword(opener(tokens, index)) ?? '').startsWith('json');
|
||||
return word === null || !NAME_AFTER.has(word);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a collation or an access method a plugin may not name: the word
|
||||
* after COLLATE, after `collation =` in a CREATE TYPE, after USING where it
|
||||
* names an index's or a table's access method, and after ACCESS METHOD. Only
|
||||
* {@link COLLATIONS} and {@link ACCESS_METHODS}, unqualified, pass.
|
||||
*/
|
||||
function checkServerNames(verb: string, tokens: readonly Token[], index: number): void {
|
||||
const word = keyword(tokens[index]);
|
||||
const before = keyword(tokens[index - 1]);
|
||||
let collation = -1;
|
||||
if (word === 'collate') collation = index + 1;
|
||||
else if (word === 'collation' && verb === 'create' && isPunct(tokens[index + 1], '=')) collation = index + 2;
|
||||
if (collation >= 0 && !named(tokens, collation, COLLATIONS)) {
|
||||
throw new PluginSqlRefused(
|
||||
'a collation other than "C", "POSIX", "default" or ucs_basic may not be named: which others exist ' +
|
||||
'depends on the server\'s locales and ICU. COLLATE "C" sorts by code point, as SQLite does',
|
||||
);
|
||||
}
|
||||
const method =
|
||||
(word === 'using' && (before === 'exclude' || (verb === 'create' && tokens[index + 1]?.kind === 'word'))) ||
|
||||
(word === 'method' && before === 'access');
|
||||
if (method && !named(tokens, index + 1, ACCESS_METHODS)) {
|
||||
throw new PluginSqlRefused(
|
||||
'only the built-in access methods may be named (btree, hash, gist, gin, spgist and brin for an index, ' +
|
||||
'heap for a table): whether another exists depends on the extensions installed on the server',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the token at `at` is one of `names`, unqualified. */
|
||||
function named(tokens: readonly Token[], at: number, names: ReadonlySet<string>): boolean {
|
||||
const token = tokens[at];
|
||||
return token?.kind === 'word' && names.has(token.value) && !isPunct(tokens[at + 1], '.');
|
||||
}
|
||||
|
||||
/** Refuses a name that reaches past the plugin's schema whatever qualifies it. */
|
||||
function checkName(name: string): void {
|
||||
if (REFUSED_NAMES.has(name) || REFUSED_PREFIXES.some((p) => name.startsWith(p)) || XML_EXPORT.test(name)) {
|
||||
|
||||
@@ -0,0 +1,337 @@
|
||||
import { openDatabase } from '../../connection';
|
||||
import type { PluginDataStore } from '../plugin-data';
|
||||
import {
|
||||
PLUGIN_DATA_CAPS,
|
||||
PLUGIN_DATA_ERRORS,
|
||||
refuseNulSql,
|
||||
refuseNulValues,
|
||||
refuseTransactionControl,
|
||||
type PluginDataBudgetedCall,
|
||||
} from '../plugin-data-contract';
|
||||
import { splitAfterSemicolons } from './sql-script';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* A plugin's own data on SQLite (`db:own`): a database file of its own, the
|
||||
* SQLite side of the `PluginDataStore` port (`../plugin-data.ts`; the Postgres
|
||||
* side is a schema per plugin, `../postgres/plugin-data-store.ts`). The host
|
||||
* owns the handle and decides where the file lies
|
||||
* (`nest/plugins/host/plugin-data.backend.ts`); the plugin child never gets a
|
||||
* path or a connection, it reaches this only through RPC (db.exec, db.query,
|
||||
* db.tx, db.migrate). Because it is a separate file, containment is a
|
||||
* filesystem fact: the plugin cannot read TREK's database, and nothing has to
|
||||
* police table names in its SQL.
|
||||
*
|
||||
* A thin guard still refuses statements that would let a plugin leave its
|
||||
* file (ATTACH another database, VACUUM INTO elsewhere, PRAGMA) or tie up the
|
||||
* host with oversize SQL, and transaction control in any statement of any call
|
||||
* (`refuseTransactionControl`, the same refusal as on Postgres: every call
|
||||
* runs in a transaction of its own and tx() makes a batch atomic). Every read
|
||||
* is held to a row cap, and `query`, `tx`
|
||||
* and an `exec` script are held to a time budget (below). A call that breaks
|
||||
* the contract is refused with the words both engines use
|
||||
* (`PLUGIN_DATA_ERRORS`).
|
||||
*
|
||||
* SQL, a bound argument or a migration id holding U+0000 (NUL) is refused
|
||||
* before anything runs, with the same error as on Postgres, whose text cannot
|
||||
* hold the character (`../plugin-data-contract.ts`). TREK never drops or changes
|
||||
* it, since `'a\0b'` and `'ab'` are different keys, so a plugin answers the
|
||||
* same on either engine and its data can move to Postgres. Rows an older TREK
|
||||
* stored with a NUL are still read as they are.
|
||||
*/
|
||||
|
||||
/** The handle `openDatabase` gives. */
|
||||
type Db = ReturnType<typeof openDatabase>;
|
||||
/** A statement of a script, prepared and ready to run without args. */
|
||||
interface ScriptStatement {
|
||||
run(): unknown;
|
||||
}
|
||||
|
||||
// RECURSIVE is the one construct that generates unbounded rows and CPU whatever
|
||||
// the (capped) data size: a `WITH RECURSIVE ...` can spin the synchronous host
|
||||
// forever even on an empty database, which neither the size quota nor the row
|
||||
// cap can stop (an aggregate over it never yields a first row). It is refused
|
||||
// outright; the row and size caps bound everything else.
|
||||
// load_extension is refused as a second line: better-sqlite3 disables
|
||||
// extension loading by default, but refusing it here means a later connection
|
||||
// option cannot turn it into loading an arbitrary shared library.
|
||||
const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i;
|
||||
// What better-sqlite3 throws from prepare() for a statement cut off before its end:
|
||||
// in an exec() script, a trigger body whose semicolon split it.
|
||||
const INCOMPLETE_INPUT = 'incomplete input';
|
||||
|
||||
/**
|
||||
* The limits one store enforces. Overridable for tests; production uses
|
||||
* `PLUGIN_DATA_CAPS`.
|
||||
*
|
||||
* The time budget: better-sqlite3 has no interrupt or progress handler, so it
|
||||
* is checked between the rows a statement yields and between the statements
|
||||
* of a batch or a script. A scan that keeps producing rows, a select whose
|
||||
* every row is expensive, or a script of many statements stops at the first
|
||||
* row or statement past the budget instead of holding the host loop. One
|
||||
* statement on its own still runs to its end (a write such as
|
||||
* `INSERT ... SELECT` over a cartesian product) or to its first row (an
|
||||
* aggregate over one); only moving plugin SQL off the host thread can bound
|
||||
* that, and the plugin-sdk README says so. `migrate` is left unbudgeted on
|
||||
* purpose: a migration stopped part of the way would fail the same way on
|
||||
* every later start, leaving a plugin that cannot update its schema at all.
|
||||
*/
|
||||
interface SqlitePluginDataLimits {
|
||||
maxRows: number;
|
||||
timeBudgetMs: number;
|
||||
/**
|
||||
* The most the file may take on disk. better-sqlite3 runs in the host
|
||||
* process, so an unbounded plugin file would fill the data volume TREK's
|
||||
* own database lives on, and a scan over it would stall the event loop.
|
||||
* `max_page_count` caps the file: a write past it fails with SQLITE_FULL,
|
||||
* contained to the plugin.
|
||||
*/
|
||||
quotaBytes: number;
|
||||
/** The clock the time budget reads. */
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
const DEFAULT_LIMITS: SqlitePluginDataLimits = {
|
||||
maxRows: PLUGIN_DATA_CAPS.maxRows,
|
||||
timeBudgetMs: PLUGIN_DATA_CAPS.timeBudgetMs,
|
||||
quotaBytes: PLUGIN_DATA_CAPS.quotaBytes,
|
||||
now: () => Date.now(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Every open store, so a backup can fold each one's WAL back into its file or
|
||||
* snapshot it before archiving: the host keeps the handles open, so a plain
|
||||
* copy of the files would miss the commits still in the `-wal` sidecar.
|
||||
*/
|
||||
const openStores = new Set<SqlitePluginDataStore>();
|
||||
|
||||
/** The stores open now, for the backup (`nest/plugins/host/plugin-data.service.ts`). */
|
||||
export function openSqlitePluginDataStores(): SqlitePluginDataStore[] {
|
||||
return [...openStores];
|
||||
}
|
||||
|
||||
export class SqlitePluginDataStore implements PluginDataStore {
|
||||
readonly engine = 'sqlite' as const;
|
||||
private readonly db: Db;
|
||||
private readonly limits: SqlitePluginDataLimits;
|
||||
|
||||
/** Opens (and creates) the plugin `pluginId`'s database at `file`. */
|
||||
constructor(
|
||||
readonly pluginId: string,
|
||||
file: string,
|
||||
limits: Partial<SqlitePluginDataLimits> = {},
|
||||
) {
|
||||
this.limits = { ...DEFAULT_LIMITS, ...limits };
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
this.db = openDatabase(file);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
openStores.add(this);
|
||||
// The size cap is per connection and not persisted, so it is set on every open.
|
||||
const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096;
|
||||
this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(this.limits.quotaBytes / pageSize))}`);
|
||||
// Records the applied migrations, so migrate() runs once per (plugin, id).
|
||||
this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`);
|
||||
}
|
||||
|
||||
private guard(sql: unknown): void {
|
||||
if (typeof sql !== 'string') throw new Error(PLUGIN_DATA_ERRORS.sqlNotString);
|
||||
if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error(PLUGIN_DATA_ERRORS.sqlTooLong);
|
||||
refuseNulSql(sql);
|
||||
if (FORBIDDEN.test(sql)) throw new Error(PLUGIN_DATA_ERRORS.statementNotAllowed);
|
||||
refuseTransactionControl(sql);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a call once {@link close} ran, with better-sqlite3's words for a
|
||||
* closed handle. A statement on a closed handle is refused by the driver
|
||||
* itself; this covers the calls that would run none (an empty or
|
||||
* comment-only script, an empty tx), so a closed store refuses every call,
|
||||
* as on Postgres.
|
||||
*/
|
||||
private refuseIfClosed(): void {
|
||||
if (!this.db.open) throw new Error(PLUGIN_DATA_ERRORS.closed);
|
||||
}
|
||||
|
||||
/** A deadline for one call: throws once the call has run past the time budget. */
|
||||
private startBudget(call: PluginDataBudgetedCall): () => void {
|
||||
const { now, timeBudgetMs } = this.limits;
|
||||
const startedAt = now();
|
||||
return () => {
|
||||
if (now() - startedAt > timeBudgetMs) throw new Error(PLUGIN_DATA_ERRORS.overBudget(call, timeBudgetMs));
|
||||
};
|
||||
}
|
||||
|
||||
/** One read statement: its rows up to the row cap, within the time budget. */
|
||||
query(sql: string, args: unknown[] = []): unknown[] {
|
||||
this.guard(sql);
|
||||
refuseNulValues(args);
|
||||
const { maxRows } = this.limits;
|
||||
const checkTime = this.startBudget('query');
|
||||
// iterate() pulls one row at a time, so a result that would grow without
|
||||
// bound is stopped at the cap instead of materialized by all(), and the
|
||||
// clock is read between rows. It is read only once another row has come
|
||||
// back, so a result read to its end is returned, however long that took.
|
||||
const rows: unknown[] = [];
|
||||
for (const row of this.db.prepare(sql).iterate(...(args as never[]))) {
|
||||
if (rows.length > 0) checkTime();
|
||||
rows.push(row);
|
||||
if (rows.length > maxRows) throw new Error(PLUGIN_DATA_ERRORS.queryTooManyRows(maxRows));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* With bound args, one statement. Without, a script of any number (a small
|
||||
* setup script, say), run one statement at a time within the time budget,
|
||||
* checked before each statement after the first.
|
||||
*/
|
||||
exec(sql: string, args: unknown[] = []): { changes: number } {
|
||||
this.guard(sql);
|
||||
refuseNulValues(args);
|
||||
if (args.length > 0) {
|
||||
const info = this.db.prepare(sql).run(...(args as never[]));
|
||||
return { changes: info.changes };
|
||||
}
|
||||
this.refuseIfClosed();
|
||||
this.runScript(sql, this.startBudget('exec'));
|
||||
return { changes: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a script the way `Database#exec` does (statement after statement, each
|
||||
* in autocommit, since the guard refuses a script that would open a
|
||||
* transaction), with `checkTime` read before every statement but the first.
|
||||
* The clock is never read after the last one, so a script whose statements
|
||||
* have all run (and committed) is reported as done. A trigger body is
|
||||
* gathered until SQLite stops calling it incomplete.
|
||||
*/
|
||||
private runScript(sql: string, checkTime: () => void): void {
|
||||
let pending = '';
|
||||
let ranOne = false;
|
||||
for (const piece of splitAfterSemicolons(sql)) {
|
||||
pending += piece;
|
||||
const stmt = this.prepareComplete(pending);
|
||||
if (stmt === 'incomplete') continue;
|
||||
pending = '';
|
||||
if (stmt === 'empty') continue;
|
||||
if (ranOne) checkTime();
|
||||
stmt.run();
|
||||
ranOne = true;
|
||||
}
|
||||
// A statement still open at the end of the script: prepared once more for
|
||||
// SQLite's own error, as Database#exec reports it.
|
||||
if (pending !== '') this.db.prepare(pending);
|
||||
}
|
||||
|
||||
/** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */
|
||||
private prepareComplete(sql: string): ScriptStatement | 'incomplete' | 'empty' {
|
||||
try {
|
||||
return this.db.prepare(sql);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : '';
|
||||
if (message === INCOMPLETE_INPUT) return 'incomplete';
|
||||
if (message === PLUGIN_DATA_ERRORS.noStatement) return 'empty';
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Statements that commit or roll back together, the primitive a plugin
|
||||
* needs for a consistent multi-write (moving an item between two tables).
|
||||
* Each op is one statement; a read (SELECT, RETURNING) yields `{ rows }`, a
|
||||
* write `{ changes }`, and a read sees the batch's own earlier writes.
|
||||
*/
|
||||
tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } {
|
||||
if (!Array.isArray(ops)) throw new Error(PLUGIN_DATA_ERRORS.txNotArray);
|
||||
if (ops.length === 0) {
|
||||
this.refuseIfClosed();
|
||||
return { results: [] };
|
||||
}
|
||||
if (ops.length > PLUGIN_DATA_CAPS.maxTxOps)
|
||||
throw new Error(PLUGIN_DATA_ERRORS.txTooManyOps(PLUGIN_DATA_CAPS.maxTxOps));
|
||||
for (const op of ops) {
|
||||
this.guard(op?.sql);
|
||||
refuseNulValues(op.args ?? []);
|
||||
}
|
||||
const { maxRows } = this.limits;
|
||||
let batchRows = 0; // one row cap for the whole batch, not per statement
|
||||
// One time budget for the whole batch too. A throw inside the transaction
|
||||
// rolls every statement of the batch back, so running out of time is all or nothing.
|
||||
const checkTime = this.startBudget('tx');
|
||||
const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => {
|
||||
const results: Array<{ changes?: number; rows?: unknown[] }> = [];
|
||||
for (const op of batch) {
|
||||
const stmt = this.db.prepare(op.sql);
|
||||
const args = (op.args ?? []) as never[];
|
||||
if (stmt.reader) {
|
||||
const rows: unknown[] = [];
|
||||
for (const row of stmt.iterate(...args)) {
|
||||
rows.push(row);
|
||||
if (++batchRows > maxRows) throw new Error(PLUGIN_DATA_ERRORS.txTooManyRows(maxRows));
|
||||
checkTime();
|
||||
}
|
||||
results.push({ rows });
|
||||
} else {
|
||||
results.push({ changes: stmt.run(...args).changes });
|
||||
}
|
||||
checkTime();
|
||||
}
|
||||
return results;
|
||||
});
|
||||
return { results: run(ops) };
|
||||
}
|
||||
|
||||
/** Runs a migration once, keyed by id. Running it again with the same id is a no-op. */
|
||||
migrate(id: string, sql: string): { applied: boolean } {
|
||||
this.guard(sql);
|
||||
refuseNulValues([id]);
|
||||
const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id);
|
||||
if (seen) return { applied: false };
|
||||
this.db.transaction(() => {
|
||||
this.db.exec(sql);
|
||||
this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now());
|
||||
})();
|
||||
return { applied: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the handle is still open. The host reads it to notice a handle a
|
||||
* terminal-failure dispose closed while it stayed cached, and opens a new
|
||||
* store instead of throwing on reuse.
|
||||
*/
|
||||
isOpen(): boolean {
|
||||
return this.db.open;
|
||||
}
|
||||
|
||||
/**
|
||||
* Folds the WAL back into the database file (checkpoint TRUNCATE), so a copy
|
||||
* of the file is a complete snapshot. No-op on a closed handle.
|
||||
*/
|
||||
checkpoint(): void {
|
||||
if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a consistent copy of the database to `destPath` with VACUUM INTO.
|
||||
* Unlike a file copy it folds in the WAL and reads one point in time, so the
|
||||
* copy is right even while the plugin writes: no torn page, no separate
|
||||
* `-wal` to keep in step. It runs the host's own SQL on the host's own
|
||||
* handle, not plugin SQL, so the plugin guard does not apply.
|
||||
*/
|
||||
snapshotInto(destPath: string): void {
|
||||
fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target exists
|
||||
this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try {
|
||||
openStores.delete(this);
|
||||
this.db.close();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -8,7 +8,7 @@
|
||||
* it prepares the pieces it has gathered so far and keeps gathering while SQLite
|
||||
* reports them as incomplete input.
|
||||
*
|
||||
* Pure, no imports: plugin-data.service.ts runs a plugin's exec() script through it,
|
||||
* Pure, no imports: plugin-data-store.ts runs a plugin's exec() script through it,
|
||||
* one statement at a time, so the time budget can be checked between statements.
|
||||
*/
|
||||
export function splitAfterSemicolons(sql: string): string[] {
|
||||
@@ -26,10 +26,11 @@ import type { AbortableOperationOptions, CompiledQuery, DatabaseConnection, Dial
|
||||
* (`--drop-nul`) and lists each value it changed; by default it refuses
|
||||
* such a value, since the move changes nothing silently.
|
||||
*
|
||||
* A plugin's own data (`db:own`) is not TREK's to change: its SQLite file
|
||||
* (`nest/plugins/host/plugin-data.service.ts`) binds a NUL exactly, so keys
|
||||
* it stored with one stay findable, and its schema on Postgres refuses an
|
||||
* argument holding one with an error (`engine/postgres/plugin-data-checks.ts`).
|
||||
* A plugin's own data (`db:own`) is not TREK's to change: the stores of both
|
||||
* engines (`engine/sqlite/plugin-data-store.ts`,
|
||||
* `engine/postgres/plugin-data-store.ts`) refuse SQL, an argument or a
|
||||
* migration id holding one with the same error (`engine/plugin-data-contract.ts`)
|
||||
* instead of dropping it, so a key never changes meaning.
|
||||
*
|
||||
* A filter is stripped as well as a write, so a lookup finds what the write
|
||||
* stored. Text that spells the character as the six characters `\u0000`
|
||||
|
||||
@@ -51,7 +51,7 @@ fallback that silently disables all checking if someone forgets to augment.
|
||||
`rpc-kit` needs none of that today. Extracting it would mean:
|
||||
|
||||
1. `PluginRpcContext.data` becomes a generic parameter instead of importing
|
||||
`PluginDataDb`.
|
||||
`PluginDataStore`.
|
||||
2. The four envelope type imports (`KnownMethod`, `MethodPermission`, `HookKey`,
|
||||
`HookPermission`) become empty-interface augmentation points, with `AssertExact`
|
||||
guards in the server proving the augmentation still matches the tables.
|
||||
|
||||
@@ -10,10 +10,11 @@ import {
|
||||
poolClients,
|
||||
PostgresPluginDataStore,
|
||||
} from '../../../db/engine/postgres/plugin-data-store';
|
||||
import { SqlitePluginDataStore } from '../../../db/engine/sqlite/plugin-data-store';
|
||||
import { logWarn } from '../../audit/audit-log.logger';
|
||||
import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../../database/database-engine.module';
|
||||
import { pluginDataDir } from '../paths';
|
||||
import { PluginDataDb, removePluginData } from './plugin-data.service';
|
||||
import { pluginDataDir, pluginDbFile } from '../paths';
|
||||
import { removePluginData } from './plugin-data.service';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import type { AbstractSqlConnection } from '@mikro-orm/sql';
|
||||
import { Inject, Injectable, type OnApplicationBootstrap, type Provider } from '@nestjs/common';
|
||||
@@ -39,13 +40,13 @@ export interface PluginDataBackend {
|
||||
|
||||
export const PLUGIN_DATA = Symbol('PLUGIN_DATA');
|
||||
|
||||
/** Each plugin's own SQLite file under the plugin data directory (`plugin-data.service.ts`). */
|
||||
/** Each plugin's own SQLite file under the plugin data directory (`db/engine/sqlite/plugin-data-store.ts`). */
|
||||
@Injectable()
|
||||
export class SqlitePluginDataBackend implements PluginDataBackend {
|
||||
readonly engine = 'sqlite' as const;
|
||||
|
||||
open(pluginId: string): PluginDataStore {
|
||||
return new PluginDataDb(pluginId);
|
||||
return new SqlitePluginDataStore(pluginId, pluginDbFile(pluginId));
|
||||
}
|
||||
|
||||
async erase(pluginId: string): Promise<void> {
|
||||
|
||||
@@ -1,313 +1,31 @@
|
||||
import { openDatabase } from '../../../db/connection';
|
||||
import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data';
|
||||
import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths';
|
||||
import { splitAfterSemicolons } from './sql-script';
|
||||
import type { PluginDataStore } from '../../../db/engine/plugin-data';
|
||||
import { openSqlitePluginDataStores, type SqlitePluginDataStore } from '../../../db/engine/sqlite/plugin-data-store';
|
||||
import { pluginDataDir, pluginsDataRoot } from '../paths';
|
||||
|
||||
import type { Database as Db, Statement } from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs).
|
||||
export type { PluginDataStore };
|
||||
|
||||
/**
|
||||
* A plugin's own sqlite database (#plugins, db:own): the SQLite side of the
|
||||
* `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a
|
||||
* schema per plugin, `db/engine/postgres/plugin-data-store.ts`). The HOST owns the handle;
|
||||
* the plugin child never gets a path or a connection — it can only reach this
|
||||
* through RPC (db.exec / db.query / db.migrate). Because it is a SEPARATE FILE,
|
||||
* containment is a filesystem fact: the plugin physically cannot read trek.db,
|
||||
* and we don't have to police table-name prefixes in its SQL.
|
||||
*
|
||||
* A thin guard still rejects statements that would let a plugin escape its file
|
||||
* (ATTACH another db, VACUUM INTO elsewhere, PRAGMA fiddling) or DoS via
|
||||
* oversize SQL, every read is held to a row cap, and query(), tx() and an exec()
|
||||
* script are held to a time budget (below).
|
||||
*
|
||||
* Bound arguments reach SQLite exactly as the plugin sent them, U+0000 (NUL)
|
||||
* included: the data is the plugin's own, so a key it stored with a NUL is
|
||||
* found again by the same key, and `'a\0bc'` and `'ab\0c'` stay two keys. TREK
|
||||
* drops NUL only from its own data (`db/nul-free.ts`); the Postgres store,
|
||||
* whose text cannot hold the character, refuses such an argument instead.
|
||||
/*
|
||||
* The plugin data directories on disk: erasing one, and copying them all for
|
||||
* a backup. On SQLite each holds the plugin's database file, which the
|
||||
* SQLite store opens (`db/engine/sqlite/plugin-data-store.ts`); on Postgres
|
||||
* it holds only the files a plugin keeps beside its schema.
|
||||
*/
|
||||
|
||||
const MAX_SQL_LENGTH = PLUGIN_DATA_CAPS.maxSqlLength;
|
||||
// RECURSIVE is the one construct that generates unbounded rows/CPU independent of
|
||||
// the (capped) data size — a `WITH RECURSIVE …` can spin the synchronous host
|
||||
// forever even with an empty database, which neither the size quota nor the
|
||||
// result-row cap can stop (an aggregate over it never yields a first row). Refuse
|
||||
// it outright; the row/size caps below bound everything else.
|
||||
// load_extension is included as defense-in-depth: better-sqlite3 disables
|
||||
// extension loading by default (so it's inert today), but banning it in the guard
|
||||
// means a future connection-option slip can't turn it into an arbitrary-.so RCE.
|
||||
const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i;
|
||||
// Transaction-control keywords, matched only at statement start (so CASE…END and
|
||||
// identifiers are unaffected). Refused inside db.tx() so a plugin can't COMMIT the
|
||||
// batch's earlier writes and then have the wrapper report failure — breaking atomicity.
|
||||
const TX_CONTROL = /^\s*(BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|END)\b/i;
|
||||
// Per-plugin on-disk quota. better-sqlite3 is synchronous and runs in the HOST
|
||||
// process, so an unbounded plugin DB is both a disk-exhaustion DoS on the shared
|
||||
// trek.db volume and (via a huge scan) an event-loop stall. max_page_count caps
|
||||
// the file (writes past it fail SQLITE_FULL, contained to the plugin) and bounds
|
||||
// the worst-case scan cost. Result sets are additionally row-capped below so a
|
||||
// recursive CTE / cartesian product can't materialize an unbounded array.
|
||||
const QUOTA_BYTES = PLUGIN_DATA_CAPS.quotaBytes;
|
||||
const MAX_ROWS = PLUGIN_DATA_CAPS.maxRows;
|
||||
// Cap statements per atomic batch so a single tx() can't monopolise the synchronous
|
||||
// host — generous for real write batches, far below anything abusive.
|
||||
const MAX_TX_OPS = PLUGIN_DATA_CAPS.maxTxOps;
|
||||
// Wall-clock budget for one query(), one whole tx() batch or one exec() script.
|
||||
// better-sqlite3 has no interrupt or progress handler, so the budget is checked
|
||||
// between the rows a statement yields and between the statements of a batch or a
|
||||
// script: a scan that keeps producing rows, a select whose every row is expensive, or
|
||||
// a script of many statements stops at the first row or statement past the budget
|
||||
// instead of holding the host loop. One statement on its own still runs to the end
|
||||
// (a write such as `INSERT … SELECT` over a cartesian product) or to its first row
|
||||
// (an aggregate over one); only moving plugin SQL off the host thread can bound that,
|
||||
// and the plugin-sdk README says so. migrate() is left unbudgeted on purpose: a
|
||||
// migration is stopped part of the way only to fail the same way on every later
|
||||
// start, which leaves a plugin that cannot update its schema at all.
|
||||
const TIME_BUDGET_MS = PLUGIN_DATA_CAPS.timeBudgetMs;
|
||||
// What better-sqlite3 throws from prepare() for a statement cut off before its end:
|
||||
// in an exec() script, a trigger body whose semicolon split it.
|
||||
const INCOMPLETE_INPUT = 'incomplete input';
|
||||
// And for a piece that holds no statement at all, only whitespace or comments.
|
||||
const NO_STATEMENT = 'The supplied SQL string contains no statements';
|
||||
|
||||
/** The caps a PluginDataDb enforces. Overridable for tests; production uses the defaults. */
|
||||
export interface PluginDataLimits {
|
||||
maxRows: number;
|
||||
timeBudgetMs: number;
|
||||
/** The clock the time budget reads. */
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
const DEFAULT_LIMITS: PluginDataLimits = { maxRows: MAX_ROWS, timeBudgetMs: TIME_BUDGET_MS, now: () => Date.now() };
|
||||
|
||||
// Every live per-plugin handle, so a backup can WAL-checkpoint them before archiving
|
||||
// (the host keeps these open, so their .db files would otherwise be copied with recent
|
||||
// commits still stranded in the -wal sidecar → a stale/torn snapshot in the backup).
|
||||
const openDbs = new Set<PluginDataDb>();
|
||||
|
||||
/** Fold the WAL back into each open plugin.db so a subsequent file copy is a complete,
|
||||
* consistent snapshot — mirrors the wal_checkpoint the core backup runs on travel.db.
|
||||
* Best-effort per handle; never throws. */
|
||||
/**
|
||||
* Folds the WAL back into each open plugin database so a file copy made next
|
||||
* is a complete, consistent snapshot, as the core backup's checkpoint does for
|
||||
* TREK's own file. Best effort per handle; never throws.
|
||||
*/
|
||||
export function checkpointAllPluginDataDbs(): void {
|
||||
for (const d of openDbs) {
|
||||
for (const store of openSqlitePluginDataStores()) {
|
||||
try {
|
||||
d.checkpoint();
|
||||
store.checkpoint();
|
||||
} catch {
|
||||
/* a busy/closed handle is skipped, not fatal */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class PluginDataDb implements PluginDataStore {
|
||||
private db: Db;
|
||||
readonly pluginId: string;
|
||||
readonly engine = 'sqlite' as const;
|
||||
private readonly limits: PluginDataLimits;
|
||||
|
||||
constructor(pluginId: string, limits: Partial<PluginDataLimits> = {}) {
|
||||
this.pluginId = pluginId;
|
||||
this.limits = { ...DEFAULT_LIMITS, ...limits };
|
||||
fs.mkdirSync(pluginDataDir(pluginId), { recursive: true });
|
||||
this.db = openDatabase(pluginDbFile(pluginId));
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
openDbs.add(this);
|
||||
// Cap the file size (per-connection; not persisted, so set on every open).
|
||||
const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096;
|
||||
this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(QUOTA_BYTES / pageSize))}`);
|
||||
// Track applied migrations so db.migrate is idempotent per (plugin, id).
|
||||
this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`);
|
||||
}
|
||||
|
||||
private guard(sql: string): void {
|
||||
if (typeof sql !== 'string') throw new Error('sql must be a string');
|
||||
if (sql.length > MAX_SQL_LENGTH) throw new Error('sql too long');
|
||||
if (FORBIDDEN.test(sql)) throw new Error('statement type not allowed for plugin databases');
|
||||
}
|
||||
|
||||
/** A deadline for one call: throws once the call has run past the time budget. */
|
||||
private startBudget(what: string): () => void {
|
||||
const { now, timeBudgetMs } = this.limits;
|
||||
const startedAt = now();
|
||||
return () => {
|
||||
if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`);
|
||||
};
|
||||
}
|
||||
|
||||
/** Read query: returns rows up to the row cap, within the time budget. Single statement only. */
|
||||
query(sql: string, args: unknown[] = []): unknown[] {
|
||||
this.guard(sql);
|
||||
const { maxRows } = this.limits;
|
||||
const checkTime = this.startBudget('query');
|
||||
// iterate() pulls one row at a time, so a recursive CTE that would yield
|
||||
// unboundedly is halted at the cap instead of materializing via all(), and the
|
||||
// clock is read between rows. It is read only once another row has come back, so
|
||||
// a result that has been read to its end is returned, however long that took.
|
||||
const rows: unknown[] = [];
|
||||
for (const row of this.db.prepare(sql).iterate(...(args as never[]))) {
|
||||
if (rows.length > 0) checkTime();
|
||||
rows.push(row);
|
||||
if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write statement(s). With bound args, exec() runs one statement. Without, it runs a
|
||||
* script of any number (e.g. a small setup script), one statement at a time and
|
||||
* within the time budget, checked before each statement after the first.
|
||||
*/
|
||||
exec(sql: string, args: unknown[] = []): { changes: number } {
|
||||
this.guard(sql);
|
||||
if (args.length > 0) {
|
||||
const info = this.db.prepare(sql).run(...(args as never[]));
|
||||
return { changes: info.changes };
|
||||
}
|
||||
this.runScript(sql, this.startBudget('exec'));
|
||||
return { changes: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a script the way `Database#exec` did (statement after statement, each in
|
||||
* autocommit unless the script opens a transaction), with `checkTime` read before
|
||||
* every statement but the first. The clock is never read after the last one, so a
|
||||
* script whose statements have all run (and committed) is reported as done. A
|
||||
* trigger body is gathered until SQLite stops calling it incomplete. A script that
|
||||
* runs out of time inside a transaction it opened itself has that transaction rolled
|
||||
* back, so the connection is not left holding it.
|
||||
*/
|
||||
private runScript(sql: string, checkTime: () => void): void {
|
||||
const openedNoTransaction = !this.db.inTransaction;
|
||||
let pending = '';
|
||||
let ranOne = false;
|
||||
for (const piece of splitAfterSemicolons(sql)) {
|
||||
pending += piece;
|
||||
const stmt = this.prepareComplete(pending);
|
||||
if (stmt === 'incomplete') continue;
|
||||
pending = '';
|
||||
if (stmt === 'empty') continue;
|
||||
if (ranOne) {
|
||||
try {
|
||||
checkTime();
|
||||
} catch (e) {
|
||||
if (openedNoTransaction && this.db.inTransaction) this.db.prepare('ROLLBACK').run();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
stmt.run();
|
||||
ranOne = true;
|
||||
}
|
||||
// A statement still open at the end of the script: prepare it once more for
|
||||
// SQLite's own error, as Database#exec reports it.
|
||||
if (pending !== '') this.db.prepare(pending);
|
||||
}
|
||||
|
||||
/** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */
|
||||
private prepareComplete(sql: string): Statement | 'incomplete' | 'empty' {
|
||||
try {
|
||||
return this.db.prepare(sql);
|
||||
} catch (e) {
|
||||
const message = e instanceof Error ? e.message : '';
|
||||
if (message === INCOMPLETE_INPUT) return 'incomplete';
|
||||
if (message === NO_STATEMENT) return 'empty';
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomic batch on the plugin's OWN db: every op runs in a single transaction, so
|
||||
* they all commit or all roll back — the primitive a plugin needs for a consistent
|
||||
* multi-write (e.g. move an item between two tables). Each op is ONE statement;
|
||||
* a read (SELECT/RETURNING) yields `{ rows }`, a write yields `{ changes }`, and
|
||||
* reads within the batch see the batch's own earlier writes (read-modify-write).
|
||||
*/
|
||||
tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } {
|
||||
if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }');
|
||||
if (ops.length === 0) return { results: [] };
|
||||
if (ops.length > MAX_TX_OPS) throw new Error(`tx allows at most ${MAX_TX_OPS} statements`);
|
||||
for (const op of ops) {
|
||||
this.guard(op?.sql);
|
||||
// Reject transaction-control statements: a raw COMMIT/ROLLBACK inside the batch
|
||||
// would break atomicity — it commits the earlier writes even though the wrapper
|
||||
// then reports the tx as failed. Strip any LEADING comments/whitespace first so a
|
||||
// `/* */COMMIT` or `-- x\nCOMMIT` can't slip past the start-anchored check; these
|
||||
// keywords are only valid at statement start, so CASE ... END is unaffected.
|
||||
const head = String(op?.sql ?? '').replace(/^(?:\s|--[^\n]*\n?|\/\*[\s\S]*?\*\/)*/, '');
|
||||
if (TX_CONTROL.test(head)) throw new Error('transaction-control statements are not allowed inside tx()');
|
||||
}
|
||||
const { maxRows } = this.limits;
|
||||
let batchRows = 0; // one row budget for the WHOLE batch, not per statement
|
||||
// One time budget for the whole batch too. A throw inside the transaction rolls
|
||||
// every statement of the batch back, so running out of time is all-or-nothing.
|
||||
const checkTime = this.startBudget('tx');
|
||||
const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => {
|
||||
const results: Array<{ changes?: number; rows?: unknown[] }> = [];
|
||||
for (const op of batch) {
|
||||
const stmt = this.db.prepare(op.sql);
|
||||
const args = (op.args ?? []) as never[];
|
||||
if (stmt.reader) {
|
||||
const rows: unknown[] = [];
|
||||
for (const row of stmt.iterate(...args)) {
|
||||
rows.push(row);
|
||||
if (++batchRows > maxRows) throw new Error(`tx returned more than ${maxRows} rows in total`);
|
||||
checkTime();
|
||||
}
|
||||
results.push({ rows });
|
||||
} else {
|
||||
results.push({ changes: stmt.run(...args).changes });
|
||||
}
|
||||
checkTime();
|
||||
}
|
||||
return results;
|
||||
});
|
||||
return { results: run(ops) };
|
||||
}
|
||||
|
||||
/** Run a migration once, keyed by id. Re-running with the same id is a no-op. */
|
||||
migrate(id: string, sql: string): { applied: boolean } {
|
||||
this.guard(sql);
|
||||
const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id);
|
||||
if (seen) return { applied: false };
|
||||
this.db.transaction(() => {
|
||||
this.db.exec(sql);
|
||||
this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now());
|
||||
})();
|
||||
return { applied: true };
|
||||
}
|
||||
|
||||
/** Whether the underlying sqlite handle is still open (better-sqlite3 `.open`).
|
||||
* The host uses this to detect a handle closed by a terminal-failure dispose that
|
||||
* left the instance cached, so it can recreate it instead of throwing on reuse. */
|
||||
isOpen(): boolean {
|
||||
return this.db.open;
|
||||
}
|
||||
|
||||
/** Fold the WAL back into the main db file (checkpoint TRUNCATE) so a file-level copy
|
||||
* is a complete snapshot. No-op on a closed handle. */
|
||||
checkpoint(): void {
|
||||
if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)');
|
||||
}
|
||||
|
||||
/** Write a fully-consistent copy of this DB to `destPath` via VACUUM INTO. Unlike a
|
||||
* file copy it folds in the WAL and reads a point-in-time snapshot, so the result is
|
||||
* correct even while the plugin is writing — no torn page, no separate -wal to keep in
|
||||
* sync. This is a host op on the host's own handle, not plugin SQL, so it bypasses the
|
||||
* FORBIDDEN guard by design. */
|
||||
snapshotInto(destPath: string): void {
|
||||
fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target already exists
|
||||
this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try {
|
||||
openDbs.delete(this);
|
||||
this.db.close();
|
||||
} catch {
|
||||
/* already closed */
|
||||
/* a busy or closed handle is skipped, not fatal */
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -321,7 +39,7 @@ export function removePluginData(pluginId: string): void {
|
||||
* Copy every plugin's data dir into `destRoot` as a CONSISTENT snapshot, for a backup to
|
||||
* archive instead of the live tree. An open plugin.db is captured with VACUUM INTO (safe
|
||||
* under concurrent writes); a plugin with no live handle is copied as-is (no writer). The
|
||||
* -wal/-shm sidecars are never copied — the snapshot folds them in, and copying them out
|
||||
* -wal/-shm sidecars are never copied: the snapshot folds them in, and copying them out
|
||||
* of step with the .db is exactly what produced torn/corrupt restores when the archiver
|
||||
* read the live files lazily while a plugin kept writing. Blobs and any other files a
|
||||
* plugin wrote to its dir are copied verbatim. Best-effort per file; never throws.
|
||||
@@ -329,8 +47,8 @@ export function removePluginData(pluginId: string): void {
|
||||
export function snapshotAllPluginDataDbs(destRoot: string): void {
|
||||
const root = pluginsDataRoot();
|
||||
if (!fs.existsSync(root)) return;
|
||||
const openById = new Map<string, PluginDataDb>();
|
||||
for (const d of openDbs) if (d.isOpen()) openById.set(d.pluginId, d);
|
||||
const openById = new Map<string, SqlitePluginDataStore>();
|
||||
for (const store of openSqlitePluginDataStores()) if (store.isOpen()) openById.set(store.pluginId, store);
|
||||
for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const srcDir = path.join(root, entry.name);
|
||||
@@ -339,7 +57,7 @@ export function snapshotAllPluginDataDbs(destRoot: string): void {
|
||||
const open = openById.get(entry.name);
|
||||
// Handle the live db up front so we know whether its WAL got folded in. If both
|
||||
// the VACUUM INTO snapshot and the checkpoint fail, the -wal/-shm are NOT folded,
|
||||
// so they must be copied alongside the .db — a .db stripped of an un-checkpointed
|
||||
// so they must be copied alongside the .db: a .db stripped of an un-checkpointed
|
||||
// WAL loses committed transactions, whereas the .db + its WAL is a recoverable set.
|
||||
let foldedIn = false;
|
||||
if (open) {
|
||||
@@ -351,18 +69,18 @@ export function snapshotAllPluginDataDbs(destRoot: string): void {
|
||||
open.checkpoint();
|
||||
foldedIn = true;
|
||||
} catch {
|
||||
/* WAL not folded — keep sidecars */
|
||||
/* WAL not folded: keep the sidecars */
|
||||
}
|
||||
try {
|
||||
fs.copyFileSync(path.join(srcDir, 'plugin.db'), path.join(destDir, 'plugin.db'));
|
||||
} catch {
|
||||
/* unreadable live db — best effort */
|
||||
/* unreadable live db: best effort */
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const f of fs.readdirSync(srcDir, { withFileTypes: true })) {
|
||||
if (f.name === 'plugin.db' && open) continue; // already snapshotted above
|
||||
// Skip the .db sidecars only when the live handle's WAL was folded in — VACUUM
|
||||
// Skip the .db sidecars only when the live handle's WAL was folded in: VACUUM
|
||||
// INTO / checkpoint absorbs them, and copying them out of step with a live writer
|
||||
// is what produced torn restores. For a plugin with NO open handle there is no
|
||||
// writer, so the -wal/-shm are a consistent set with the .db; copy them too, or an
|
||||
|
||||
@@ -29,7 +29,7 @@ export function getPluginDataDb(id: string, backend: PluginDataBackend): PluginD
|
||||
let d = dataDbs.get(id);
|
||||
// A cached handle can be CLOSED without being evicted: the supervisor's terminal
|
||||
// failure paths (activation timeout / load-error / crash auto-disable) call
|
||||
// rpcHost.dispose() → PluginDataDb.close() directly, never closePluginDataDb. A
|
||||
// rpcHost.dispose() → the store's close() directly, never closePluginDataDb. A
|
||||
// plain admin re-enable would then reuse the closed handle and every db.* call
|
||||
// would throw 'database connection is not open'. Recreate when the handle is shut.
|
||||
if (!d || !d.isOpen()) {
|
||||
|
||||
Reference in New Issue
Block a user