From c970245e0d022f36bdca2d80b4f6cdde92acae66 Mon Sep 17 00:00:00 2001 From: Maurice Date: Wed, 18 Nov 2026 13:20:07 +0100 Subject: [PATCH] fix(plugins): hold plugin data on Postgres to its own schema and to one contract on both engines The plugin SQL guard now refuses what still reached outside a plugin's schema or depended on the server: - reads of the system columns (xmin, xmax, cmin, cmax, ctid, tableoid), which show transaction ids and storage layout and do not exist on SQLite; a value of the plugin's own may still be named xmin with AS - the xid and xid8 types wherever the word stands for the type, since age() of an xid is the cluster's transaction counter - the text search functions that coerce a text argument to regconfig, a catalog lookup by name, which leave the allowlist - collations and access methods outside the set every server has, which could tell whether a locale or an extension exists Every plugin transaction pins, with SET LOCAL, the settings that decide how values print and parse (TimeZone UTC, DateStyle, the C locales, float, bytea and xml output, the text search configuration), so a plugin's output neither depends on nor reveals the server's configuration. Both stores now keep one contract. Its texts and caps live once, in plugin-data-contract.ts, and both stores throw only those: NUL in SQL, an argument or a migration id is refused on SQLite as on Postgres, transaction control is refused in every call on both engines (tx() is how a batch becomes atomic), a closed store refuses every call, and the Postgres store answers the cases better-sqlite3 names with its words. The SQLite store moves beside the Postgres one as SqlitePluginDataStore in db/engine/sqlite, takes its file from the backend so src/db never imports src/nest, and no longer needs the driver exemption. A contract test runs every violation on both stores and fails when they answer differently. The Postgres lane holds the allowlist, the pinned settings and the SQL plugins commonly write against a real database. --- server/assets/postgres/plugin-functions.txt | 17 +- server/eslint.config.mjs | 4 - server/src/db/engine/plugin-data-contract.ts | 355 ++++++++++++++++++ server/src/db/engine/plugin-data.ts | 30 +- .../db/engine/postgres/plugin-data-checks.ts | 21 -- .../db/engine/postgres/plugin-data-store.ts | 97 +++-- .../src/db/engine/postgres/plugin-session.ts | 61 +++ .../engine/postgres/plugin-sql-functions.ts | 24 +- .../db/engine/postgres/plugin-sql-guard.ts | 224 ++++++++++- .../src/db/engine/sqlite/plugin-data-store.ts | 337 +++++++++++++++++ .../host => db/engine/sqlite}/sql-script.ts | 2 +- server/src/db/nul-free.ts | 9 +- server/src/nest-rpc/rpc-kit/README.md | 2 +- .../nest/plugins/host/plugin-data.backend.ts | 9 +- .../nest/plugins/host/plugin-data.service.ts | 328 ++-------------- .../nest/plugins/host/plugin-host-state.ts | 2 +- .../integration/plugins/supervisor.test.ts | 7 +- server/tests/postgres/plugin-data.test.ts | 278 +++++++++++++- .../db/engine/plugin-data-contract.test.ts | 326 ++++++++++++++++ .../engine/postgres/plugin-data-store.test.ts | 47 ++- .../postgres/plugin-sql-functions.test.ts | 21 +- .../engine/postgres/plugin-sql-guard.test.ts | 93 ++++- .../restricted-imports-xml.test.ts | 9 +- .../unit/plugins/plugin-data-backend.test.ts | 4 +- server/tests/unit/plugins/plugin-data.test.ts | 134 ++++--- server/tests/unit/plugins/sql-script.test.ts | 4 +- 26 files changed, 1932 insertions(+), 513 deletions(-) create mode 100644 server/src/db/engine/plugin-data-contract.ts create mode 100644 server/src/db/engine/postgres/plugin-session.ts create mode 100644 server/src/db/engine/sqlite/plugin-data-store.ts rename server/src/{nest/plugins/host => db/engine/sqlite}/sql-script.ts (96%) create mode 100644 server/tests/unit/db/engine/plugin-data-contract.test.ts diff --git a/server/assets/postgres/plugin-functions.txt b/server/assets/postgres/plugin-functions.txt index 3eb3ab44d..d96545a91 100644 --- a/server/assets/postgres/plugin-functions.txt +++ b/server/assets/postgres/plugin-functions.txt @@ -399,25 +399,24 @@ netmask network set_masklen -# Full text search. ts_stat and ts_rewrite run SQL handed to them as text and stay refused. +# Full text search on vectors and queries a plugin builds itself (array_to_tsvector, +# 'a & b'::tsquery). The functions that parse text into them (to_tsvector, to_tsquery, +# plainto_tsquery, phraseto_tsquery, websearch_to_tsquery, json_to_tsvector, +# jsonb_to_tsvector, ts_headline) take a text search configuration, a regconfig looked up +# by a name in a string, so to_tsvector('schema.cfg', ...) tells whether a schema or a +# configuration exists; they stay refused, as do ts_stat and ts_rewrite, which run SQL +# handed to them as text. No function here may take or return a type that looks an object +# up by name (the Postgres lane checks every overload). array_to_tsvector -json_to_tsvector -jsonb_to_tsvector numnode -phraseto_tsquery -plainto_tsquery querytree setweight strip -to_tsquery -to_tsvector ts_delete ts_filter -ts_headline ts_rank ts_rank_cd tsvector_to_array -websearch_to_tsquery # The TABLESAMPLE methods (`TABLESAMPLE SYSTEM (10)`). bernoulli diff --git a/server/eslint.config.mjs b/server/eslint.config.mjs index 5be7a1ff0..eeeb64ef9 100644 --- a/server/eslint.config.mjs +++ b/server/eslint.config.mjs @@ -423,9 +423,6 @@ export default tseslint.config( 'src/db/connection.ts', 'src/db/orm-driver.ts', 'src/db/durability.ts', - // The plugin sandbox opens its own per-plugin database files, deliberately - // outside the app's ORM and connection. - 'src/nest/plugins/host/plugin-data.service.ts', // Frozen forever (ruling 7): invoked from inside numbered, already-shipped // MikroORM migrations: rewriting it onto the ORM would change what a // re-run of an old migration against an old schema snapshot produces, @@ -445,7 +442,6 @@ export default tseslint.config( 'src/db/connection.ts', 'src/db/orm-driver.ts', 'src/db/durability.ts', - 'src/nest/plugins/host/plugin-data.service.ts', 'src/db/reseat-booked-nights.ts', ], rules: { diff --git a/server/src/db/engine/plugin-data-contract.ts b/server/src/db/engine/plugin-data-contract.ts new file mode 100644 index 000000000..c6b19be7a --- /dev/null +++ b/server/src/db/engine/plugin-data-contract.ts @@ -0,0 +1,355 @@ +/** + * The contract of a plugin's own database (`db:own`) that holds on every + * engine: the caps, the words a call that breaks it is refused with, and the + * refusals of U+0000 (NUL) and of transaction control, so a plugin answers the + * same whichever engine TREK runs on. + * + * TREK's two stores (`server/src/db/engine/sqlite/plugin-data-store.ts` and + * `server/src/db/engine/postgres/plugin-data-store.ts`) throw these texts; + * `server/tests/unit/db/engine/plugin-data-contract.test.ts` runs every case + * on both and fails on a store that answers one differently. + * + * The plugin SDK keeps a byte-identical copy (`plugin-sdk/src/plugin-data-contract.ts`) + * so `trek-plugin dev` and the mock host refuse what TREK refuses, with the + * same words; its `test/permissions-parity.test.ts` compares the two files. + * Change both together. No imports: the SDK ships standalone. + */ + +/** + * The limits every plugin database is held to, whatever the engine. The + * plugin-sdk README's "Runtime limits" table documents them. + */ +export const PLUGIN_DATA_CAPS = { + /** The longest SQL text one call may carry. */ + maxSqlLength: 100_000, + /** The most rows a query, or a whole tx batch, returns. */ + maxRows: 100_000, + /** The most statements one tx batch holds. */ + maxTxOps: 100, + /** Wall-clock time for one query, one tx batch or one exec script. */ + timeBudgetMs: 2_000, + /** The most a plugin's data may take on disk. */ + quotaBytes: 256 * 1024 * 1024, +} as const; + +/** The calls a time budget holds. */ +export type PluginDataBudgetedCall = 'query' | 'exec' | 'tx'; + +/** The start of every refusal of a statement a plugin may not run. */ +const STATEMENT_NOT_ALLOWED = 'statement type not allowed for plugin databases'; + +/** + * What a plugin is told when a call breaks the contract. Both stores throw + * these and nothing else for these cases. + * + * `noStatement`, `moreThanOneStatement` and `closed` are better-sqlite3's own + * messages: the SQLite store passes them on as the driver throws them, and + * the Postgres store says the same. `statementNotAllowed` starts every + * refusal of a statement the store does not run for a plugin; the Postgres + * guard adds the reason after a colon. `txControl` is such a refusal with its + * reason, in the same words on both engines ({@link refuseTransactionControl}). + * `parameterBeforeParenthesis` is the refusal of a spelling no engine TREK + * runs accepts, which the scanner behind `refuseTransactionControl` cannot + * read the way every SQLite build does. + */ +export const PLUGIN_DATA_ERRORS = { + sqlNotString: 'sql must be a string', + sqlTooLong: 'sql too long', + statementNotAllowed: STATEMENT_NOT_ALLOWED, + txControl: + `${STATEMENT_NOT_ALLOWED}: transaction-control statements are not allowed: ` + + 'each call runs in its own transaction, and tx() makes a batch atomic', + parameterBeforeParenthesis: + `${STATEMENT_NOT_ALLOWED}: a parameter name ($name, :name, @name) directly followed by "(" is not allowed: ` + + 'no engine TREK runs accepts it, and the SQLite of trek-plugin dev would read the parentheses as part of the name', + noStatement: 'The supplied SQL string contains no statements', + moreThanOneStatement: 'The supplied SQL string contains more than one statement', + closed: 'The database connection is not open', + txNotArray: 'tx requires an array of { sql, args }', + txTooManyOps: (max: number): string => `tx allows at most ${max} statements`, + queryTooManyRows: (max: number): string => `query returned more than ${max} rows`, + txTooManyRows: (max: number): string => `tx returned more than ${max} rows in total`, + overBudget: (call: PluginDataBudgetedCall, ms: number): string => `${call} exceeded its ${ms} ms time budget`, +} as const; + +/** + * Why a plugin's SQL holding NUL is refused: Postgres ends a statement's text + * at a NUL and its text cannot hold one. On SQLite a NUL in a string literal + * would store a value the same plugin could never store on Postgres. + */ +export const PLUGIN_SQL_NUL_REFUSAL = + 'the SQL holds the character U+0000 (NUL), which plugin databases refuse on every engine: ' + + 'Postgres cannot read it in a statement or store it in text. Remove it from the SQL'; + +/** + * Why a plugin's bound value or migration id holding NUL is refused: TREK + * will not change what a plugin asked to store or compare (`'a\0b'` and + * `'ab'` are different keys). SQLite could store it, but then the plugin + * would answer differently on Postgres and its data could not move there. + */ +export const PLUGIN_VALUE_NUL_REFUSAL = + 'a plugin argument holds the character U+0000 (NUL), which plugin databases refuse on every engine: ' + + 'Postgres text cannot store it, and TREK refuses it instead of changing the value. ' + + 'Remove it, or bind binary data as a Buffer (a BLOB on SQLite, bytea on Postgres)'; + +/** Refuses plugin SQL holding NUL ({@link PLUGIN_SQL_NUL_REFUSAL}). */ +export function refuseNulSql(sql: string): void { + if (sql.includes('\u0000')) throw new Error(PLUGIN_SQL_NUL_REFUSAL); +} + +/** + * Refuses plugin values holding NUL ({@link PLUGIN_VALUE_NUL_REFUSAL}): a + * string, or a string anywhere in an array or a plain object (SQLite binds an + * object's fields as named parameters, `pg` sends one as JSON). A Buffer is + * binary data and holds any byte. + */ +export function refuseNulValues(values: readonly unknown[]): void { + if (holdsNul(values, new Set())) throw new Error(PLUGIN_VALUE_NUL_REFUSAL); +} + +function holdsNul(value: unknown, seen: Set): boolean { + if (typeof value === 'string') return value.includes('\u0000'); + if (typeof value !== 'object' || value === null || seen.has(value)) return false; + seen.add(value); + if (Array.isArray(value)) return value.some((item) => holdsNul(item, seen)); + const proto: unknown = Object.getPrototypeOf(value); + if (proto !== Object.prototype && proto !== null) return false; + return Object.values(value).some((item) => holdsNul(item, seen)); +} + +/** + * The verbs of transaction control, in lower case. TREK runs every call in a + * transaction of its own and `tx()` makes a batch atomic, so a plugin never + * opens, ends or nests one: a COMMIT in the middle would keep writes the call + * then reports as failed. SQLite knows no START or ABORT, but they are refused + * there too, so a plugin is told the same on both engines. + */ +export const PLUGIN_TRANSACTION_CONTROL: ReadonlySet = new Set([ + 'begin', + 'start', + 'commit', + 'end', + 'rollback', + 'abort', + 'savepoint', + 'release', +]); + +/** + * Refuses SQL in which any statement starts with transaction control + * ({@link PLUGIN_DATA_ERRORS.txControl}), in a query, a tx op, a migration or + * any statement of an exec script, before anything runs. The SQLite store and + * the SDK harnesses call it; the Postgres guard finds the same verbs with its + * own lexer. + * + * A named parameter directly followed by `(` is refused as well + * ({@link PLUGIN_DATA_ERRORS.parameterBeforeParenthesis}). A SQLite built with + * Tcl variables, as node:sqlite is under `trek-plugin dev`, reads `$a(...)` + * up to the closing parenthesis as one token, quotes and semicolons + * included, so `$a(')); COMMIT; --')` would hide a COMMIT from this scan. + * better-sqlite3 and Postgres refuse the spelling anyway. + * + * `dialect` says whose spelling `sql` is. The stores and `trek-plugin dev` + * run SQLite and pass nothing. The SDK's mock host emulating Postgres passes + * `'postgres'`, so `$1`, dollar quotes (`$$...$$`, `$tag$...$tag$`) and + * escape strings (`E'...'`) are read as Postgres reads them, `:`, `@` and `#` + * are operators there, and a trigger has no body of statements. + */ +export function refuseTransactionControl(sql: string, dialect: PluginSqlDialect = 'sqlite'): void { + if (statementVerbs(sql, dialect).some((verb) => PLUGIN_TRANSACTION_CONTROL.has(verb))) { + throw new Error(PLUGIN_DATA_ERRORS.txControl); + } +} + +/** Whose spelling a plugin's SQL is in. */ +export type PluginSqlDialect = 'sqlite' | 'postgres'; + +/** A name or keyword as SQLite reads one, unquoted. */ +const WORD = /[A-Za-z_\u0080-￿][A-Za-z0-9_$\u0080-￿]*/y; + +/** A character SQLite reads as part of a name. */ +const NAME_CHAR = /[A-Za-z0-9_$\u0080-￿]/; + +/** The characters a named parameter starts with in SQLite. */ +const PARAMETER_PREFIXES = new Set(['$', ':', '@', '#']); + +/** + * How many leading words of a statement are kept: enough for + * EXPLAIN QUERY PLAN CREATE TEMP TRIGGER. + */ +const LEADING_WORDS = 6; + +/** + * The first word of every statement in `sql`, in lower case, read the way + * SQLite splits a script: quoted text ('...', "...", `...`, [...]) and + * comments are skipped, and a semicolon ends a statement, except inside + * the body of a CREATE TRIGGER, which only `; END ;` ends (as in SQLite's + * sqlite3_complete). The statements of a trigger body are not counted, so + * its closing END is not taken for transaction control. A statement that + * starts with a quoted name or a parenthesis has no verb here. A named + * parameter is read as SQLite reads it ({@link afterParameter}). + */ +function statementVerbs(sql: string, dialect: PluginSqlDialect): string[] { + const verbs: string[] = []; + let words: string[] = []; + let tokens = 0; + let inTrigger = false; + // In a trigger body: whether the last tokens were `;`, or `;` and END. + let afterSemicolon = false; + let afterEnd = false; + let i = 0; + while (i < sql.length) { + const c = sql[i]; + const next = sql[i + 1]; + if (/\s/.test(c)) { + i++; + continue; + } + if (c === '-' && next === '-') { + i = indexAfter(sql, '\n', i + 2); + continue; + } + if (c === '/' && next === '*') { + i = indexAfter(sql, '*/', i + 2); + continue; + } + if (c === ';') { + if (!inTrigger || afterEnd) { + words = []; + tokens = 0; + inTrigger = false; + } + afterSemicolon = inTrigger; + afterEnd = false; + i++; + continue; + } + WORD.lastIndex = i; + const word = WORD.exec(sql)?.[0]; + tokens++; + if (word === undefined) { + i = afterOtherToken(sql, i, dialect); + afterSemicolon = false; + afterEnd = false; + continue; + } + i += word.length; + const lower = word.toLowerCase(); + if (tokens === 1) verbs.push(lower); + if (tokens === words.length + 1 && words.length < LEADING_WORDS) words.push(lower); + if (!inTrigger && dialect === 'sqlite') inTrigger = opensTrigger(words); + afterEnd = afterSemicolon && lower === 'end'; + afterSemicolon = false; + } + return verbs; +} + +/** + * Whether a statement starting with `words` is + * [EXPLAIN [QUERY PLAN]] CREATE [TEMP | TEMPORARY] TRIGGER. + */ +function opensTrigger(words: readonly string[]): boolean { + let at = 0; + if (words[0] === 'explain') at = words[1] === 'query' && words[2] === 'plan' ? 3 : 1; + if (words[at] !== 'create') return false; + at += words[at + 1] === 'temp' || words[at + 1] === 'temporary' ? 2 : 1; + return words[at] === 'trigger'; +} + +/** The index after the token at `start` that is no word: quoted text, a parameter, or one character. */ +function afterOtherToken(sql: string, start: number, dialect: PluginSqlDialect): number { + if (dialect === 'postgres') return afterPostgresToken(sql, start); + const c = sql[start]; + if (c === '[') return indexAfter(sql, ']', start + 1); + if (c === "'" || c === '"' || c === '`') return afterQuoted(sql, start); + if (PARAMETER_PREFIXES.has(c)) return afterParameter(sql, start); + return start + 1; +} + +/** A dollar quote's opening tag, `$$` or `$tag$`, as Postgres reads one. */ +const DOLLAR_TAG = /\$(?:[A-Za-z_\u0080-￿][A-Za-z0-9_\u0080-￿]*)?\$/y; + +/** + * The index after the token at `start` that is no word, read as Postgres reads + * it: quoted text and quoted names, a dollar-quoted string up to its closing + * tag, an escape string (`E'...'`, where a backslash escapes the next + * character), or one character (`$1` is `$` and a number, `::` two colons). + */ +function afterPostgresToken(sql: string, start: number): number { + const c = sql[start]; + if (c === '$') { + DOLLAR_TAG.lastIndex = start; + const tag = DOLLAR_TAG.exec(sql)?.[0]; + if (tag !== undefined) return indexAfter(sql, tag, start + tag.length); + return start + 1; + } + if (c === "'" && isEscapeStringPrefix(sql, start - 1)) return afterEscapeString(sql, start); + if (c === "'" || c === '"') return afterQuoted(sql, start); + return start + 1; +} + +/** Whether the character at `at` is a lone `E`/`e` that makes the string after it an escape string. */ +function isEscapeStringPrefix(sql: string, at: number): boolean { + return (sql[at] === 'E' || sql[at] === 'e') && (at === 0 || !NAME_CHAR.test(sql[at - 1])); +} + +/** The index after the escape string whose quote is at `start`. */ +function afterEscapeString(sql: string, start: number): number { + let i = start + 1; + while (i < sql.length) { + if (sql[i] === '\\') i += 2; + else if (sql[i] !== "'") i++; + else if (sql[i + 1] === "'") i += 2; + else return i + 1; + } + return sql.length; +} + +/** + * The index after the named parameter starting at `start`, read as SQLite's + * tokenizer reads one: name characters, with `::` allowed inside. Refuses a + * name directly followed by `(` ({@link PLUGIN_DATA_ERRORS.parameterBeforeParenthesis}). + * + * A prefix without a name is no parameter, and SQLite refuses the statement + * holding it. Its colons are skipped together, so the scan does not read a + * Postgres cast (`x::numeric(10)`, which the mock host may be handed) as a + * parameter. + */ +function afterParameter(sql: string, start: number): number { + let i = start + 1; + let named = false; + while (i < sql.length) { + if (NAME_CHAR.test(sql[i])) { + named = true; + i++; + } else if (sql[i] === ':' && sql[i + 1] === ':') { + i += 2; + } else { + break; + } + } + if (!named) { + while (sql[i] === ':') i++; + return i; + } + if (sql[i] === '(') throw new Error(PLUGIN_DATA_ERRORS.parameterBeforeParenthesis); + return i; +} + +/** The index after the first `token` from `from`, or the end of `sql`. */ +function indexAfter(sql: string, token: string, from: number): number { + const at = sql.indexOf(token, from); + return at < 0 ? sql.length : at + token.length; +} + +/** The index after the quoted text starting at `start`; a doubled quote stands for itself. */ +function afterQuoted(sql: string, start: number): number { + const quote = sql[start]; + let i = start + 1; + while (i < sql.length) { + if (sql[i] !== quote) i++; + else if (sql[i + 1] === quote) i += 2; + else return i + 1; + } + return sql.length; +} diff --git a/server/src/db/engine/plugin-data.ts b/server/src/db/engine/plugin-data.ts index edcca30a2..9a0014f75 100644 --- a/server/src/db/engine/plugin-data.ts +++ b/server/src/db/engine/plugin-data.ts @@ -2,11 +2,14 @@ import type { EngineId } from '../../app-config/db-settings'; /** * A plugin's own database (`db:own`), as the host's RPC handlers reach it. - * One implementation per engine: on SQLite a file of the plugin's own - * (`nest/plugins/host/plugin-data.service.ts`), on Postgres a schema of the - * plugin's own (`postgres/plugin-data-store.ts`). The plugin's SQL is its - * own, written for the engine `engine` names; neither implementation - * translates it. + * One implementation per engine, each in its engine's folder: on SQLite a + * file of the plugin's own (`SqlitePluginDataStore`, + * `sqlite/plugin-data-store.ts`), on Postgres a schema of the plugin's own + * (`PostgresPluginDataStore`, `postgres/plugin-data-store.ts`). The plugin's + * SQL is its own, written for the engine `engine` names; neither + * implementation translates it. Both hold a plugin to the caps and refuse a + * call that breaks the contract with the same words, from + * `plugin-data-contract.ts`. * * The SQLite handle answers synchronously, the Postgres one with a promise, * so every method may return either. The caller awaits the result. @@ -40,20 +43,3 @@ export interface PluginTxOp { export interface PluginTxResult { results: Array<{ changes?: number; rows?: unknown[] }>; } - -/** - * The limits every plugin database is held to, whatever the engine. The - * plugin-sdk README's "Runtime limits" table documents them. - */ -export const PLUGIN_DATA_CAPS = { - /** The longest SQL text one call may carry. */ - maxSqlLength: 100_000, - /** The most rows a query, or a whole tx batch, returns. */ - maxRows: 100_000, - /** The most statements one tx batch holds. */ - maxTxOps: 100, - /** Wall-clock time for one query, one tx batch or one exec script. */ - timeBudgetMs: 2_000, - /** The most a plugin's data may take on disk. */ - quotaBytes: 256 * 1024 * 1024, -} as const; diff --git a/server/src/db/engine/postgres/plugin-data-checks.ts b/server/src/db/engine/postgres/plugin-data-checks.ts index 2e6a09651..e0e695b36 100644 --- a/server/src/db/engine/postgres/plugin-data-checks.ts +++ b/server/src/db/engine/postgres/plugin-data-checks.ts @@ -138,24 +138,3 @@ export async function schemaUsage(client: PluginSqlClient, schema: string): Prom const [row] = result.rows as Array<{ bytes?: number | string; objects?: number | string }>; return { bytes: Number(row?.bytes ?? 0), objects: Number(row?.objects ?? 0) }; } - -/** - * Refuses a plugin's bound value that holds U+0000 (NUL): Postgres text - * cannot store the character, and TREK will not change what a plugin asked - * to store or compare (`'a\0b'` and `'ab'` are different keys on SQLite). - * A string, or a string anywhere in an array, is checked; a Buffer goes to - * `bytea`, which holds any byte. - */ -export function refuseNulValues(values: readonly unknown[]): void { - if (values.some(holdsNul)) { - throw new Error( - 'a plugin argument holds the character U+0000 (NUL), which Postgres text cannot store; ' + - 'TREK refuses it instead of changing the value. Remove it, or bind binary data as a Buffer (bytea)', - ); - } -} - -function holdsNul(value: unknown): boolean { - if (typeof value === 'string') return value.includes('\u0000'); - return Array.isArray(value) && value.some(holdsNul); -} diff --git a/server/src/db/engine/postgres/plugin-data-store.ts b/server/src/db/engine/postgres/plugin-data-store.ts index aeb131627..dd8de9ab8 100644 --- a/server/src/db/engine/postgres/plugin-data-store.ts +++ b/server/src/db/engine/postgres/plugin-data-store.ts @@ -1,14 +1,21 @@ -import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data'; +import type { PluginDataStore, PluginTxOp, PluginTxResult } from '../plugin-data'; +import { + PLUGIN_DATA_CAPS, + PLUGIN_DATA_ERRORS, + refuseNulSql, + refuseNulValues, + type PluginDataBudgetedCall, +} from '../plugin-data-contract'; import { PLUGIN_SCHEMA_MAX_OBJECTS, PrivilegedRoleGate, refuseForeignTargets, - refuseNulValues, schemaUsage, type PluginRoleGate, type SchemaUsage, } from './plugin-data-checks'; import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema'; +import { pluginSessionSql } from './plugin-session'; import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard'; import { CompiledQuery, type Kysely } from 'kysely'; @@ -25,8 +32,11 @@ import Cursor from 'pg-cursor'; * once per connection). The transaction first makes sure the schema and its * migration table exist, then puts only that schema on the `search_path`, * with `pg_temp` explicitly last so a temporary table is never found before - * the plugin's own, and sets `statement_timeout` to the call's time budget; - * every setting is `local` and ends with the transaction. The plugin's SQL is + * the plugin's own, sets `statement_timeout` to the call's time budget, and + * pins the settings that decide how values print and parse + * (`plugin-session.ts`: the time zone, the locale, ...), so what a plugin + * reads does not depend on the server's configuration; every setting is + * `local` and ends with the transaction. The plugin's SQL is * checked by `plugin-sql-guard.ts` before it runs, so it cannot name another * schema; right before each statement, the existing objects it changes are * resolved by Postgres and must lie in the plugin's schema @@ -34,8 +44,9 @@ import Cursor from 'pg-cursor'; * over the extended query protocol, which runs one statement and no more: a * write as a query with `queryMode: 'extended'`, which `pg` honours even * without values, and a read through a `pg-cursor`, which always uses that - * protocol. A bound value or a migration id holding U+0000 is refused before - * anything runs, since Postgres text cannot hold it and changing it would + * protocol. SQL, a bound value or a migration id holding U+0000 is refused + * before anything runs, on this engine as on SQLite (`refuseNulValues` in + * `../plugin-data-contract.ts`): Postgres text cannot hold it, and changing it would * change the plugin's data. Reads stop at the row cap. A call that may write * measures the schema before and after (`schemaUsage`: the bytes of every * relation with its TOAST table, and the catalog objects), and rolls back one @@ -50,12 +61,15 @@ import Cursor from 'pg-cursor'; * (`pluginDataSlots`), so a busy plugin never takes the connections TREK's * own requests need. * - * The SQLite store's semantics carry over: `query` and `tx` are held to the - * row cap, `query`, a `tx` batch and an `exec` script to the time budget, a - * script's statements each commit on their own, and `migrate` runs once per - * id with no time budget. What Postgres adds is that the budget also stops a - * single statement (`statement_timeout`), which SQLite cannot, and that an - * `exec` with bound args is held to it as well. + * The SQLite store's semantics (`../sqlite/plugin-data-store.ts`) carry + * over: `query` and `tx` are held to the row cap, `query`, a `tx` batch and + * an `exec` script to the time budget, a script's statements each commit on + * their own, and `migrate` runs once per id with no time budget. What + * Postgres adds is that the budget also stops a single statement + * (`statement_timeout`), which SQLite cannot, and that an `exec` with bound + * args is held to it as well. A call that breaks the contract is refused + * with the words the SQLite store uses (`PLUGIN_DATA_ERRORS`), + * better-sqlite3's own among them. */ /** The limits one store enforces. Overridable for tests; production uses `PLUGIN_DATA_CAPS`. */ @@ -81,8 +95,6 @@ const DEFAULT_LIMITS: PostgresPluginDataLimits = { const QUERY_CANCELED = '57014'; /** Rows a cursor fetches at once. */ const CHUNK_ROWS = 500; -/** What better-sqlite3 says about a closed handle, which the host recognizes. */ -const CLOSED = 'The database connection is not open'; /** * What a client runs before it goes back to the pool, outside any @@ -206,7 +218,7 @@ export class PostgresPluginDataStore implements PluginDataStore { const { maxRows } = this.limits; return this.call('query', (budget) => this.transaction(budget.ms, [statement], (client) => { - const rows = { count: 0, max: maxRows, message: `query returned more than ${maxRows} rows` }; + const rows = { count: 0, max: maxRows, message: PLUGIN_DATA_ERRORS.queryTooManyRows(maxRows) }; return this.stream(client, statement, args, rows, budget.check); }), ); @@ -222,6 +234,7 @@ export class PostgresPluginDataStore implements PluginDataStore { ); } const statements = this.parse(sql); + this.refuseIfClosed(); return this.call('exec', async (budget) => { // A script runs as SQLite runs one: each statement commits on its own, and // the clock is read before every statement but the first. @@ -234,16 +247,19 @@ export class PostgresPluginDataStore implements PluginDataStore { } async tx(ops: PluginTxOp[]): Promise { - if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }'); - if (ops.length === 0) return { results: [] }; + if (!Array.isArray(ops)) throw new Error(PLUGIN_DATA_ERRORS.txNotArray); + if (ops.length === 0) { + this.refuseIfClosed(); + return { results: [] }; + } if (ops.length > PLUGIN_DATA_CAPS.maxTxOps) { - throw new Error(`tx allows at most ${PLUGIN_DATA_CAPS.maxTxOps} statements`); + throw new Error(PLUGIN_DATA_ERRORS.txTooManyOps(PLUGIN_DATA_CAPS.maxTxOps)); } const statements = ops.map((op) => this.single(op?.sql, op?.args ?? [])); const { maxRows } = this.limits; return this.call('tx', (budget) => this.transaction(budget.ms, statements, async (client) => { - const rows = { count: 0, max: maxRows, message: `tx returned more than ${maxRows} rows in total` }; + const rows = { count: 0, max: maxRows, message: PLUGIN_DATA_ERRORS.txTooManyRows(maxRows) }; const results = await inOrder(statements, async (statement, index) => { const args = ops[index].args ?? []; // One budget for the whole batch: each statement after the first gets what is left of it. @@ -263,6 +279,7 @@ export class PostgresPluginDataStore implements PluginDataStore { async migrate(id: string, sql: string): Promise<{ applied: boolean }> { const statements = this.parse(sql); refuseNulValues([id]); + this.refuseIfClosed(); return this.enqueue(() => this.transaction(null, statements, async (client) => { const seen = await client.query({ @@ -280,23 +297,37 @@ export class PostgresPluginDataStore implements PluginDataStore { ); } - /** `sql` as exactly one statement, checked against `args`. */ + /** + * `sql` as exactly one statement, checked against `args`. The checks run in + * the order the SQLite store meets them (the text, the args, then what + * better-sqlite3's prepare says: a closed handle, no statement, more than + * one), so a call that breaks several rules is told the same on both. + */ private single(sql: unknown, args: readonly unknown[]): PluginStatement { const statements = this.parse(sql); - if (statements.length !== 1) throw new Error('sql must hold exactly one statement here; use exec() for a script'); - checkPlaceholders(statements[0], args); refuseNulValues(args); + this.refuseIfClosed(); + if (statements.length === 0) throw new Error(PLUGIN_DATA_ERRORS.noStatement); + if (statements.length > 1) throw new Error(PLUGIN_DATA_ERRORS.moreThanOneStatement); + checkPlaceholders(statements[0], args); return statements[0]; } - /** The statements of `sql`, refused when the guard does not allow one or none is there. */ + /** + * The statements of `sql`, refused when it is no string, too long, holds NUL + * or holds a statement the guard does not allow. A script may hold none, + * as on SQLite, where an empty script runs nothing. + */ private parse(sql: unknown): PluginStatement[] { - if (!this.open) throw new Error(CLOSED); - if (typeof sql !== 'string') throw new Error('sql must be a string'); - if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error('sql too long'); - const statements = parsePluginSql(sql); - if (statements.length === 0) throw new Error('The supplied SQL string contains no statements'); - return statements; + if (typeof sql !== 'string') throw new Error(PLUGIN_DATA_ERRORS.sqlNotString); + if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error(PLUGIN_DATA_ERRORS.sqlTooLong); + refuseNulSql(sql); + return parsePluginSql(sql); + } + + /** Refuses a call once {@link close} ran, with better-sqlite3's words for a closed handle. */ + private refuseIfClosed(): void { + if (!this.open) throw new Error(PLUGIN_DATA_ERRORS.closed); } /** @@ -304,7 +335,7 @@ export class PostgresPluginDataStore implements PluginDataStore { * turn comes, and reports a statement Postgres cancelled at * `statement_timeout` as the budget running out. */ - private call(what: string, work: (budget: Budget) => Promise): Promise { + private call(what: PluginDataBudgetedCall, work: (budget: Budget) => Promise): Promise { const { now, timeBudgetMs } = this.limits; return this.enqueue(async () => { const startedAt = now(); @@ -312,14 +343,14 @@ export class PostgresPluginDataStore implements PluginDataStore { ms: timeBudgetMs, remaining: () => Math.max(1, timeBudgetMs - (now() - startedAt)), check: () => { - if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`); + if (now() - startedAt > timeBudgetMs) throw new Error(PLUGIN_DATA_ERRORS.overBudget(what, timeBudgetMs)); }, }; try { return await work(budget); } catch (err) { if (sqlState(err) === QUERY_CANCELED) { - throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`, { cause: err }); + throw new Error(PLUGIN_DATA_ERRORS.overBudget(what, timeBudgetMs), { cause: err }); } throw err; } @@ -382,7 +413,7 @@ export class PostgresPluginDataStore implements PluginDataStore { `create schema if not exists ${this.quoted}`, `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, `set local search_path to ${this.quoted}, pg_temp`, - 'set local standard_conforming_strings to on', + pluginSessionSql(), ]; if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); await client.query(setup.join(';\n')); diff --git a/server/src/db/engine/postgres/plugin-session.ts b/server/src/db/engine/postgres/plugin-session.ts new file mode 100644 index 000000000..5e5f91404 --- /dev/null +++ b/server/src/db/engine/postgres/plugin-session.ts @@ -0,0 +1,61 @@ +/** + * The settings every plugin transaction on Postgres runs with, so that what + * a plugin reads never depends on how the server is configured. + * + * Postgres formats and parses values by settings the server, the database or + * TREK's role may set: `TimeZone` decides how a `timestamptz` prints and + * where `date_trunc('day', now())` cuts, `lc_monetary` how `money` prints, + * `lc_numeric` and `lc_time` what `to_char` writes, `DateStyle` how + * `'01/02/2020'` reads, `default_text_search_config` what `text @@ text` + * matches. Left to the server, a plugin would answer differently on two + * installs, and its output would tell it how the server is set up (its time + * zone, its locale). SQLite has no such settings: its dates are UTC and its + * numbers plain. + * + * So the store sets each of these with `SET LOCAL` at the start of every + * plugin transaction (`plugin-data-store.ts`), to Postgres's own built-in + * default (what a server with an untouched configuration gives), with the + * time zone UTC as on SQLite. `SET LOCAL` ends with the transaction, so + * nothing reaches TREK's own use of the pool connection. A plugin cannot + * change them: the guard refuses `SET`, `set_config` and `current_setting`. + * + * The `search_path` and `statement_timeout` are set by the store per call, + * since they depend on the plugin and its time budget. The Postgres lane + * (PGPLUG-023) lists every setting of the server a client may change in the + * categories that cover output and statement behaviour, and fails on one + * that is neither pinned here nor named there as one that cannot change what + * a plugin reads, so a setting a new Postgres adds is decided on, not missed. + */ +export const PLUGIN_SESSION_SETTINGS: Readonly> = { + // How dates, times and intervals print and parse. + TimeZone: 'UTC', + DateStyle: 'ISO, MDY', + IntervalStyle: 'postgres', + timezone_abbreviations: 'Default', + // How money, numbers and month and day names print (`money`, `to_char`, `to_number`). + lc_monetary: 'C', + lc_numeric: 'C', + lc_time: 'C', + // How floating point numbers, bytea and xml print and parse. + extra_float_digits: '1', + bytea_output: 'hex', + xmlbinary: 'base64', + xmloption: 'content', + // The configuration `text @@ text` and the other text search defaults use. + default_text_search_config: 'pg_catalog.simple', + // How SQL reads: string literals, NULL in array input, `x = NULL`, `quote_ident`. + standard_conforming_strings: 'on', + backslash_quote: 'safe_encoding', + array_nulls: 'on', + transform_null_equals: 'off', + quote_all_identifiers: 'off', + // How many rows a GIN index search may return; 0 is all of them. + gin_fuzzy_search_limit: '0', +}; + +/** The `SET LOCAL` statements for {@link PLUGIN_SESSION_SETTINGS}, in one string. */ +export function pluginSessionSql(): string { + return Object.entries(PLUGIN_SESSION_SETTINGS) + .map(([name, value]) => `set local ${name} to '${value}'`) + .join(';\n'); +} diff --git a/server/src/db/engine/postgres/plugin-sql-functions.ts b/server/src/db/engine/postgres/plugin-sql-functions.ts index bbb5b934d..92e69af0f 100644 --- a/server/src/db/engine/postgres/plugin-sql-functions.ts +++ b/server/src/db/engine/postgres/plugin-sql-functions.ts @@ -19,9 +19,18 @@ import path from 'node:path'; * refused unless its name is in {@link PLUGIN_FUNCTIONS_FILE}, the string, math, * date and time, JSON, array, range, aggregate, window, UUID and text search * functions a data plugin plausibly needs, each of which works only on its - * arguments. A name counts as called when `(` follows it (`version()`, - * `"version" ()`) or when it follows a dot, Postgres's column notation for - * a call (`(16386::oid).obj_description`, `u.obj_description` over `FROM + * arguments. None of them takes or returns a type that looks an object up by + * a name (`regconfig`, `regclass`, ..., `aclitem`): such a parameter turns a + * text argument into a catalog lookup, so `to_tsvector('schema.cfg', ...)` + * would tell whether a schema exists. None returns a transaction id type + * either, so the only ways to an `xid` value, which `age(xid)` turns into + * the server's transaction counter, are the system columns and the type's + * name, which the guard refuses. PGPLUG-022 checks every overload of every + * listed name on the lane's server. + * + * A name counts as called when `(` follows it (`version()`, `"version" ()`) + * or when it follows a dot, Postgres's column notation for a call + * (`(16386::oid).obj_description`, `u.obj_description` over `FROM * unnest(...) u`, `$1.obj_description`), unless it is one of * {@link COLUMN_NAMES}. A plugin's own table or type may still carry such a * name where it cannot be a call: after INTO, TABLE, VIEW, REFERENCES or IF @@ -63,12 +72,13 @@ export const SESSION_KEYWORDS: ReadonlySet = new Set([ * returns a single value (`FROM unnest(...) u`, `FROM generate_series(...) * g`) or a parameter (`$1.f`), that value is the argument. None of these * reads anything called that way: `version` takes no argument, so the - * notation never calls it, and `money`, `name`, `oid`, `xid` and `xml` only + * notation never calls it, and `money`, `name`, `oid` and `xml` only * convert their argument to the type of that name. Every other refused name - * is refused after a dot too. PGPLUG-021 holds each name here to that on the - * lane's server. + * is refused after a dot too. `xid` is not here: `u.xid` casts `u` to the + * transaction id type, whose `age()` reads the server's transaction counter. + * PGPLUG-021 holds each name here to that on the lane's server. */ -export const COLUMN_NAMES: ReadonlySet = new Set(['money', 'name', 'oid', 'version', 'xid', 'xml']); +export const COLUMN_NAMES: ReadonlySet = new Set(['money', 'name', 'oid', 'version', 'xml']); /** The function names of `pg_catalog` on every supported major, one per line, sorted. */ export const CATALOG_FUNCTIONS_FILE = path.join( diff --git a/server/src/db/engine/postgres/plugin-sql-guard.ts b/server/src/db/engine/postgres/plugin-sql-guard.ts index 7765e77ed..72c548502 100644 --- a/server/src/db/engine/postgres/plugin-sql-guard.ts +++ b/server/src/db/engine/postgres/plugin-sql-guard.ts @@ -1,3 +1,4 @@ +import { PLUGIN_DATA_ERRORS, PLUGIN_TRANSACTION_CONTROL, refuseNulSql } from '../plugin-data-contract'; import { PLUGIN_SCHEMA_PREFIX } from './plugin-schema'; import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plugin-sql-functions'; @@ -83,6 +84,24 @@ import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plu * `regcollation`, and their array types `_regclass`, ...), in every * spelling: `'x'::regtype`, `CAST('x' AS regtype)`, `regtype 'x'`, quoted; * and `aclitem`, whose input and output name roles; + * - a read of a system column every Postgres table has (`xmin`, `xmax`, + * `cmin`, `cmax`, `ctid`, `tableoid`), quoted or not: they show the + * server's transaction ids, where a row sits on disk and the table's OID, + * and SQLite has none of them. Nothing of TREK's reads them either; a + * plugin that wants optimistic locking keeps a version column of its own. + * The name may still be given to a value of the plugin's own, after AS or + * in the column list of a WITH query or an alias (`min(lng) AS xmin`); + * - the transaction id types `xid` and `xid8` (and `_xid`, `_xid8`) where + * the word stands for the type: `age('3'::xid)` is how many transactions + * the cluster ran since. That is after `::`, before a string, as a call + * or after a dot (`xid(x)` and `x.xid` are casts when `x` has no such + * column), after AS in a CAST, and after any word but the keywords that + * precede a column (`c xid` in a column definition, `TYPE xid`). A column + * of the plugin's own may be called `xid` (`SELECT xid FROM payments`); + * - a collation other than `"C"`, `"POSIX"`, `"default"` and `ucs_basic`, + * and an access method other than the built-in ones (`USING bloom`): which + * others exist depends on the server's locales, ICU and extensions, so + * naming one would tell a plugin about the server; * - a temporary object: CREATE TEMP or TEMPORARY (also after OR REPLACE, * LOCAL or GLOBAL), and SELECT ... INTO TEMP; a temporary table outlives * the call on a pooled connection. Elsewhere `temp` is an ordinary name; @@ -104,11 +123,23 @@ import { refusedCatalogCall, refusedCatalogField, SESSION_KEYWORDS } from './plu /** Why a plugin's SQL is refused. The message is the plugin's to read, so it says what to change. */ export class PluginSqlRefused extends Error { constructor(reason: string) { - super(`statement type not allowed for plugin databases: ${reason}`); + super(`${PLUGIN_DATA_ERRORS.statementNotAllowed}: ${reason}`); this.name = 'PluginSqlRefused'; } } +/** + * Transaction control, refused with the contract's whole sentence + * (`PLUGIN_DATA_ERRORS.txControl`), which the SQLite store and the SDK throw + * too, so a plugin reads the same words on both engines. + */ +class TransactionControlRefused extends PluginSqlRefused { + constructor() { + super(''); + this.message = PLUGIN_DATA_ERRORS.txControl; + } +} + /** One token of plugin SQL. A word is a name or a keyword: `value` is folded to lower case unless it was quoted. */ type Token = ( | { kind: 'word'; value: string; quoted: boolean } @@ -177,9 +208,6 @@ const STATEMENTS = new Set([ 'drop', ]); -/** Transaction control: every call already runs in a transaction of the host's. */ -const TRANSACTION_CONTROL = new Set(['begin', 'start', 'commit', 'end', 'rollback', 'abort', 'savepoint', 'release']); - /** What a CREATE, ALTER or DROP may act on. */ const OBJECTS = new Set(['table', 'index', 'view', 'materialized', 'sequence', 'type']); @@ -217,6 +245,48 @@ const NAME_LOOKUP_TYPES = [ */ const ROLE_LOOKUP_TYPES = ['aclitem', '_aclitem']; +/** The system columns of every Postgres table, refused where they are read ({@link checkSystemColumn}). */ +const SYSTEM_COLUMNS = new Set(['xmin', 'xmax', 'cmin', 'cmax', 'ctid', 'tableoid']); + +/** + * The transaction id types and their array types, refused where the word + * stands for the type ({@link checkXidType}): an `xid` value is all + * `age(xid)` needs to show the server's transaction counter. + */ +const XID_TYPES = new Set(['xid', '_xid', 'xid8', '_xid8']); + +/** + * The keywords after which a word names a column, a table or an alias and + * never a type: `SELECT xid`, `WHERE xid = $1`, `ORDER BY xid`, `ADD xid + * text`, `INSERT INTO xid`. After AS and RETURNING it depends on what + * encloses them ({@link namesType}); after any other word the word is a type + * (`c xid` in a column definition, `TYPE xid`) or a bare alias, which a + * plugin then writes with AS. + */ +const NAME_AFTER = new Set( + ( + 'select distinct where and or not by set on then else when case between like ilike having from join into ' + + 'update table column add alter drop rename to with index view exists only lateral over window constraint ' + + 'references using' + ).split(' '), +); + +/** The calls in which AS is followed by a type: `CAST(x AS xid)`. */ +const CAST_CALLS = new Set(['cast', 'treat', 'xmlserialize']); + +/** The words a WITH query's body starts with, after `AS (`. */ +const QUERY_START = new Set(['select', 'values', 'table', 'insert', 'update', 'delete', 'merge', 'with']); + +/** + * The collations every Postgres server has. Which others exist depends on + * the server's locales and on ICU, so naming one tells a plugin whether the + * server has it. `"C"` sorts by code point, as SQLite's BINARY does. + */ +const COLLATIONS = new Set(['C', 'POSIX', 'default', 'ucs_basic']); + +/** The built-in access methods. Another one comes from an extension, whose presence it would tell. */ +const ACCESS_METHODS = new Set(['btree', 'hash', 'gist', 'gin', 'spgist', 'brin', 'heap']); + /** Functions and types refused under any spelling, quoted or not. */ const REFUSED_NAMES = new Set([ 'set_config', @@ -317,12 +387,10 @@ function truncateName(name: string): string { /** Splits `sql` into tokens, or refuses what this guard does not read the way Postgres would. */ export function tokenize(sql: string): Token[] { - if (sql.includes('\u0000')) { - // The protocol ends a statement's text at a NUL, so what Postgres would read is not what was checked. - throw new PluginSqlRefused( - 'the SQL holds the character U+0000 (NUL), which Postgres cannot read in a statement or store in text', - ); - } + // The protocol ends a statement's text at a NUL, so what Postgres would read + // is not what was checked. The store refuses it before the guard runs, with + // the words every engine uses; the guard refuses it too, so it holds on its own. + refuseNulSql(sql); const tokens: Token[] = []; let i = 0; const n = sql.length; @@ -466,11 +534,8 @@ function describe(text: string, tokens: Token[]): PluginStatement { const first = tokens[0]; const opensWithParen = first.kind === 'punct' && first.value === '('; const verb = opensWithParen ? 'select' : keyword(first); - if (verb !== null && TRANSACTION_CONTROL.has(verb)) { - throw new PluginSqlRefused( - 'transaction-control statements are not allowed: each call runs in its own transaction, and tx() makes a batch atomic', - ); - } + // Transaction control: every call already runs in a transaction of the host's. + if (verb !== null && PLUGIN_TRANSACTION_CONTROL.has(verb)) throw new TransactionControlRefused(); if (verb === null || !STATEMENTS.has(verb)) { throw new PluginSqlRefused(`a statement may not start with ${describeToken(first)}`); } @@ -490,11 +555,14 @@ function describe(text: string, tokens: Token[]): PluginStatement { if (token.kind === 'punct' && token.value === ')') depth--; if (token.kind !== 'word') continue; checkName(token.value); + checkSystemColumn(tokens, index); + checkXidType(tokens, index); checkCall(tokens, index); const after = tokens[index + 1]; if (after?.kind === 'punct' && after.value === '.') qualifies = true; if (token.quoted) continue; checkKeyword(token.value); + checkServerNames(verb, tokens, index); if (token.value === 'into') checkInto(tokens, index, mainAt); if (verb === 'alter' && (token.value === 'schema' || token.value === 'owner')) { throw new PluginSqlRefused(`an ALTER may not change the ${token.value}`); @@ -720,6 +788,132 @@ function depthAt(tokens: readonly Token[], index: number): number { return depth; } +/** The token before the `(` that encloses the token at `index`, the name of a call: `cast` in `CAST(x AS t)`. */ +function opener(tokens: readonly Token[], index: number): Token | undefined { + let depth = 0; + for (let i = index - 1; i >= 0; i--) { + if (isPunct(tokens[i], ')')) depth++; + if (isPunct(tokens[i], '(')) { + if (depth === 0) return tokens[i - 1]; + depth--; + } + } + return undefined; +} + +/** + * Refuses a read of a system column ({@link SYSTEM_COLUMNS}): the word + * anywhere but where it gives a name ({@link givesName}). `t.xmin`, `xmin` + * in WHERE or RETURNING and `"xmin"` read the column of the table in FROM. + */ +function checkSystemColumn(tokens: readonly Token[], index: number): void { + const token = tokens[index]; + if (token.kind !== 'word' || !SYSTEM_COLUMNS.has(token.value) || givesName(tokens, index)) return; + throw new PluginSqlRefused( + `"${token.value}" reads a Postgres system column, which a plugin may not read: xmin, xmax, cmin, cmax, ` + + "ctid and tableoid show the server's transaction ids and where rows are stored, and SQLite has none of " + + 'them. Keep a column of your own (a version number, an updated_at) for optimistic locking. A value of ' + + `your own may carry the name where you give it (min(lng) AS ${token.value}) and be read from the result, ` + + 'but the SQL may not refer to it by that name', + ); +} + +/** + * Whether the word at `index` gives a name rather than reads one: right + * after AS (`min(lng) AS xmin`, `FROM t AS xmin`), or in the column list of + * an alias (`AS b(xmin, xmax)`) or of a WITH query (`WITH b(xmin) AS + * (SELECT ...)`). A word read wrongly here as a reference is only refused; a + * reference read wrongly as a name would let the column through, so a list + * counts only when it holds names alone and stands where a list of names + * does. `f(xmin) AS (a int)` is a call with a column definition list. + */ +function givesName(tokens: readonly Token[], index: number): boolean { + if (keyword(tokens[index - 1]) === 'as') return true; + let first = index; + while (isPunct(tokens[first - 1], ',') && tokens[first - 2]?.kind === 'word') first -= 2; + let last = index; + while (isPunct(tokens[last + 1], ',') && tokens[last + 2]?.kind === 'word') last += 2; + if (!isPunct(tokens[first - 1], '(') || !isPunct(tokens[last + 1], ')') || tokens[first - 2]?.kind !== 'word') { + return false; + } + if (keyword(tokens[first - 3]) === 'as') return true; + if (keyword(tokens[first - 3]) !== 'with' && !isPunct(tokens[first - 3], ',')) return false; + const as = last + 2; + if (keyword(tokens[as]) !== 'as') return false; + const body = skip(tokens, skip(tokens, as + 1, 'not'), 'materialized'); + return ( + isPunct(tokens[body], '(') && (isPunct(tokens[body + 1], '(') || QUERY_START.has(keyword(tokens[body + 1]) ?? '')) + ); +} + +/** Refuses a transaction id type ({@link XID_TYPES}) where the word stands for the type ({@link namesType}). */ +function checkXidType(tokens: readonly Token[], index: number): void { + const token = tokens[index]; + if (token.kind !== 'word' || !XID_TYPES.has(token.value) || !namesType(tokens, index)) return; + throw new PluginSqlRefused( + `"${token.value}" stands for a transaction id type here, which a plugin may not use: age() of one shows ` + + `how many transactions the server ran. A column of your own may be called ${token.value}, but only ` + + `without its table: Postgres reads x.${token.value} as the cast ${token.value}(x) when x has no such ` + + `column, so where you need x.${token.value} (excluded.${token.value} in an upsert) rename the column`, + ); +} + +/** + * Whether the word at `index` stands for a type: after `::`, before a string + * (`xid '3'`), as a call or after a dot (both casts: `xid(x)`, `x.xid`), + * after `subtype =` (a range type), after AS in a CAST, after RETURNING in + * a JSON function, and after any word but {@link NAME_AFTER} (`c xid`, + * `TYPE xid`). Anywhere else, after a comma, a parenthesis or an operator, + * it is a column. + */ +function namesType(tokens: readonly Token[], index: number): boolean { + const before = tokens[index - 1]; + const after = tokens[index + 1]; + if (isPunct(before, '::') || isPunct(before, '.') || after?.kind === 'string') return true; + if (isPunct(after, '(')) return !namesObject(tokens, index); + if (isPunct(before, '=')) return keyword(tokens[index - 2]) === 'subtype'; + if (before?.kind !== 'word') return false; + const word = keyword(before); + if (word === 'as') return CAST_CALLS.has(keyword(opener(tokens, index)) ?? ''); + if (word === 'returning') return (keyword(opener(tokens, index)) ?? '').startsWith('json'); + return word === null || !NAME_AFTER.has(word); +} + +/** + * Refuses a collation or an access method a plugin may not name: the word + * after COLLATE, after `collation =` in a CREATE TYPE, after USING where it + * names an index's or a table's access method, and after ACCESS METHOD. Only + * {@link COLLATIONS} and {@link ACCESS_METHODS}, unqualified, pass. + */ +function checkServerNames(verb: string, tokens: readonly Token[], index: number): void { + const word = keyword(tokens[index]); + const before = keyword(tokens[index - 1]); + let collation = -1; + if (word === 'collate') collation = index + 1; + else if (word === 'collation' && verb === 'create' && isPunct(tokens[index + 1], '=')) collation = index + 2; + if (collation >= 0 && !named(tokens, collation, COLLATIONS)) { + throw new PluginSqlRefused( + 'a collation other than "C", "POSIX", "default" or ucs_basic may not be named: which others exist ' + + 'depends on the server\'s locales and ICU. COLLATE "C" sorts by code point, as SQLite does', + ); + } + const method = + (word === 'using' && (before === 'exclude' || (verb === 'create' && tokens[index + 1]?.kind === 'word'))) || + (word === 'method' && before === 'access'); + if (method && !named(tokens, index + 1, ACCESS_METHODS)) { + throw new PluginSqlRefused( + 'only the built-in access methods may be named (btree, hash, gist, gin, spgist and brin for an index, ' + + 'heap for a table): whether another exists depends on the extensions installed on the server', + ); + } +} + +/** Whether the token at `at` is one of `names`, unqualified. */ +function named(tokens: readonly Token[], at: number, names: ReadonlySet): boolean { + const token = tokens[at]; + return token?.kind === 'word' && names.has(token.value) && !isPunct(tokens[at + 1], '.'); +} + /** Refuses a name that reaches past the plugin's schema whatever qualifies it. */ function checkName(name: string): void { if (REFUSED_NAMES.has(name) || REFUSED_PREFIXES.some((p) => name.startsWith(p)) || XML_EXPORT.test(name)) { diff --git a/server/src/db/engine/sqlite/plugin-data-store.ts b/server/src/db/engine/sqlite/plugin-data-store.ts new file mode 100644 index 000000000..a9aae8f0b --- /dev/null +++ b/server/src/db/engine/sqlite/plugin-data-store.ts @@ -0,0 +1,337 @@ +import { openDatabase } from '../../connection'; +import type { PluginDataStore } from '../plugin-data'; +import { + PLUGIN_DATA_CAPS, + PLUGIN_DATA_ERRORS, + refuseNulSql, + refuseNulValues, + refuseTransactionControl, + type PluginDataBudgetedCall, +} from '../plugin-data-contract'; +import { splitAfterSemicolons } from './sql-script'; + +import fs from 'node:fs'; +import path from 'node:path'; + +/** + * A plugin's own data on SQLite (`db:own`): a database file of its own, the + * SQLite side of the `PluginDataStore` port (`../plugin-data.ts`; the Postgres + * side is a schema per plugin, `../postgres/plugin-data-store.ts`). The host + * owns the handle and decides where the file lies + * (`nest/plugins/host/plugin-data.backend.ts`); the plugin child never gets a + * path or a connection, it reaches this only through RPC (db.exec, db.query, + * db.tx, db.migrate). Because it is a separate file, containment is a + * filesystem fact: the plugin cannot read TREK's database, and nothing has to + * police table names in its SQL. + * + * A thin guard still refuses statements that would let a plugin leave its + * file (ATTACH another database, VACUUM INTO elsewhere, PRAGMA) or tie up the + * host with oversize SQL, and transaction control in any statement of any call + * (`refuseTransactionControl`, the same refusal as on Postgres: every call + * runs in a transaction of its own and tx() makes a batch atomic). Every read + * is held to a row cap, and `query`, `tx` + * and an `exec` script are held to a time budget (below). A call that breaks + * the contract is refused with the words both engines use + * (`PLUGIN_DATA_ERRORS`). + * + * SQL, a bound argument or a migration id holding U+0000 (NUL) is refused + * before anything runs, with the same error as on Postgres, whose text cannot + * hold the character (`../plugin-data-contract.ts`). TREK never drops or changes + * it, since `'a\0b'` and `'ab'` are different keys, so a plugin answers the + * same on either engine and its data can move to Postgres. Rows an older TREK + * stored with a NUL are still read as they are. + */ + +/** The handle `openDatabase` gives. */ +type Db = ReturnType; +/** A statement of a script, prepared and ready to run without args. */ +interface ScriptStatement { + run(): unknown; +} + +// RECURSIVE is the one construct that generates unbounded rows and CPU whatever +// the (capped) data size: a `WITH RECURSIVE ...` can spin the synchronous host +// forever even on an empty database, which neither the size quota nor the row +// cap can stop (an aggregate over it never yields a first row). It is refused +// outright; the row and size caps bound everything else. +// load_extension is refused as a second line: better-sqlite3 disables +// extension loading by default, but refusing it here means a later connection +// option cannot turn it into loading an arbitrary shared library. +const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i; +// What better-sqlite3 throws from prepare() for a statement cut off before its end: +// in an exec() script, a trigger body whose semicolon split it. +const INCOMPLETE_INPUT = 'incomplete input'; + +/** + * The limits one store enforces. Overridable for tests; production uses + * `PLUGIN_DATA_CAPS`. + * + * The time budget: better-sqlite3 has no interrupt or progress handler, so it + * is checked between the rows a statement yields and between the statements + * of a batch or a script. A scan that keeps producing rows, a select whose + * every row is expensive, or a script of many statements stops at the first + * row or statement past the budget instead of holding the host loop. One + * statement on its own still runs to its end (a write such as + * `INSERT ... SELECT` over a cartesian product) or to its first row (an + * aggregate over one); only moving plugin SQL off the host thread can bound + * that, and the plugin-sdk README says so. `migrate` is left unbudgeted on + * purpose: a migration stopped part of the way would fail the same way on + * every later start, leaving a plugin that cannot update its schema at all. + */ +interface SqlitePluginDataLimits { + maxRows: number; + timeBudgetMs: number; + /** + * The most the file may take on disk. better-sqlite3 runs in the host + * process, so an unbounded plugin file would fill the data volume TREK's + * own database lives on, and a scan over it would stall the event loop. + * `max_page_count` caps the file: a write past it fails with SQLITE_FULL, + * contained to the plugin. + */ + quotaBytes: number; + /** The clock the time budget reads. */ + now: () => number; +} + +const DEFAULT_LIMITS: SqlitePluginDataLimits = { + maxRows: PLUGIN_DATA_CAPS.maxRows, + timeBudgetMs: PLUGIN_DATA_CAPS.timeBudgetMs, + quotaBytes: PLUGIN_DATA_CAPS.quotaBytes, + now: () => Date.now(), +}; + +/** + * Every open store, so a backup can fold each one's WAL back into its file or + * snapshot it before archiving: the host keeps the handles open, so a plain + * copy of the files would miss the commits still in the `-wal` sidecar. + */ +const openStores = new Set(); + +/** The stores open now, for the backup (`nest/plugins/host/plugin-data.service.ts`). */ +export function openSqlitePluginDataStores(): SqlitePluginDataStore[] { + return [...openStores]; +} + +export class SqlitePluginDataStore implements PluginDataStore { + readonly engine = 'sqlite' as const; + private readonly db: Db; + private readonly limits: SqlitePluginDataLimits; + + /** Opens (and creates) the plugin `pluginId`'s database at `file`. */ + constructor( + readonly pluginId: string, + file: string, + limits: Partial = {}, + ) { + this.limits = { ...DEFAULT_LIMITS, ...limits }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + this.db = openDatabase(file); + this.db.pragma('journal_mode = WAL'); + this.db.pragma('foreign_keys = ON'); + openStores.add(this); + // The size cap is per connection and not persisted, so it is set on every open. + const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096; + this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(this.limits.quotaBytes / pageSize))}`); + // Records the applied migrations, so migrate() runs once per (plugin, id). + this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`); + } + + private guard(sql: unknown): void { + if (typeof sql !== 'string') throw new Error(PLUGIN_DATA_ERRORS.sqlNotString); + if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error(PLUGIN_DATA_ERRORS.sqlTooLong); + refuseNulSql(sql); + if (FORBIDDEN.test(sql)) throw new Error(PLUGIN_DATA_ERRORS.statementNotAllowed); + refuseTransactionControl(sql); + } + + /** + * Refuses a call once {@link close} ran, with better-sqlite3's words for a + * closed handle. A statement on a closed handle is refused by the driver + * itself; this covers the calls that would run none (an empty or + * comment-only script, an empty tx), so a closed store refuses every call, + * as on Postgres. + */ + private refuseIfClosed(): void { + if (!this.db.open) throw new Error(PLUGIN_DATA_ERRORS.closed); + } + + /** A deadline for one call: throws once the call has run past the time budget. */ + private startBudget(call: PluginDataBudgetedCall): () => void { + const { now, timeBudgetMs } = this.limits; + const startedAt = now(); + return () => { + if (now() - startedAt > timeBudgetMs) throw new Error(PLUGIN_DATA_ERRORS.overBudget(call, timeBudgetMs)); + }; + } + + /** One read statement: its rows up to the row cap, within the time budget. */ + query(sql: string, args: unknown[] = []): unknown[] { + this.guard(sql); + refuseNulValues(args); + const { maxRows } = this.limits; + const checkTime = this.startBudget('query'); + // iterate() pulls one row at a time, so a result that would grow without + // bound is stopped at the cap instead of materialized by all(), and the + // clock is read between rows. It is read only once another row has come + // back, so a result read to its end is returned, however long that took. + const rows: unknown[] = []; + for (const row of this.db.prepare(sql).iterate(...(args as never[]))) { + if (rows.length > 0) checkTime(); + rows.push(row); + if (rows.length > maxRows) throw new Error(PLUGIN_DATA_ERRORS.queryTooManyRows(maxRows)); + } + return rows; + } + + /** + * With bound args, one statement. Without, a script of any number (a small + * setup script, say), run one statement at a time within the time budget, + * checked before each statement after the first. + */ + exec(sql: string, args: unknown[] = []): { changes: number } { + this.guard(sql); + refuseNulValues(args); + if (args.length > 0) { + const info = this.db.prepare(sql).run(...(args as never[])); + return { changes: info.changes }; + } + this.refuseIfClosed(); + this.runScript(sql, this.startBudget('exec')); + return { changes: 0 }; + } + + /** + * Runs a script the way `Database#exec` does (statement after statement, each + * in autocommit, since the guard refuses a script that would open a + * transaction), with `checkTime` read before every statement but the first. + * The clock is never read after the last one, so a script whose statements + * have all run (and committed) is reported as done. A trigger body is + * gathered until SQLite stops calling it incomplete. + */ + private runScript(sql: string, checkTime: () => void): void { + let pending = ''; + let ranOne = false; + for (const piece of splitAfterSemicolons(sql)) { + pending += piece; + const stmt = this.prepareComplete(pending); + if (stmt === 'incomplete') continue; + pending = ''; + if (stmt === 'empty') continue; + if (ranOne) checkTime(); + stmt.run(); + ranOne = true; + } + // A statement still open at the end of the script: prepared once more for + // SQLite's own error, as Database#exec reports it. + if (pending !== '') this.db.prepare(pending); + } + + /** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */ + private prepareComplete(sql: string): ScriptStatement | 'incomplete' | 'empty' { + try { + return this.db.prepare(sql); + } catch (e) { + const message = e instanceof Error ? e.message : ''; + if (message === INCOMPLETE_INPUT) return 'incomplete'; + if (message === PLUGIN_DATA_ERRORS.noStatement) return 'empty'; + throw e; + } + } + + /** + * Statements that commit or roll back together, the primitive a plugin + * needs for a consistent multi-write (moving an item between two tables). + * Each op is one statement; a read (SELECT, RETURNING) yields `{ rows }`, a + * write `{ changes }`, and a read sees the batch's own earlier writes. + */ + tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } { + if (!Array.isArray(ops)) throw new Error(PLUGIN_DATA_ERRORS.txNotArray); + if (ops.length === 0) { + this.refuseIfClosed(); + return { results: [] }; + } + if (ops.length > PLUGIN_DATA_CAPS.maxTxOps) + throw new Error(PLUGIN_DATA_ERRORS.txTooManyOps(PLUGIN_DATA_CAPS.maxTxOps)); + for (const op of ops) { + this.guard(op?.sql); + refuseNulValues(op.args ?? []); + } + const { maxRows } = this.limits; + let batchRows = 0; // one row cap for the whole batch, not per statement + // One time budget for the whole batch too. A throw inside the transaction + // rolls every statement of the batch back, so running out of time is all or nothing. + const checkTime = this.startBudget('tx'); + const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => { + const results: Array<{ changes?: number; rows?: unknown[] }> = []; + for (const op of batch) { + const stmt = this.db.prepare(op.sql); + const args = (op.args ?? []) as never[]; + if (stmt.reader) { + const rows: unknown[] = []; + for (const row of stmt.iterate(...args)) { + rows.push(row); + if (++batchRows > maxRows) throw new Error(PLUGIN_DATA_ERRORS.txTooManyRows(maxRows)); + checkTime(); + } + results.push({ rows }); + } else { + results.push({ changes: stmt.run(...args).changes }); + } + checkTime(); + } + return results; + }); + return { results: run(ops) }; + } + + /** Runs a migration once, keyed by id. Running it again with the same id is a no-op. */ + migrate(id: string, sql: string): { applied: boolean } { + this.guard(sql); + refuseNulValues([id]); + const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id); + if (seen) return { applied: false }; + this.db.transaction(() => { + this.db.exec(sql); + this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now()); + })(); + return { applied: true }; + } + + /** + * Whether the handle is still open. The host reads it to notice a handle a + * terminal-failure dispose closed while it stayed cached, and opens a new + * store instead of throwing on reuse. + */ + isOpen(): boolean { + return this.db.open; + } + + /** + * Folds the WAL back into the database file (checkpoint TRUNCATE), so a copy + * of the file is a complete snapshot. No-op on a closed handle. + */ + checkpoint(): void { + if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)'); + } + + /** + * Writes a consistent copy of the database to `destPath` with VACUUM INTO. + * Unlike a file copy it folds in the WAL and reads one point in time, so the + * copy is right even while the plugin writes: no torn page, no separate + * `-wal` to keep in step. It runs the host's own SQL on the host's own + * handle, not plugin SQL, so the plugin guard does not apply. + */ + snapshotInto(destPath: string): void { + fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target exists + this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`); + } + + close(): void { + try { + openStores.delete(this); + this.db.close(); + } catch { + /* already closed */ + } + } +} diff --git a/server/src/nest/plugins/host/sql-script.ts b/server/src/db/engine/sqlite/sql-script.ts similarity index 96% rename from server/src/nest/plugins/host/sql-script.ts rename to server/src/db/engine/sqlite/sql-script.ts index c5767ba40..b6cf2da62 100644 --- a/server/src/nest/plugins/host/sql-script.ts +++ b/server/src/db/engine/sqlite/sql-script.ts @@ -8,7 +8,7 @@ * it prepares the pieces it has gathered so far and keeps gathering while SQLite * reports them as incomplete input. * - * Pure, no imports: plugin-data.service.ts runs a plugin's exec() script through it, + * Pure, no imports: plugin-data-store.ts runs a plugin's exec() script through it, * one statement at a time, so the time budget can be checked between statements. */ export function splitAfterSemicolons(sql: string): string[] { diff --git a/server/src/db/nul-free.ts b/server/src/db/nul-free.ts index b88580365..aea976508 100644 --- a/server/src/db/nul-free.ts +++ b/server/src/db/nul-free.ts @@ -26,10 +26,11 @@ import type { AbortableOperationOptions, CompiledQuery, DatabaseConnection, Dial * (`--drop-nul`) and lists each value it changed; by default it refuses * such a value, since the move changes nothing silently. * - * A plugin's own data (`db:own`) is not TREK's to change: its SQLite file - * (`nest/plugins/host/plugin-data.service.ts`) binds a NUL exactly, so keys - * it stored with one stay findable, and its schema on Postgres refuses an - * argument holding one with an error (`engine/postgres/plugin-data-checks.ts`). + * A plugin's own data (`db:own`) is not TREK's to change: the stores of both + * engines (`engine/sqlite/plugin-data-store.ts`, + * `engine/postgres/plugin-data-store.ts`) refuse SQL, an argument or a + * migration id holding one with the same error (`engine/plugin-data-contract.ts`) + * instead of dropping it, so a key never changes meaning. * * A filter is stripped as well as a write, so a lookup finds what the write * stored. Text that spells the character as the six characters `\u0000` diff --git a/server/src/nest-rpc/rpc-kit/README.md b/server/src/nest-rpc/rpc-kit/README.md index 4351e366a..c3233cb10 100644 --- a/server/src/nest-rpc/rpc-kit/README.md +++ b/server/src/nest-rpc/rpc-kit/README.md @@ -51,7 +51,7 @@ fallback that silently disables all checking if someone forgets to augment. `rpc-kit` needs none of that today. Extracting it would mean: 1. `PluginRpcContext.data` becomes a generic parameter instead of importing - `PluginDataDb`. + `PluginDataStore`. 2. The four envelope type imports (`KnownMethod`, `MethodPermission`, `HookKey`, `HookPermission`) become empty-interface augmentation points, with `AssertExact` guards in the server proving the augmentation still matches the tables. diff --git a/server/src/nest/plugins/host/plugin-data.backend.ts b/server/src/nest/plugins/host/plugin-data.backend.ts index abad4f1c5..0ee09e27e 100644 --- a/server/src/nest/plugins/host/plugin-data.backend.ts +++ b/server/src/nest/plugins/host/plugin-data.backend.ts @@ -10,10 +10,11 @@ import { poolClients, PostgresPluginDataStore, } from '../../../db/engine/postgres/plugin-data-store'; +import { SqlitePluginDataStore } from '../../../db/engine/sqlite/plugin-data-store'; import { logWarn } from '../../audit/audit-log.logger'; import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../../database/database-engine.module'; -import { pluginDataDir } from '../paths'; -import { PluginDataDb, removePluginData } from './plugin-data.service'; +import { pluginDataDir, pluginDbFile } from '../paths'; +import { removePluginData } from './plugin-data.service'; import { MikroORM } from '@mikro-orm/core'; import type { AbstractSqlConnection } from '@mikro-orm/sql'; import { Inject, Injectable, type OnApplicationBootstrap, type Provider } from '@nestjs/common'; @@ -39,13 +40,13 @@ export interface PluginDataBackend { export const PLUGIN_DATA = Symbol('PLUGIN_DATA'); -/** Each plugin's own SQLite file under the plugin data directory (`plugin-data.service.ts`). */ +/** Each plugin's own SQLite file under the plugin data directory (`db/engine/sqlite/plugin-data-store.ts`). */ @Injectable() export class SqlitePluginDataBackend implements PluginDataBackend { readonly engine = 'sqlite' as const; open(pluginId: string): PluginDataStore { - return new PluginDataDb(pluginId); + return new SqlitePluginDataStore(pluginId, pluginDbFile(pluginId)); } async erase(pluginId: string): Promise { diff --git a/server/src/nest/plugins/host/plugin-data.service.ts b/server/src/nest/plugins/host/plugin-data.service.ts index 8a9813b88..486223916 100644 --- a/server/src/nest/plugins/host/plugin-data.service.ts +++ b/server/src/nest/plugins/host/plugin-data.service.ts @@ -1,313 +1,31 @@ -import { openDatabase } from '../../../db/connection'; -import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data'; -import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths'; -import { splitAfterSemicolons } from './sql-script'; +import type { PluginDataStore } from '../../../db/engine/plugin-data'; +import { openSqlitePluginDataStores, type SqlitePluginDataStore } from '../../../db/engine/sqlite/plugin-data-store'; +import { pluginDataDir, pluginsDataRoot } from '../paths'; -import type { Database as Db, Statement } from 'better-sqlite3'; import fs from 'node:fs'; import path from 'node:path'; // The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs). export type { PluginDataStore }; -/** - * A plugin's own sqlite database (#plugins, db:own): the SQLite side of the - * `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a - * schema per plugin, `db/engine/postgres/plugin-data-store.ts`). The HOST owns the handle; - * the plugin child never gets a path or a connection — it can only reach this - * through RPC (db.exec / db.query / db.migrate). Because it is a SEPARATE FILE, - * containment is a filesystem fact: the plugin physically cannot read trek.db, - * and we don't have to police table-name prefixes in its SQL. - * - * A thin guard still rejects statements that would let a plugin escape its file - * (ATTACH another db, VACUUM INTO elsewhere, PRAGMA fiddling) or DoS via - * oversize SQL, every read is held to a row cap, and query(), tx() and an exec() - * script are held to a time budget (below). - * - * Bound arguments reach SQLite exactly as the plugin sent them, U+0000 (NUL) - * included: the data is the plugin's own, so a key it stored with a NUL is - * found again by the same key, and `'a\0bc'` and `'ab\0c'` stay two keys. TREK - * drops NUL only from its own data (`db/nul-free.ts`); the Postgres store, - * whose text cannot hold the character, refuses such an argument instead. +/* + * The plugin data directories on disk: erasing one, and copying them all for + * a backup. On SQLite each holds the plugin's database file, which the + * SQLite store opens (`db/engine/sqlite/plugin-data-store.ts`); on Postgres + * it holds only the files a plugin keeps beside its schema. */ -const MAX_SQL_LENGTH = PLUGIN_DATA_CAPS.maxSqlLength; -// RECURSIVE is the one construct that generates unbounded rows/CPU independent of -// the (capped) data size — a `WITH RECURSIVE …` can spin the synchronous host -// forever even with an empty database, which neither the size quota nor the -// result-row cap can stop (an aggregate over it never yields a first row). Refuse -// it outright; the row/size caps below bound everything else. -// load_extension is included as defense-in-depth: better-sqlite3 disables -// extension loading by default (so it's inert today), but banning it in the guard -// means a future connection-option slip can't turn it into an arbitrary-.so RCE. -const FORBIDDEN = /\b(ATTACH|DETACH|VACUUM|PRAGMA|RECURSIVE|LOAD_EXTENSION)\b/i; -// Transaction-control keywords, matched only at statement start (so CASE…END and -// identifiers are unaffected). Refused inside db.tx() so a plugin can't COMMIT the -// batch's earlier writes and then have the wrapper report failure — breaking atomicity. -const TX_CONTROL = /^\s*(BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|END)\b/i; -// Per-plugin on-disk quota. better-sqlite3 is synchronous and runs in the HOST -// process, so an unbounded plugin DB is both a disk-exhaustion DoS on the shared -// trek.db volume and (via a huge scan) an event-loop stall. max_page_count caps -// the file (writes past it fail SQLITE_FULL, contained to the plugin) and bounds -// the worst-case scan cost. Result sets are additionally row-capped below so a -// recursive CTE / cartesian product can't materialize an unbounded array. -const QUOTA_BYTES = PLUGIN_DATA_CAPS.quotaBytes; -const MAX_ROWS = PLUGIN_DATA_CAPS.maxRows; -// Cap statements per atomic batch so a single tx() can't monopolise the synchronous -// host — generous for real write batches, far below anything abusive. -const MAX_TX_OPS = PLUGIN_DATA_CAPS.maxTxOps; -// Wall-clock budget for one query(), one whole tx() batch or one exec() script. -// better-sqlite3 has no interrupt or progress handler, so the budget is checked -// between the rows a statement yields and between the statements of a batch or a -// script: a scan that keeps producing rows, a select whose every row is expensive, or -// a script of many statements stops at the first row or statement past the budget -// instead of holding the host loop. One statement on its own still runs to the end -// (a write such as `INSERT … SELECT` over a cartesian product) or to its first row -// (an aggregate over one); only moving plugin SQL off the host thread can bound that, -// and the plugin-sdk README says so. migrate() is left unbudgeted on purpose: a -// migration is stopped part of the way only to fail the same way on every later -// start, which leaves a plugin that cannot update its schema at all. -const TIME_BUDGET_MS = PLUGIN_DATA_CAPS.timeBudgetMs; -// What better-sqlite3 throws from prepare() for a statement cut off before its end: -// in an exec() script, a trigger body whose semicolon split it. -const INCOMPLETE_INPUT = 'incomplete input'; -// And for a piece that holds no statement at all, only whitespace or comments. -const NO_STATEMENT = 'The supplied SQL string contains no statements'; - -/** The caps a PluginDataDb enforces. Overridable for tests; production uses the defaults. */ -export interface PluginDataLimits { - maxRows: number; - timeBudgetMs: number; - /** The clock the time budget reads. */ - now: () => number; -} - -const DEFAULT_LIMITS: PluginDataLimits = { maxRows: MAX_ROWS, timeBudgetMs: TIME_BUDGET_MS, now: () => Date.now() }; - -// Every live per-plugin handle, so a backup can WAL-checkpoint them before archiving -// (the host keeps these open, so their .db files would otherwise be copied with recent -// commits still stranded in the -wal sidecar → a stale/torn snapshot in the backup). -const openDbs = new Set(); - -/** Fold the WAL back into each open plugin.db so a subsequent file copy is a complete, - * consistent snapshot — mirrors the wal_checkpoint the core backup runs on travel.db. - * Best-effort per handle; never throws. */ +/** + * Folds the WAL back into each open plugin database so a file copy made next + * is a complete, consistent snapshot, as the core backup's checkpoint does for + * TREK's own file. Best effort per handle; never throws. + */ export function checkpointAllPluginDataDbs(): void { - for (const d of openDbs) { + for (const store of openSqlitePluginDataStores()) { try { - d.checkpoint(); + store.checkpoint(); } catch { - /* a busy/closed handle is skipped, not fatal */ - } - } -} - -export class PluginDataDb implements PluginDataStore { - private db: Db; - readonly pluginId: string; - readonly engine = 'sqlite' as const; - private readonly limits: PluginDataLimits; - - constructor(pluginId: string, limits: Partial = {}) { - this.pluginId = pluginId; - this.limits = { ...DEFAULT_LIMITS, ...limits }; - fs.mkdirSync(pluginDataDir(pluginId), { recursive: true }); - this.db = openDatabase(pluginDbFile(pluginId)); - this.db.pragma('journal_mode = WAL'); - this.db.pragma('foreign_keys = ON'); - openDbs.add(this); - // Cap the file size (per-connection; not persisted, so set on every open). - const pageSize = Number(this.db.pragma('page_size', { simple: true })) || 4096; - this.db.pragma(`max_page_count = ${Math.max(1, Math.floor(QUOTA_BYTES / pageSize))}`); - // Track applied migrations so db.migrate is idempotent per (plugin, id). - this.db.exec(`CREATE TABLE IF NOT EXISTS _plugin_migrations (id TEXT PRIMARY KEY, applied_at INTEGER)`); - } - - private guard(sql: string): void { - if (typeof sql !== 'string') throw new Error('sql must be a string'); - if (sql.length > MAX_SQL_LENGTH) throw new Error('sql too long'); - if (FORBIDDEN.test(sql)) throw new Error('statement type not allowed for plugin databases'); - } - - /** A deadline for one call: throws once the call has run past the time budget. */ - private startBudget(what: string): () => void { - const { now, timeBudgetMs } = this.limits; - const startedAt = now(); - return () => { - if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`); - }; - } - - /** Read query: returns rows up to the row cap, within the time budget. Single statement only. */ - query(sql: string, args: unknown[] = []): unknown[] { - this.guard(sql); - const { maxRows } = this.limits; - const checkTime = this.startBudget('query'); - // iterate() pulls one row at a time, so a recursive CTE that would yield - // unboundedly is halted at the cap instead of materializing via all(), and the - // clock is read between rows. It is read only once another row has come back, so - // a result that has been read to its end is returned, however long that took. - const rows: unknown[] = []; - for (const row of this.db.prepare(sql).iterate(...(args as never[]))) { - if (rows.length > 0) checkTime(); - rows.push(row); - if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`); - } - return rows; - } - - /** - * Write statement(s). With bound args, exec() runs one statement. Without, it runs a - * script of any number (e.g. a small setup script), one statement at a time and - * within the time budget, checked before each statement after the first. - */ - exec(sql: string, args: unknown[] = []): { changes: number } { - this.guard(sql); - if (args.length > 0) { - const info = this.db.prepare(sql).run(...(args as never[])); - return { changes: info.changes }; - } - this.runScript(sql, this.startBudget('exec')); - return { changes: 0 }; - } - - /** - * Runs a script the way `Database#exec` did (statement after statement, each in - * autocommit unless the script opens a transaction), with `checkTime` read before - * every statement but the first. The clock is never read after the last one, so a - * script whose statements have all run (and committed) is reported as done. A - * trigger body is gathered until SQLite stops calling it incomplete. A script that - * runs out of time inside a transaction it opened itself has that transaction rolled - * back, so the connection is not left holding it. - */ - private runScript(sql: string, checkTime: () => void): void { - const openedNoTransaction = !this.db.inTransaction; - let pending = ''; - let ranOne = false; - for (const piece of splitAfterSemicolons(sql)) { - pending += piece; - const stmt = this.prepareComplete(pending); - if (stmt === 'incomplete') continue; - pending = ''; - if (stmt === 'empty') continue; - if (ranOne) { - try { - checkTime(); - } catch (e) { - if (openedNoTransaction && this.db.inTransaction) this.db.prepare('ROLLBACK').run(); - throw e; - } - } - stmt.run(); - ranOne = true; - } - // A statement still open at the end of the script: prepare it once more for - // SQLite's own error, as Database#exec reports it. - if (pending !== '') this.db.prepare(pending); - } - - /** `sql` prepared, or why it cannot be yet: cut off before its end, or no statement at all. */ - private prepareComplete(sql: string): Statement | 'incomplete' | 'empty' { - try { - return this.db.prepare(sql); - } catch (e) { - const message = e instanceof Error ? e.message : ''; - if (message === INCOMPLETE_INPUT) return 'incomplete'; - if (message === NO_STATEMENT) return 'empty'; - throw e; - } - } - - /** - * Atomic batch on the plugin's OWN db: every op runs in a single transaction, so - * they all commit or all roll back — the primitive a plugin needs for a consistent - * multi-write (e.g. move an item between two tables). Each op is ONE statement; - * a read (SELECT/RETURNING) yields `{ rows }`, a write yields `{ changes }`, and - * reads within the batch see the batch's own earlier writes (read-modify-write). - */ - tx(ops: Array<{ sql: string; args?: unknown[] }>): { results: Array<{ changes?: number; rows?: unknown[] }> } { - if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }'); - if (ops.length === 0) return { results: [] }; - if (ops.length > MAX_TX_OPS) throw new Error(`tx allows at most ${MAX_TX_OPS} statements`); - for (const op of ops) { - this.guard(op?.sql); - // Reject transaction-control statements: a raw COMMIT/ROLLBACK inside the batch - // would break atomicity — it commits the earlier writes even though the wrapper - // then reports the tx as failed. Strip any LEADING comments/whitespace first so a - // `/* */COMMIT` or `-- x\nCOMMIT` can't slip past the start-anchored check; these - // keywords are only valid at statement start, so CASE ... END is unaffected. - const head = String(op?.sql ?? '').replace(/^(?:\s|--[^\n]*\n?|\/\*[\s\S]*?\*\/)*/, ''); - if (TX_CONTROL.test(head)) throw new Error('transaction-control statements are not allowed inside tx()'); - } - const { maxRows } = this.limits; - let batchRows = 0; // one row budget for the WHOLE batch, not per statement - // One time budget for the whole batch too. A throw inside the transaction rolls - // every statement of the batch back, so running out of time is all-or-nothing. - const checkTime = this.startBudget('tx'); - const run = this.db.transaction((batch: Array<{ sql: string; args?: unknown[] }>) => { - const results: Array<{ changes?: number; rows?: unknown[] }> = []; - for (const op of batch) { - const stmt = this.db.prepare(op.sql); - const args = (op.args ?? []) as never[]; - if (stmt.reader) { - const rows: unknown[] = []; - for (const row of stmt.iterate(...args)) { - rows.push(row); - if (++batchRows > maxRows) throw new Error(`tx returned more than ${maxRows} rows in total`); - checkTime(); - } - results.push({ rows }); - } else { - results.push({ changes: stmt.run(...args).changes }); - } - checkTime(); - } - return results; - }); - return { results: run(ops) }; - } - - /** Run a migration once, keyed by id. Re-running with the same id is a no-op. */ - migrate(id: string, sql: string): { applied: boolean } { - this.guard(sql); - const seen = this.db.prepare('SELECT 1 FROM _plugin_migrations WHERE id = ?').get(id); - if (seen) return { applied: false }; - this.db.transaction(() => { - this.db.exec(sql); - this.db.prepare('INSERT INTO _plugin_migrations (id, applied_at) VALUES (?, ?)').run(id, Date.now()); - })(); - return { applied: true }; - } - - /** Whether the underlying sqlite handle is still open (better-sqlite3 `.open`). - * The host uses this to detect a handle closed by a terminal-failure dispose that - * left the instance cached, so it can recreate it instead of throwing on reuse. */ - isOpen(): boolean { - return this.db.open; - } - - /** Fold the WAL back into the main db file (checkpoint TRUNCATE) so a file-level copy - * is a complete snapshot. No-op on a closed handle. */ - checkpoint(): void { - if (this.db.open) this.db.pragma('wal_checkpoint(TRUNCATE)'); - } - - /** Write a fully-consistent copy of this DB to `destPath` via VACUUM INTO. Unlike a - * file copy it folds in the WAL and reads a point-in-time snapshot, so the result is - * correct even while the plugin is writing — no torn page, no separate -wal to keep in - * sync. This is a host op on the host's own handle, not plugin SQL, so it bypasses the - * FORBIDDEN guard by design. */ - snapshotInto(destPath: string): void { - fs.rmSync(destPath, { force: true }); // VACUUM INTO fails if the target already exists - this.db.exec(`VACUUM INTO '${destPath.replaceAll("'", "''")}'`); - } - - close(): void { - try { - openDbs.delete(this); - this.db.close(); - } catch { - /* already closed */ + /* a busy or closed handle is skipped, not fatal */ } } } @@ -321,7 +39,7 @@ export function removePluginData(pluginId: string): void { * Copy every plugin's data dir into `destRoot` as a CONSISTENT snapshot, for a backup to * archive instead of the live tree. An open plugin.db is captured with VACUUM INTO (safe * under concurrent writes); a plugin with no live handle is copied as-is (no writer). The - * -wal/-shm sidecars are never copied — the snapshot folds them in, and copying them out + * -wal/-shm sidecars are never copied: the snapshot folds them in, and copying them out * of step with the .db is exactly what produced torn/corrupt restores when the archiver * read the live files lazily while a plugin kept writing. Blobs and any other files a * plugin wrote to its dir are copied verbatim. Best-effort per file; never throws. @@ -329,8 +47,8 @@ export function removePluginData(pluginId: string): void { export function snapshotAllPluginDataDbs(destRoot: string): void { const root = pluginsDataRoot(); if (!fs.existsSync(root)) return; - const openById = new Map(); - for (const d of openDbs) if (d.isOpen()) openById.set(d.pluginId, d); + const openById = new Map(); + for (const store of openSqlitePluginDataStores()) if (store.isOpen()) openById.set(store.pluginId, store); for (const entry of fs.readdirSync(root, { withFileTypes: true })) { if (!entry.isDirectory()) continue; const srcDir = path.join(root, entry.name); @@ -339,7 +57,7 @@ export function snapshotAllPluginDataDbs(destRoot: string): void { const open = openById.get(entry.name); // Handle the live db up front so we know whether its WAL got folded in. If both // the VACUUM INTO snapshot and the checkpoint fail, the -wal/-shm are NOT folded, - // so they must be copied alongside the .db — a .db stripped of an un-checkpointed + // so they must be copied alongside the .db: a .db stripped of an un-checkpointed // WAL loses committed transactions, whereas the .db + its WAL is a recoverable set. let foldedIn = false; if (open) { @@ -351,18 +69,18 @@ export function snapshotAllPluginDataDbs(destRoot: string): void { open.checkpoint(); foldedIn = true; } catch { - /* WAL not folded — keep sidecars */ + /* WAL not folded: keep the sidecars */ } try { fs.copyFileSync(path.join(srcDir, 'plugin.db'), path.join(destDir, 'plugin.db')); } catch { - /* unreadable live db — best effort */ + /* unreadable live db: best effort */ } } } for (const f of fs.readdirSync(srcDir, { withFileTypes: true })) { if (f.name === 'plugin.db' && open) continue; // already snapshotted above - // Skip the .db sidecars only when the live handle's WAL was folded in — VACUUM + // Skip the .db sidecars only when the live handle's WAL was folded in: VACUUM // INTO / checkpoint absorbs them, and copying them out of step with a live writer // is what produced torn restores. For a plugin with NO open handle there is no // writer, so the -wal/-shm are a consistent set with the .db; copy them too, or an diff --git a/server/src/nest/plugins/host/plugin-host-state.ts b/server/src/nest/plugins/host/plugin-host-state.ts index b694c9450..a7b897811 100644 --- a/server/src/nest/plugins/host/plugin-host-state.ts +++ b/server/src/nest/plugins/host/plugin-host-state.ts @@ -29,7 +29,7 @@ export function getPluginDataDb(id: string, backend: PluginDataBackend): PluginD let d = dataDbs.get(id); // A cached handle can be CLOSED without being evicted: the supervisor's terminal // failure paths (activation timeout / load-error / crash auto-disable) call - // rpcHost.dispose() → PluginDataDb.close() directly, never closePluginDataDb. A + // rpcHost.dispose() → the store's close() directly, never closePluginDataDb. A // plain admin re-enable would then reuse the closed handle and every db.* call // would throw 'database connection is not open'. Recreate when the handle is shut. if (!d || !d.isOpen()) { diff --git a/server/tests/integration/plugins/supervisor.test.ts b/server/tests/integration/plugins/supervisor.test.ts index 093a2e481..5973a9c94 100644 --- a/server/tests/integration/plugins/supervisor.test.ts +++ b/server/tests/integration/plugins/supervisor.test.ts @@ -7,10 +7,11 @@ * - disable() tears the child down. * The child runs its own process — its crash/throw can never reach this test. */ +import { SqlitePluginDataStore } from '../../../src/db/engine/sqlite/plugin-data-store'; import { createTestPluginRegistry } from '../../../src/nest-rpc/rpc-kit/testing'; -import { PluginDataDb } from '../../../src/nest/plugins/host/plugin-data.service'; import { PluginRpcHost, type HostDeps } from '../../../src/nest/plugins/host/rpc-host'; import { DbRpc } from '../../../src/nest/plugins/host/rpc/db.rpc'; +import { pluginDbFile } from '../../../src/nest/plugins/paths'; import type { PluginUserSettingsService } from '../../../src/nest/plugins/plugin-user-settings.service'; import { PluginSupervisor, @@ -51,7 +52,7 @@ function makeSupervisor( ): PluginSupervisor { const createRpcHost = (id: string, granted: ReadonlySet): PluginRpcHost => { const deps: HostDeps = { - data: new PluginDataDb(id), + data: new SqlitePluginDataStore(id, pluginDbFile(id)), callPlugin: async () => undefined, emitPluginEvent: async (event, payload) => { broadcasts.push({ id, event, payload }); @@ -74,7 +75,7 @@ function makeSupervisor( // through PluginSupervisor.onMessage's 'req' branch over real child-process // IPC, which now THROWS without a resolveOrm thunk rather than dispatching // unwrapped — so this double needs a real (if otherwise unused) ORM, even - // though PluginDataDb/DbRpc here are raw better-sqlite3, not MikroORM. + // though the plugin store and DbRpc here are raw better-sqlite3, not MikroORM. return new PluginSupervisor(createRpcHost, hooks, tuning, () => ormHandle.orm); } diff --git a/server/tests/postgres/plugin-data.test.ts b/server/tests/postgres/plugin-data.test.ts index cdb9f70a7..48f270fd4 100644 --- a/server/tests/postgres/plugin-data.test.ts +++ b/server/tests/postgres/plugin-data.test.ts @@ -1,5 +1,5 @@ /** - * PGPLUG-001..021 — a plugin's own data (`db:own`) on Postgres, against the + * PGPLUG-001..024: a plugin's own data (`db:own`) on Postgres, against the * real app on the lane's database: each plugin gets a schema of its own and no * SQLite file, its SQL keeps the SQLite store's semantics (migrate once per * id, a script's statements committed one by one, an atomic tx, the row cap, @@ -16,6 +16,7 @@ */ import type { DbSettings } from '../../src/app-config'; import { buildApp } from '../../src/bootstrap'; +import { PLUGIN_SQL_NUL_REFUSAL, PLUGIN_VALUE_NUL_REFUSAL } from '../../src/db/engine/plugin-data-contract'; import { PRIVILEGED_ROLE_PLUGIN_REFUSAL, type PluginRoleGate } from '../../src/db/engine/postgres/plugin-data-checks'; import { type ConnectPluginClient, @@ -24,7 +25,12 @@ import { poolClients, PostgresPluginDataStore, } from '../../src/db/engine/postgres/plugin-data-store'; -import { catalogFunctionNames, COLUMN_NAMES } from '../../src/db/engine/postgres/plugin-sql-functions'; +import { PLUGIN_SESSION_SETTINGS } from '../../src/db/engine/postgres/plugin-session'; +import { + catalogFunctionNames, + COLUMN_NAMES, + pluginFunctionNames, +} from '../../src/db/engine/postgres/plugin-sql-functions'; import { TrekPostgreSqlDriver } from '../../src/db/engine/postgres/postgres-driver'; import { putTempSchemaLast } from '../../src/db/engine/postgres/session-search-path'; import { DATABASE_SETTINGS } from '../../src/nest/database/database-engine.module'; @@ -375,7 +381,7 @@ describe('plugin data on Postgres', () => { closePluginDataDb('pg-rpc'); }); - it("PGPLUG-009: the review's bypasses are refused before they run: SQL hidden behind a CR, and SELECT ... INTO TEMP", async () => { + it('PGPLUG-009: bypasses an earlier guard let through are refused before they run: SQL hidden behind a CR, and SELECT ... INTO TEMP', async () => { const store = data.open('pg-bypass'); await store.migrate('001', 'CREATE TABLE notes (id bigserial PRIMARY KEY, body text NOT NULL)'); const reads = [ @@ -525,7 +531,7 @@ describe('plugin data on Postgres', () => { expect(schemas).not.toContain('trek_plugin_pg_files'); }); - // The re-review's escape: an unqualified ALTER TYPE renamed pg_catalog's types for the whole database. + // An unqualified ALTER TYPE could rename pg_catalog's types for the whole database. const CATALOG_TARGETS = [ 'alter type int4 rename to int4x', 'alter type text rename to evil', @@ -679,12 +685,12 @@ describe('plugin data on Postgres', () => { it('PGPLUG-020: a NUL in an argument or in the SQL is refused with the reason, and nothing is stored', async () => { const store = data.open('pg-nul'); await store.migrate('001', 'CREATE TABLE kv (k text PRIMARY KEY)'); - const reason = /U\+0000 \(NUL\), which Postgres text cannot store/; + const reason = PLUGIN_VALUE_NUL_REFUSAL; await expect(store.exec('INSERT INTO kv VALUES ($1)', ['a\u0000b'])).rejects.toThrow(reason); await expect(store.query('SELECT k FROM kv WHERE k = $1', ['ab\u0000'])).rejects.toThrow(reason); await expect(store.tx([{ sql: 'INSERT INTO kv VALUES ($1)', args: [['x', '\u0000']] }])).rejects.toThrow(reason); await expect(store.migrate('002\u0000', 'CREATE TABLE more (x int)')).rejects.toThrow(reason); - await expect(store.exec("INSERT INTO kv VALUES ('a\u0000b')")).rejects.toThrow(/U\+0000 \(NUL\)/); + await expect(store.exec("INSERT INTO kv VALUES ('a\u0000b')")).rejects.toThrow(PLUGIN_SQL_NUL_REFUSAL); expect(await store.query('SELECT count(*) AS n FROM kv')).toEqual([{ n: 0 }]); // Binary data with a zero byte goes to bytea. await store.migrate('003', 'CREATE TABLE raw (b bytea)'); @@ -758,4 +764,264 @@ describe('plugin data on Postgres', () => { ]); await asSuper('comment on table users is null'); }); + + it('PGPLUG-022: no allowed function looks an object up by a name, and system columns and transaction ids stay out', async () => { + const names = [...pluginFunctionNames()].map((name) => `'${name}'`).join(', '); + const lookups = await asSuper<{ f: string }>( + 'with lookup as (select t.oid from pg_type t where t.typnamespace = ' + + "'pg_catalog'::regnamespace and (t.typname ~ '^_?reg' or t.typname in ('aclitem', '_aclitem'))) " + + 'select p.oid::regprocedure::text as f from pg_proc p ' + + `where p.pronamespace = 'pg_catalog'::regnamespace and p.proname in (${names}) ` + + 'and (p.prorettype in (select oid from lookup) or exists (select 1 from ' + + 'unnest(coalesce(p.proallargtypes, p.proargtypes::oid[])) a(t) where a.t in (select oid from lookup))) ' + + 'order by 1', + ); + expect( + lookups.map((row) => row.f), + 'These functions of server/assets/postgres/plugin-functions.txt take or return a type that looks an ' + + 'object up by a name (a reg* type or aclitem): Postgres turns a text argument into a catalog lookup, so ' + + 'the call tells a plugin which schemas and objects exist. Remove each name from the allowlist.', + ).toEqual([]); + // min(xid8) and max(xid8) return what they are given; a function that makes a transaction id + // out of anything else is the leak. + const xids = await asSuper<{ f: string }>( + "with xid as (select t.oid from pg_type t where t.typname in ('xid', 'xid8', '_xid', '_xid8')) " + + 'select p.oid::regprocedure::text as f from pg_proc p ' + + `where p.pronamespace = 'pg_catalog'::regnamespace and p.proname in (${names}) ` + + 'and p.prorettype in (select oid from xid) and not exists (select 1 from ' + + 'unnest(coalesce(p.proallargtypes, p.proargtypes::oid[])) a(t) where a.t in (select oid from xid)) ' + + 'order by 1', + ); + expect( + xids.map((row) => row.f), + 'These functions of server/assets/postgres/plugin-functions.txt make a transaction id out of other ' + + 'values: age() of one is how many transactions the server ran, and the guard keeps xid values out only by ' + + "refusing the type's name and the system columns. Remove each name from the allowlist.", + ).toEqual([]); + + const stores = [data.open('pg-lookup'), superuserStore('pg-lookup-su')]; + await Promise.all( + stores.map(async (store) => { + await store.migrate('001', 'CREATE TABLE t (id bigserial PRIMARY KEY, tags text[])'); + await store.exec("INSERT INTO t (tags) VALUES (array['a', 'b'])"); + await Promise.all( + [ + "select to_tsvector('trek_plugin_other.cfg', 'a')", + "select to_tsvector('public.x', 'a')", + "select plainto_tsquery('english', 'a')", + "select ts_headline('a', 'a'::tsquery)", + ].map((sql) => expect(store.query(sql), sql).rejects.toThrow(/a plugin may not call/)), + ); + await expect(store.query('select websearch_to_tsquery($1, $2)', ['english', 'a'])).rejects.toThrow( + /a plugin may not call/, + ); + await Promise.all( + [ + 'select xmin from t', + 'select t.xmax, t.cmin, t.cmax from t', + 'select ctid from t', + 'select "tableoid" from t', + "update t set tags = '{}' where xmin = '1'", + ].map((sql) => expect(store.query(sql), sql).rejects.toThrow(/reads a Postgres system column/)), + ); + await Promise.all( + [ + "select age('3'::xid)", + "select cast('3' as xid8)", + "select age(xid('3'))", + "select age(u.xid) from unnest(array['3']) u", + 'select age(x.a) from json_to_record(\'{"a": "3"}\') as x(a xid)', + ].map((sql) => expect(store.query(sql), sql).rejects.toThrow(/stands for a transaction id type/)), + ); + await expect(store.query('select age($1.xid)', ['3'])).rejects.toThrow(/stands for a transaction id type/); + await Promise.all( + [ + `select 'a' collate "de_DE.utf8"`, + `select 'a' collate "und-x-icu"`, + 'create index i on t using bloom (id)', + 'create table c (x int) using columnar', + ].map((sql) => expect(store.query(sql), sql).rejects.toThrow(/a collation other than|built-in access/)), + ); + // A value of the plugin's own may carry a system column's name, and a column of its own be called xid. + expect( + await store.query( + 'SELECT min(id) AS xmin, max(id) AS xmax FROM t UNION ALL ' + + 'SELECT * FROM (WITH b(xmin, xmax) AS (SELECT 5, 6) SELECT * FROM b) AS b(cmin, ctid)', + ), + ).toEqual([ + { xmin: 1, xmax: 1 }, + { xmin: 5, xmax: 6 }, + ]); + await store.migrate( + '002', + 'CREATE TABLE pay (id bigserial PRIMARY KEY, xid text NOT NULL UNIQUE); ' + + 'CREATE INDEX pay_xid ON pay USING btree (xid COLLATE "C")', + ); + await store.exec('INSERT INTO pay (xid) VALUES ($1) ON CONFLICT (xid) DO NOTHING', ['b']); + await store.exec("INSERT INTO pay (xid) VALUES ('a'), ('B')"); + expect(await store.query('SELECT xid FROM pay WHERE xid <> $1 ORDER BY xid COLLATE "C"', ['zzz'])).toEqual([ + { xid: 'B' }, + { xid: 'a' }, + { xid: 'b' }, + ]); + // Text search on vectors a plugin builds itself needs no configuration and still works. + expect( + await store.query( + "SELECT ts_rank(array_to_tsvector(tags), 'a & b'::tsquery) > 0 AS hit, " + + "array_to_tsvector(tags) @@ 'b'::tsquery AS b FROM t", + ), + ).toEqual([{ hit: true, b: true }]); + }), + ); + }); + + /** + * The settings a client may change in the categories that decide how values + * print and parse and how statements behave, that a plugin transaction + * leaves as the server has them, with the reason each cannot change what a + * plugin reads. Every other one must be pinned in PLUGIN_SESSION_SETTINGS. + */ + const UNPINNED: Record = { + client_encoding: 'pg sets UTF8 when it connects, and a plugin cannot SET it', + search_path: "set by the store per call to the plugin's schema", + statement_timeout: "set by the store per call to the call's time budget", + check_function_bodies: 'a plugin cannot create functions', + client_min_messages: 'notices never reach the plugin', + createrole_self_grant: 'a plugin cannot create roles', + icu_validation_level: 'a plugin cannot create collations', + row_security: 'a plugin cannot create policies', + local_preload_libraries: 'read when a connection starts', + default_table_access_method: 'where rows are stored, not what a query answers', + default_tablespace: 'where rows are stored, not what a query answers', + temp_tablespaces: 'where rows are stored, not what a query answers', + default_toast_compression: 'how rows are stored, not what a query answers', + gin_pending_list_limit: 'when an index is maintained, not what it finds', + restrict_nonsystem_relation_kind: 'TREK sets no restriction a plugin could see', + synchronize_seqscans: 'rows without ORDER BY have no order on Postgres in any case', + escape_string_warning: 'a warning, never an answer', + default_transaction_deferrable: "the transaction is TREK's", + default_transaction_isolation: "the transaction is TREK's", + default_transaction_read_only: "the transaction is TREK's", + transaction_deferrable: "the transaction is TREK's", + transaction_isolation: "the transaction is TREK's", + transaction_read_only: "the transaction is TREK's", + transaction_timeout: "the transaction is TREK's", + idle_in_transaction_session_timeout: "the connection is TREK's", + idle_session_timeout: "the connection is TREK's", + lock_timeout: "the connection is TREK's", + vacuum_failsafe_age: 'maintenance, not an answer', + vacuum_freeze_min_age: 'maintenance, not an answer', + vacuum_freeze_table_age: 'maintenance, not an answer', + vacuum_multixact_failsafe_age: 'maintenance, not an answer', + vacuum_multixact_freeze_min_age: 'maintenance, not an answer', + vacuum_multixact_freeze_table_age: 'maintenance, not an answer', + }; + + it('PGPLUG-023: a plugin reads values formatted the same on every server, and its settings end with its call', async () => { + const settings = await asSuper<{ name: string }>( + "select name from pg_settings where context = 'user' and (category like 'Client Connection Defaults%' " + + "or category like 'Version and Platform Compatibility%') order by name", + ); + const known = new Set(settings.map((row) => row.name)); + expect( + settings.map((row) => row.name).filter((name) => !(name in PLUGIN_SESSION_SETTINGS) && !(name in UNPINNED)), + 'These settings of this Postgres may change how a value prints or a statement behaves. Pin each in ' + + 'PLUGIN_SESSION_SETTINGS (server/src/db/engine/postgres/plugin-session.ts) to its built-in default, or add ' + + 'it to UNPINNED in this test with the reason it cannot change what a plugin reads.', + ).toEqual([]); + expect(Object.keys(PLUGIN_SESSION_SETTINGS).filter((name) => !known.has(name))).toEqual([]); + + const store = data.open('pg-settings'); + await store.migrate('001', 'CREATE TABLE notes (id bigserial PRIMARY KEY)'); + // A server set up far from the defaults: the session of the very client the plugin call uses. + const odd = [ + "set timezone to 'Pacific/Chatham'", + "set datestyle to 'German, DMY'", + "set intervalstyle to 'iso_8601'", + "set lc_monetary to 'en_US.utf8'", + "set lc_numeric to 'en_US.utf8'", + "set lc_time to 'en_US.utf8'", + 'set extra_float_digits to 0', + "set bytea_output to 'escape'", + "set default_text_search_config to 'pg_catalog.english'", + 'set quote_all_identifiers to on', + 'set transform_null_equals to on', + 'set array_nulls to off', + ]; + const read = + "SELECT '2024-01-02 03:04:05+00'::timestamptz::text AS ts, '2024-01-02'::date::text AS d, " + + "'1 day 02:00'::interval::text AS i, 12.5::money::text AS m, to_char(1234.5, 'FM9G999D9') AS n, " + + "to_char('2024-03-01'::date, 'TMMonth') AS month, 0.1::float8::text AS f, '\\x01'::bytea::text AS b, " + + "quote_ident('a') AS q, 'The Cats' @@ 'cat' AS fts, (null = null) IS NULL AS nulls, " + + "('{NULL}'::text[])[1] IS NULL AS arr, date_trunc('day', '2024-01-01 23:00:00+00'::timestamptz)::text AS day"; + const pinned = { + ts: '2024-01-02 03:04:05+00', + d: '2024-01-02', + i: '1 day 02:00:00', + m: '$12.50', + n: '1,234.5', + month: 'March', + f: '0.1', + b: '\\x01', + q: 'a', + fts: false, + nulls: true, + arr: true, + day: '2024-01-01 00:00:00+00', + }; + const client = await appClients()(); + try { + await inSequence(odd, (sql) => client.query(sql)); + const show = 'select current_setting($1) as v'; + const current = () => + Promise.all( + ['TimeZone', 'DateStyle', 'default_text_search_config'].map( + async (name) => (await client.query({ text: show, values: [name] })).rows, + ), + ); + const before = await current(); + const onClient = new PostgresPluginDataStore('pg-settings', keep(client), new PluginDataSlots(1)); + expect(await onClient.query(read)).toEqual([pinned]); + expect(await onClient.tx([{ sql: read }])).toEqual({ results: [{ rows: [pinned] }] }); + // The session still answers as it was set: the plugin's settings were local to its transaction. + const session = (await client.query(read)).rows[0] as Record; + expect(session.ts).not.toBe(pinned.ts); + expect(session.fts).toBe(true); + expect(await current()).toEqual(before); + } finally { + // The test changed this client's session, so it is closed rather than handed back. + client.release(true); + } + }); + + it('PGPLUG-024: the SQL plugins commonly write runs unchanged', async () => { + const store = data.open('pg-common'); + await store.migrate( + '001', + 'CREATE TABLE t (id bigserial PRIMARY KEY, name text UNIQUE NOT NULL, n int NOT NULL DEFAULT 0, ' + + 'at timestamptz NOT NULL DEFAULT now(), uid uuid NOT NULL DEFAULT gen_random_uuid())', + ); + await store.exec('INSERT INTO t (name, n) VALUES ($1, $2), ($3, $4)', ['a', 1, 'b', 2]); + await store.exec('INSERT INTO t (name, n) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET n = t.n + excluded.n', [ + 'a', + 10, + ]); + await store.exec('INSERT INTO t (name) VALUES ($1) ON CONFLICT DO NOTHING', ['b']); + expect( + await store.query( + 'WITH ranked AS (SELECT t.name, t.n, row_number() OVER (ORDER BY t.n DESC) AS r FROM t) ' + + "SELECT name, n, r, coalesce(nullif(name, 'a'), 'first') AS c FROM ranked ORDER BY r", + ), + ).toEqual([ + { name: 'a', n: 11, r: 1, c: 'first' }, + { name: 'b', n: 2, r: 2, c: 'b' }, + ]); + expect( + await store.query( + "SELECT count(*) AS total, string_agg(name, ',' ORDER BY name) AS names, " + + "bool_and(at <= now()) AS past, bool_and(date_trunc('day', at) <= at) AS trunc, " + + 'min(extract(year FROM at)) >= 2024 AS year, count(DISTINCT uid) AS uids FROM t', + ), + ).toEqual([{ total: 2, names: 'a,b', past: true, trunc: true, year: true, uids: 2 }]); + }); }); diff --git a/server/tests/unit/db/engine/plugin-data-contract.test.ts b/server/tests/unit/db/engine/plugin-data-contract.test.ts new file mode 100644 index 000000000..f401c274e --- /dev/null +++ b/server/tests/unit/db/engine/plugin-data-contract.test.ts @@ -0,0 +1,326 @@ +/** + * The contract a plugin's own database (`db:own`) answers alike on both + * engines: every call that breaks it, run on the SQLite store + * (src/db/engine/sqlite/plugin-data-store.ts) and on the Postgres store + * (src/db/engine/postgres/plugin-data-store.ts), must be refused with the + * same words, and those words are the ones `plugin-data-contract.ts` + * holds. A store that answers one case differently, or a + * text spelled by hand in one store, fails here. + * + * The SQLite store runs on a file of its own. The Postgres store runs on a + * stand-in `pg` client that answers every statement with no rows and every + * cursor with the rows of the seeded table, which is all these cases reach; + * the same store on a real database is the Postgres lane's + * tests/postgres/plugin-data.test.ts. + */ +import type { PluginDataStore } from '../../../../src/db/engine/plugin-data'; +import { + PLUGIN_DATA_CAPS, + PLUGIN_DATA_ERRORS, + PLUGIN_SQL_NUL_REFUSAL, + PLUGIN_VALUE_NUL_REFUSAL, + refuseTransactionControl, +} from '../../../../src/db/engine/plugin-data-contract'; +import { + PluginDataSlots, + PostgresPluginDataStore, + type PluginSqlClient, +} from '../../../../src/db/engine/postgres/plugin-data-store'; +import { SqlitePluginDataStore } from '../../../../src/db/engine/sqlite/plugin-data-store'; + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import Cursor from 'pg-cursor'; +import { afterAll, describe, expect, it } from 'vitest'; + +/** The rows of `seq` on both engines: five of them, more than the row cap below. */ +const SEQ_ROWS = [1, 2, 3, 4, 5].map((n) => ({ n })); + +interface Limits { + maxRows?: number; + timeBudgetMs?: number; + now?: () => number; +} + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-plugin-contract-')); +let stores = 0; + +afterAll(() => { + fs.rmSync(dir, { recursive: true, force: true }); +}); + +/** A SQLite store on a file of its own, holding the table `seq`. */ +function sqliteStore(limits: Limits): SqlitePluginDataStore { + const id = `contract-${++stores}`; + const store = new SqlitePluginDataStore(id, path.join(dir, id, 'plugin.db'), limits); + store.migrate('seq', 'CREATE TABLE seq (n INTEGER)'); + store.exec(`INSERT INTO seq (n) VALUES ${SEQ_ROWS.map((row) => `(${row.n})`).join(', ')}`); + return store; +} + +/** A Postgres store whose client reads {@link SEQ_ROWS} from every cursor, one row per read. */ +function postgresStore(limits: Limits): PostgresPluginDataStore { + const client = { + query(arg: unknown): unknown { + if (arg instanceof Cursor) { + let next = 0; + return { + read: async () => (next < SEQ_ROWS.length ? [SEQ_ROWS[next++]] : []), + close: async () => undefined, + }; + } + return Promise.resolve({ rows: [], rowCount: 0 }); + }, + release: () => undefined, + }; + return new PostgresPluginDataStore( + 'contract', + async () => client as unknown as PluginSqlClient, + new PluginDataSlots(1), + limits, + { refuseIfPrivileged: async () => undefined }, + ); +} + +/** A clock that moves `step` ms each time it is read, so a budget runs out on cue. */ +function steppingClock(step: number): () => number { + let t = 0; + return () => (t += step); +} + +/** What a call answered: its result, or the message it was refused with. */ +async function outcome(call: () => unknown): Promise { + try { + return `answered ${JSON.stringify(await call())}`; + } catch (err) { + return `refused: ${err instanceof Error ? err.message : String(err)}`; + } +} + +type Case = [name: string, call: (store: PluginDataStore) => unknown, expected: string]; + +const refused = (message: string) => `refused: ${message}`; +const tooManyOps = Array.from({ length: PLUGIN_DATA_CAPS.maxTxOps + 1 }, () => ({ sql: 'SELECT 1' })); + +/** + * Transaction control in any call and any statement position, in every spelling + * either engine knows, refused with one sentence; and the words where they are no + * statement's verb, which run. + */ +const TX_CONTROL_CASES: Case[] = [ + ...[ + 'BEGIN', + 'begin transaction', + 'START TRANSACTION', + 'COMMIT', + 'End', + 'ROLLBACK', + 'ABORT', + 'SAVEPOINT a', + 'RELEASE a', + '-- a comment first\nCOMMIT', + '/* x */ROLLBACK', + '/* a; b */ /* c */ end', + ].map((sql): Case => [ + `a query of ${JSON.stringify(sql)}`, + (s) => s.query(sql), + refused(PLUGIN_DATA_ERRORS.txControl), + ]), + [ + 'a script that opens a transaction', + (s) => s.exec('BEGIN; INSERT INTO seq (n) VALUES (6); COMMIT;'), + refused(PLUGIN_DATA_ERRORS.txControl), + ], + [ + 'a script that ends with COMMIT', + (s) => s.exec('INSERT INTO seq (n) VALUES (6);\n-- done\ncommit'), + refused(PLUGIN_DATA_ERRORS.txControl), + ], + ['an exec with args of COMMIT', (s) => s.exec('COMMIT', [1]), refused(PLUGIN_DATA_ERRORS.txControl)], + [ + 'a tx op of transaction control', + (s) => s.tx([{ sql: 'UPDATE seq SET n = n' }, { sql: ' /* x */ SAVEPOINT a' }]), + refused(PLUGIN_DATA_ERRORS.txControl), + ], + [ + 'a migration that commits', + (s) => s.migrate('m', 'CREATE TABLE t (x INTEGER); COMMIT; CREATE TABLE u (x INTEGER)'), + refused(PLUGIN_DATA_ERRORS.txControl), + ], + ['a CASE ... END expression', (s) => s.exec("SELECT CASE WHEN 1 = 1 THEN 'y' END AS s"), 'answered {"changes":0}'], + ['the words in a string', (s) => s.exec("SELECT 'BEGIN; COMMIT' AS s"), 'answered {"changes":0}'], + ['the words in a comment', (s) => s.exec('SELECT 1; -- COMMIT\n/* END; */'), 'answered {"changes":0}'], +]; + +/** Calls that break the contract, or sit at its edge, on a store that is open. */ +const OPEN_CASES: Case[] = [ + ['query with SQL that is no string', (s) => s.query(42 as never), refused(PLUGIN_DATA_ERRORS.sqlNotString)], + ['exec with SQL that is no string', (s) => s.exec(null as never), refused(PLUGIN_DATA_ERRORS.sqlNotString)], + ['migrate with SQL that is no string', (s) => s.migrate('m', {} as never), refused(PLUGIN_DATA_ERRORS.sqlNotString)], + ['a tx op with SQL that is no string', (s) => s.tx([{ sql: 7 as never }]), refused(PLUGIN_DATA_ERRORS.sqlNotString)], + ['a tx op that is no object', (s) => s.tx([null as never]), refused(PLUGIN_DATA_ERRORS.sqlNotString)], + [ + 'SQL past the length cap', + (s) => s.query('x'.repeat(PLUGIN_DATA_CAPS.maxSqlLength + 1)), + refused(PLUGIN_DATA_ERRORS.sqlTooLong), + ], + ['SQL holding NUL', (s) => s.query("SELECT 'a\u0000'"), refused(PLUGIN_SQL_NUL_REFUSAL)], + ['a script holding NUL', (s) => s.exec("SELECT 'a\u0000'"), refused(PLUGIN_SQL_NUL_REFUSAL)], + ['an argument holding NUL', (s) => s.query('SELECT 1', ['a\u0000']), refused(PLUGIN_VALUE_NUL_REFUSAL)], + ['a migration id holding NUL', (s) => s.migrate('m\u0000', 'SELECT 1'), refused(PLUGIN_VALUE_NUL_REFUSAL)], + ['a query without a statement', (s) => s.query(''), refused(PLUGIN_DATA_ERRORS.noStatement)], + ['a query of a comment only', (s) => s.query('-- nothing'), refused(PLUGIN_DATA_ERRORS.noStatement)], + ['a query of two statements', (s) => s.query('SELECT 1; SELECT 2'), refused(PLUGIN_DATA_ERRORS.moreThanOneStatement)], + [ + 'an exec with args of two statements', + (s) => s.exec('SELECT 1; SELECT 2', [1]), + refused(PLUGIN_DATA_ERRORS.moreThanOneStatement), + ], + ['an empty script', (s) => s.exec(''), 'answered {"changes":0}'], + ['a script of a comment only', (s) => s.exec('-- nothing'), 'answered {"changes":0}'], + ['a tx that is no array', (s) => s.tx('nope' as never), refused(PLUGIN_DATA_ERRORS.txNotArray)], + ['an empty tx', (s) => s.tx([]), 'answered {"results":[]}'], + [ + 'a tx past the statement cap', + (s) => s.tx(tooManyOps), + refused(PLUGIN_DATA_ERRORS.txTooManyOps(PLUGIN_DATA_CAPS.maxTxOps)), + ], + ['a query past the row cap', (s) => s.query('SELECT n FROM seq'), refused(PLUGIN_DATA_ERRORS.queryTooManyRows(3))], + ['a tx past the row cap', (s) => s.tx([{ sql: 'SELECT n FROM seq' }]), refused(PLUGIN_DATA_ERRORS.txTooManyRows(3))], + ...TX_CONTROL_CASES, +]; + +/** Calls on a store whose clock runs out of its time budget while it reads. */ +const BUDGET_CASES: Case[] = [ + [ + 'a query past its time budget', + (s) => s.query('SELECT n FROM seq'), + refused(PLUGIN_DATA_ERRORS.overBudget('query', 100)), + ], + [ + 'a tx past its time budget', + (s) => s.tx([{ sql: 'SELECT n FROM seq' }]), + refused(PLUGIN_DATA_ERRORS.overBudget('tx', 100)), + ], +]; + +/** Calls on a store that was closed. */ +const CLOSED_CASES: Case[] = [ + ['a query', (s) => s.query('SELECT 1'), refused(PLUGIN_DATA_ERRORS.closed)], + ['an exec with args', (s) => s.exec('SELECT ?', [1]), refused(PLUGIN_DATA_ERRORS.closed)], + ['a script', (s) => s.exec('SELECT 1'), refused(PLUGIN_DATA_ERRORS.closed)], + ['an empty script', (s) => s.exec(''), refused(PLUGIN_DATA_ERRORS.closed)], + ['a script of a comment only', (s) => s.exec('-- nothing'), refused(PLUGIN_DATA_ERRORS.closed)], + ['an empty tx', (s) => s.tx([]), refused(PLUGIN_DATA_ERRORS.closed)], + ['a tx', (s) => s.tx([{ sql: 'SELECT 1' }]), refused(PLUGIN_DATA_ERRORS.closed)], + ['a migration', (s) => s.migrate('m', 'SELECT 1'), refused(PLUGIN_DATA_ERRORS.closed)], +]; + +/** What each engine's store answers `call` with, on a store made by `limits` and `prepare`. */ +async function bothAnswer( + call: Case[1], + limits: () => Limits, + prepare: (store: PluginDataStore) => void = () => undefined, +): Promise<{ sqlite: string; postgres: string }> { + const sqlite = sqliteStore(limits()); + const postgres = postgresStore(limits()); + prepare(sqlite); + prepare(postgres); + try { + return { sqlite: await outcome(() => call(sqlite)), postgres: await outcome(() => call(postgres)) }; + } finally { + sqlite.close(); + } +} + +describe('plugin data contract on both engines', () => { + it.each(OPEN_CASES)('PLUGCONTRACT-001: %s is answered alike', async (_name, call, expected) => { + const answers = await bothAnswer(call, () => ({ maxRows: 3 })); + expect(answers).toEqual({ sqlite: expected, postgres: expected }); + }); + + it.each(BUDGET_CASES)('PLUGCONTRACT-002: %s is answered alike', async (_name, call, expected) => { + const answers = await bothAnswer(call, () => ({ timeBudgetMs: 100, now: steppingClock(30) })); + expect(answers).toEqual({ sqlite: expected, postgres: expected }); + }); + + it.each(CLOSED_CASES)('PLUGCONTRACT-003: %s on a closed store is answered alike', async (_name, call, expected) => { + const answers = await bothAnswer( + call, + () => ({}), + (store) => store.close(), + ); + expect(answers).toEqual({ sqlite: expected, postgres: expected }); + }); +}); + +/** + * The scan behind refuseTransactionControl, which the SQLite store and the SDK + * run (the Postgres guard reads plugin SQL with its own lexer). A SQLite built + * with Tcl variables (node:sqlite under trek-plugin dev) reads `$a(...)` up to + * the closing parenthesis as one parameter, quotes and semicolons included. + */ +describe('refuseTransactionControl', () => { + const hiddenCommit = "INSERT INTO seq (n) VALUES ($a(')); COMMIT; --')"; + + it.each([ + hiddenCommit, + "SELECT @a(')", + "SELECT :a(')", + "SELECT #a(')", + "SELECT $a::b(')", + "SELECT $::a(')", + 'SELECT $1(1)', + "SELECT 1; SELECT $a('x')", + ])('PLUGCONTRACT-004: refuses a parameter name directly followed by "(" in %j', (sql) => { + expect(() => refuseTransactionControl(sql)).toThrow(new Error(PLUGIN_DATA_ERRORS.parameterBeforeParenthesis)); + }); + + it.each([ + 'SELECT $a, :b, @c, #d, ?1', + 'SELECT $a (1)', + 'SELECT 1 WHERE $a::b = 1', + "SELECT '$a(' AS s", + 'SELECT "$a(" FROM seq', + 'SELECT [$a(] FROM seq', + 'SELECT 1 -- $a(\n', + 'SELECT x::numeric(10, 2), y #> $1, z @> $2, w := 1 FROM seq', + 'SELECT $ (1), @(1), :(1)', + ])('PLUGCONTRACT-005: lets %j through', (sql) => { + expect(() => refuseTransactionControl(sql)).not.toThrow(); + }); + + it.each([ + 'EXPLAIN CREATE TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; END', + 'explain query plan create temp trigger t after insert on seq begin select 1; end;', + 'EXPLAIN CREATE TEMPORARY TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; SELECT 2; END; SELECT 3', + ])('PLUGCONTRACT-006: reads the body of %j as a trigger body', (sql) => { + expect(() => refuseTransactionControl(sql)).not.toThrow(); + }); + + it.each([ + 'EXPLAIN CREATE TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; END; COMMIT', + 'EXPLAIN QUERY PLAN CREATE TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; END; ROLLBACK', + 'EXPLAIN SELECT 1; END', + 'EXPLAIN QUERY SELECT 1; END', + ])('PLUGCONTRACT-007: refuses the transaction control in %j', (sql) => { + expect(() => refuseTransactionControl(sql)).toThrow(new Error(PLUGIN_DATA_ERRORS.txControl)); + }); + + it('PLUGCONTRACT-008: the SQLite store refuses the hidden COMMIT and runs EXPLAIN CREATE TRIGGER', () => { + const store = sqliteStore({}); + try { + expect(() => store.exec(hiddenCommit)).toThrow(new Error(PLUGIN_DATA_ERRORS.parameterBeforeParenthesis)); + expect(store.query('SELECT count(*) AS c FROM seq')).toEqual([{ c: SEQ_ROWS.length }]); + expect(store.query('EXPLAIN CREATE TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; END').length).toBeGreaterThan(0); + expect( + store.query('EXPLAIN QUERY PLAN CREATE TEMP TRIGGER t AFTER INSERT ON seq BEGIN SELECT 1; END'), + ).toBeInstanceOf(Array); + // EXPLAIN only describes the statement: no trigger was made. + expect(store.query("SELECT count(*) AS c FROM sqlite_master WHERE type = 'trigger'")).toEqual([{ c: 0 }]); + } finally { + store.close(); + } + }); +}); diff --git a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts index bd7a31a4f..1d29ab1f9 100644 --- a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts @@ -10,6 +10,11 @@ * store on a real database is the Postgres lane's * tests/postgres/plugin-data.test.ts. */ +import { + PLUGIN_DATA_ERRORS, + PLUGIN_SQL_NUL_REFUSAL, + PLUGIN_VALUE_NUL_REFUSAL, +} from '../../../../../src/db/engine/plugin-data-contract'; import { PRIVILEGED_ROLE_PLUGIN_REFUSAL } from '../../../../../src/db/engine/postgres/plugin-data-checks'; import { dropPluginSchema, @@ -19,6 +24,7 @@ import { PostgresPluginDataStore, type PluginSqlClient, } from '../../../../../src/db/engine/postgres/plugin-data-store'; +import { PLUGIN_SESSION_SETTINGS } from '../../../../../src/db/engine/postgres/plugin-session'; import { PRIVILEGED_ROLE_SQL } from '../../../../../src/db/engine/postgres/server-checks'; import type { Kysely } from 'kysely'; @@ -144,7 +150,7 @@ function pgError(code: string, message: string): Error { } describe('Postgres plugin data store', () => { - it('PGSTORE-001: a query runs in a transaction that makes the schema, scopes the search_path and sets the budget', async () => { + it('PGSTORE-001: a query runs in a transaction that makes the schema, scopes the search_path, pins the settings and sets the budget', async () => { const { store: s, log } = store({ rows: { 'SELECT body FROM notes': [{ body: 'a' }, { body: 'b' }, { body: 'c' }] }, }); @@ -157,7 +163,24 @@ describe('Postgres plugin data store', () => { 'create schema if not exists "trek_plugin_notes"', 'create table if not exists "trek_plugin_notes"._plugin_migrations (id text primary key, applied_at bigint)', 'set local search_path to "trek_plugin_notes", pg_temp', - 'set local standard_conforming_strings to on', + "set local TimeZone to 'UTC'", + "set local DateStyle to 'ISO, MDY'", + "set local IntervalStyle to 'postgres'", + "set local timezone_abbreviations to 'Default'", + "set local lc_monetary to 'C'", + "set local lc_numeric to 'C'", + "set local lc_time to 'C'", + "set local extra_float_digits to '1'", + "set local bytea_output to 'hex'", + "set local xmlbinary to 'base64'", + "set local xmloption to 'content'", + "set local default_text_search_config to 'pg_catalog.simple'", + "set local standard_conforming_strings to 'on'", + "set local backslash_quote to 'safe_encoding'", + "set local array_nulls to 'on'", + "set local transform_null_equals to 'off'", + "set local quote_all_identifiers to 'off'", + "set local gin_fuzzy_search_limit to '0'", 'set local statement_timeout to 2000', ].join(';\n'), 'SELECT body FROM notes', @@ -166,6 +189,13 @@ describe('Postgres plugin data store', () => { ]); }); + it('PGSTORE-020: every pinned setting is a plain name and value, set with SET LOCAL', () => { + for (const [name, value] of Object.entries(PLUGIN_SESSION_SETTINGS)) { + expect(name).toMatch(/^[A-Za-z_]+$/); + expect(value).not.toMatch(/['\\]/); + } + }); + it("PGSTORE-002: a qualified name is checked against the database's schemas before the statement runs", async () => { const { store: s, log } = store({ schemas: ['public', 'auth', 'trek_plugin_notes'] }); await expect(s.query('SELECT x FROM auth.secret')).rejects.toThrow(/"auth" is another schema/); @@ -252,7 +282,7 @@ describe('Postgres plugin data store', () => { await expect(s.tx([])).resolves.toEqual({ results: [] }); await expect(s.tx(Array.from({ length: 101 }, () => ({ sql: 'SELECT 1' })))).rejects.toThrow(/at most 100/); - await expect(s.tx([{ sql: '/* x */ COMMIT' }])).rejects.toThrow(/transaction-control/); + await expect(s.tx([{ sql: '/* x */ COMMIT' }])).rejects.toThrow(PLUGIN_DATA_ERRORS.txControl); await expect(s.tx('nope' as never)).rejects.toThrow(/requires an array/); }); @@ -281,7 +311,7 @@ describe('Postgres plugin data store', () => { await expect(s.query(42 as never)).rejects.toThrow('sql must be a string'); await expect(s.query('x'.repeat(100_001))).rejects.toThrow('sql too long'); await expect(s.query('-- only a comment')).rejects.toThrow('contains no statements'); - await expect(s.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); + await expect(s.query('SELECT 1; SELECT 2')).rejects.toThrow(PLUGIN_DATA_ERRORS.moreThanOneStatement); await expect(s.exec('DO $$ BEGIN END $$')).rejects.toThrow(/dollar-quoted/); expect(log).toEqual([]); expect(releases).toEqual([]); @@ -342,9 +372,12 @@ describe('Postgres plugin data store', () => { expect(configs.find((q) => q.text.startsWith('INSERT'))?.values).toEqual(['cd']); }); - it('PGSTORE-016: an argument or a migration id holding NUL is refused before a client is taken, never altered', async () => { + it('PGSTORE-016: SQL, an argument or a migration id holding NUL is refused before a client is taken, never altered', async () => { const { store: s, log, releases } = store(); - const nul = /U\+0000 \(NUL\), which Postgres text cannot store/; + const nul = PLUGIN_VALUE_NUL_REFUSAL; + // The same refusal as on SQLite, before the guard reads the SQL. + await expect(s.exec("INSERT INTO t VALUES ('a\u0000')")).rejects.toThrow(PLUGIN_SQL_NUL_REFUSAL); + await expect(s.query('SELECT 1 FROM t WHERE k = $1', [{ nested: ['\u0000'] }])).rejects.toThrow(nul); await expect(s.query('SELECT 1 FROM t WHERE k = $1', ['a\u0000b'])).rejects.toThrow(nul); await expect(s.exec('INSERT INTO t VALUES ($1)', [['x', 'c\u0000d']])).rejects.toThrow(nul); await expect(s.tx([{ sql: 'SELECT 1' }, { sql: 'DELETE FROM t WHERE k = $1', args: ['\u0000'] }])).rejects.toThrow( @@ -416,7 +449,7 @@ describe('Postgres plugin data store', () => { it('PGSTORE-014: each transaction checks out a client of its own and gives it back clean, or closes it', async () => { const ok = store({ changes: 1 }); await ok.store.exec('INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)'); - await expect(ok.store.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); + await expect(ok.store.query('SELECT 1; SELECT 2')).rejects.toThrow(PLUGIN_DATA_ERRORS.moreThanOneStatement); // One client per statement of the script, each after its session state was dropped; the refused read took none. expect(ok.releases).toEqual([false, false]); expect(ok.log.filter((sql) => sql === END_SESSION)).toHaveLength(2); diff --git a/server/tests/unit/db/engine/postgres/plugin-sql-functions.test.ts b/server/tests/unit/db/engine/postgres/plugin-sql-functions.test.ts index 3e689f39b..e8179e1a3 100644 --- a/server/tests/unit/db/engine/postgres/plugin-sql-functions.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-sql-functions.test.ts @@ -60,6 +60,12 @@ describe('functions a plugin may call on Postgres', () => { "select has_largeobject_privilege(1, 'select')", 'select acldefault(1, 10)', 'select setseed(0.5)', + // A text search configuration is a regconfig, looked up by a name in a string. + "select to_tsvector('trek_plugin_other.cfg', 'a')", + "select to_tsvector('english', body) @@ plainto_tsquery('english', $1) from notes", + "select to_tsquery('a'), phraseto_tsquery('a'), websearch_to_tsquery('a')", + "select ts_headline('a', 'a'::tsquery)", + "select jsonb_to_tsvector('{}'::jsonb, '[\"all\"]')", // Every spelling of the call. 'SELECT OBJ_DESCRIPTION(16386)', 'select "obj_description"(16386)', @@ -92,6 +98,15 @@ describe('functions a plugin may call on Postgres', () => { expect(() => parsePluginSql(sql)).toThrow(/a plugin may not (call|read)/); }); + it.each([ + // xid is the transaction id type age() reads the server's counter with; after a dot it is a cast. + "select age('3'::xid)", + 'select n.xid from notes n', + 'select $1.xid', + ])('refuses %s as a transaction id', (sql) => { + expect(() => parsePluginSql(sql)).toThrow(/stands for a transaction id type/); + }); + it.each([ "select lower(body), substring(body from 2 for 3), regexp_replace(body, 'a', 'b') from notes", "select now(), date_trunc('day', now()), extract(epoch from now()), to_char(now(), 'YYYY')", @@ -100,7 +115,9 @@ describe('functions a plugin may call on Postgres', () => { 'select gen_random_uuid(), md5(body), sha256(body::bytea) from notes', 'select x from generate_series(1, 5) as g(x)', 'select unnest(array[1, 2]), cardinality(array[1])', - "select to_tsvector('english', body) @@ plainto_tsquery('english', $1) from notes", + // Text search on vectors a plugin builds itself, with no configuration to look up. + "select ts_rank(array_to_tsvector(tags), 'a & b'::tsquery), setweight(array_to_tsvector(tags), 'A') from notes", + 'select body from notes where array_to_tsvector(tags) @@ $1::tsquery', 'select coalesce(a, b), nullif(a, 0), greatest(a, b), least(a, b) from notes', "select cast(n as numeric(10, 2)), body::varchar(20), 'now'::timestamp(3) from notes", 'select current_date, current_timestamp, localtimestamp', @@ -115,7 +132,7 @@ describe('functions a plugin may call on Postgres', () => { 'CREATE VIEW version (a) AS SELECT 1', 'select n.version, n.user_id, "user", "current_user" from notes n', // The refused names a table may well have as a column, after a dot. - 'select n.name, n.money, n.oid, n.xid, n.xml, n.version from notes n', + 'select n.name, n.money, n.oid, n.xml, n.version from notes n', 'select (n).name, $1.name from notes n', "insert into notes (id, name) values (1, 'a') on conflict (id) do update set name = excluded.name", 'select n.body from trek_plugin_notes.notes n join trek_plugin_notes.version v on v.id = n.id', diff --git a/server/tests/unit/db/engine/postgres/plugin-sql-guard.test.ts b/server/tests/unit/db/engine/postgres/plugin-sql-guard.test.ts index 56ae4d7cb..6eb06b5da 100644 --- a/server/tests/unit/db/engine/postgres/plugin-sql-guard.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-sql-guard.test.ts @@ -5,6 +5,7 @@ * the way Postgres does, and what it tells the store about a statement. The * same rules against a real database are PGPLUG-003/004 in the Postgres lane. */ +import { PLUGIN_SQL_NUL_REFUSAL } from '../../../../../src/db/engine/plugin-data-contract'; import { checkPlaceholders, checkQualifiers, @@ -57,6 +58,31 @@ describe('plugin SQL guard', () => { 'SELECT 1 -- public.users\r', 'SELECT 1 -- public.users\r\nFROM notes', 'SELECT\t1\f', + // A value of the plugin's own may carry a system column's name where it is given. + 'SELECT min(lat) AS ymin, min(lng) AS xmin, max(lng) AS xmax FROM places', + 'SELECT min(lng) AS "xmin" FROM places', + 'WITH b(xmin, ymin) AS (SELECT 1, 2) SELECT * FROM b', + 'WITH a AS (SELECT 1), b(ctid) AS NOT MATERIALIZED (SELECT 1) SELECT * FROM b', + 'SELECT * FROM (SELECT 1, 2) AS b(xmin, xmax)', + // A column of the plugin's own may be called xid, without its table. + 'CREATE TABLE payments (id bigserial PRIMARY KEY, xid text NOT NULL UNIQUE)', + 'ALTER TABLE payments ADD COLUMN xid text', + 'SELECT xid, count(*) FROM payments WHERE xid = $1 AND NOT xid IS NULL GROUP BY xid ORDER BY xid', + 'INSERT INTO payments (id, xid) VALUES (1, $1) ON CONFLICT (xid) DO NOTHING', + 'UPDATE payments SET xid = $1 WHERE id = $2 RETURNING xid', + 'SELECT id AS xid FROM payments', + "SELECT CASE WHEN xid = 'a' THEN xid ELSE NULL END, coalesce(xid, '') FROM payments", + // The collations and access methods every server has. + 'SELECT body FROM notes ORDER BY body COLLATE "C", body COLLATE "POSIX", body COLLATE "default"', + 'SELECT body COLLATE ucs_basic FROM notes', + 'CREATE INDEX i ON notes USING gin (tags)', + 'CREATE INDEX i ON notes USING brin (id)', + 'CREATE TABLE t (x int) USING heap', + 'ALTER TABLE notes ADD EXCLUDE USING gist (r WITH &&)', + 'ALTER TABLE notes ALTER id TYPE int USING id::int', + 'SELECT * FROM notes JOIN tags USING (id)', + 'DELETE FROM notes USING trash WHERE notes.id = trash.note_id', + "SELECT collation FROM notes WHERE collation = 'x'", // Characters JavaScript reads differently are fine inside strings, quoted names and comments. "SELECT 'a b
c\u000bd'", 'SELECT 1 AS "a b"', @@ -149,7 +175,7 @@ describe('plugin SQL guard', () => { expect(() => check(sql)).toThrow(reason); }); - // The two bypasses of the final review, with their variants: each was let through, and Postgres ran it. + // Bypasses an earlier guard let through and Postgres ran, with their variants. it.each([ // A `--` comment ended by a bare CR hid the rest from the guard, which Postgres then ran. ['select 1 --\rfrom public.users', /another schema/], @@ -179,7 +205,7 @@ describe('plugin SQL guard', () => { ['CREATE TEMP VIEW v AS SELECT 1', /CREATE TEMP is not allowed/], ['CREATE OR REPLACE TEMP VIEW v AS SELECT 1', /CREATE TEMP is not allowed/], ['CREATE TEMPORARY SEQUENCE s', /CREATE TEMPORARY is not allowed/], - // The re-review's bypasses. A column alias named insert or merge in front of INTO made a table. + // A column alias named insert or merge in front of INTO made a table. ['select 1 insert into t1', /SELECT \.\.\. INTO/], ['select 1 merge into t9', /SELECT \.\.\. INTO/], ['select 1 insert into unlogged t10', /SELECT \.\.\. INTO/], @@ -187,6 +213,53 @@ describe('plugin SQL guard', () => { ['WITH x AS (SELECT 1) SELECT 1 insert INTO t', /SELECT \.\.\. INTO/], ['insert into notes select 1 insert into t', /SELECT \.\.\. INTO/], ['select 1 insert into temp t', /SELECT \.\.\. INTO TEMP/], + // A read of a system column, quoted or not, wherever it stands. + ['select xmin, xmax from notes', /"xmin" reads a Postgres system column/], + ['select n.xmax from notes n', /"xmax" reads a Postgres system column/], + ['select cmin, cmax from notes', /"cmin" reads a Postgres system column/], + ['select ctid from notes order by 1', /"ctid" reads a Postgres system column/], + ['select "tableoid" from notes', /"tableoid" reads a Postgres system column/], + ['update notes set body = $1 where id = $2 and xmin = $3', /"xmin" reads a Postgres system column/], + ['delete from notes returning ctid', /"ctid" reads a Postgres system column/], + ['select min(id) xmin from notes', /"xmin" reads a Postgres system column/], + ['WITH b(xmin) AS (SELECT 1) SELECT xmin FROM b', /"xmin" reads a Postgres system column/], + ['select * from notes n, f(xmin) as (a int)', /"xmin" reads a Postgres system column/], + ['select * from notes n, f(xmin, ctid) as (a int)', /"xmin" reads a Postgres system column/], + // A transaction id type where the word stands for the type. + ["select age('3'::xid)", /"xid" stands for a transaction id type/], + ["select cast('3' as xid8)", /"xid8" stands for a transaction id type/], + ["select treat('3' as xid)", /"xid" stands for a transaction id type/], + ["select age(xid '3')", /"xid" stands for a transaction id type/], + ["select age(xid('3'))", /"xid" stands for a transaction id type/], + ['select age($1.xid)', /"xid" stands for a transaction id type/], + ["select age(u.xid) from unnest(array['3']) u", /"xid" stands for a transaction id type/], + ["select age(('3').xid)", /"xid" stands for a transaction id type/], + ['CREATE TABLE t (x _xid)', /"_xid" stands for a transaction id type/], + ['CREATE TABLE t ("x" xid)', /"xid" stands for a transaction id type/], + ["select '{}'::xid8[]", /"xid8" stands for a transaction id type/], + ['select * from notes n, json_to_record(n.j) as x(a xid)', /"xid" stands for a transaction id type/], + ['ALTER TABLE t ALTER COLUMN c TYPE xid', /"xid" stands for a transaction id type/], + ['ALTER TABLE t ADD COLUMN c xid', /"xid" stands for a transaction id type/], + ["select json_value('3', '$' returning xid)", /"xid" stands for a transaction id type/], + ['CREATE TYPE r AS RANGE (subtype = xid8)', /"xid8" stands for a transaction id type/], + ['CREATE TYPE c AS (a xid)', /"xid" stands for a transaction id type/], + [ + 'insert into pay (id, xid) values (1, $1) on conflict (id) do update set xid = excluded.xid', + /"xid" stands for a transaction id type.*excluded\.xid in an upsert/, + ], + // A collation or an access method whose presence depends on the server. + [`select 'a' collate "de-x-icu"`, /a collation other than "C"/], + [`select body from notes order by body collate "de_DE.utf8"`, /a collation other than "C"/], + ['select body from notes order by body collate C', /a collation other than "C"/], + [`select 'a' collate trek_plugin_notes."C"`, /a collation other than "C"/], + [`CREATE TABLE t (a text COLLATE "und-x-icu")`, /a collation other than "C"/], + [`CREATE TYPE r AS RANGE (subtype = text, collation = "de-x-icu")`, /a collation other than "C"/], + ['CREATE INDEX i ON notes USING bloom (body)', /built-in access methods/], + ['CREATE INDEX i ON notes USING "BTREE" (body)', /built-in access methods/], + ['CREATE TABLE t (x int) USING columnar', /built-in access methods/], + ['CREATE MATERIALIZED VIEW mv USING columnar AS SELECT 1', /built-in access methods/], + ['ALTER TABLE notes SET ACCESS METHOD columnar', /built-in access methods/], + ['ALTER TABLE notes ADD EXCLUDE USING bloom (body WITH =)', /built-in access methods/], // The reg* types and functions that look an object up by a name in a string, in every spelling. ["select 'public.users'::regtype::oid", /"regtype" may not be named/], ["select cast('int4' as regtype)", /"regtype" may not be named/], @@ -208,8 +281,6 @@ describe('plugin SQL guard', () => { ['CREATE TABLE grants (a aclitem)', /"aclitem" may not be named/], // A base type is built from C functions, which only a superuser may do. ['CREATE TYPE fake (INPUT = int4in, OUTPUT = int4out, INTERNALLENGTH = 1000)', /composite, enum or range/], - // The protocol ends a statement's text at a NUL, in a string as anywhere else. - ["select 'a\u0000' -- then\nfrom public.users", /U\+0000/], ['CREATE INDEX i (x)', /must name its table with ON/], ['ALTER TABLE', /could not be read/], ])('refuses the bypass %j', (sql, reason) => { @@ -224,7 +295,6 @@ describe('plugin SQL guard', () => { ['PARAGRAPH SEPARATOR', '
', 'U+2029'], ['IDEOGRAPHIC SPACE', ' ', 'U+3000'], ['vertical tab', '\u000b', 'U+000B'], - ['NUL', '\u0000', 'U+0000'], ['escape', '\u001b', 'U+001B'], ['DEL', '\u007f', 'U+007F'], ])('refuses a %s outside a string, which JavaScript and Postgres read differently', (_name, ch, code) => { @@ -239,6 +309,19 @@ describe('plugin SQL guard', () => { } }); + it('refuses a NUL anywhere, in a string too, with the words the stores of both engines use', () => { + // The protocol ends a statement's text at a NUL, so what Postgres would read is not what was checked. + for (const sql of [ + "select 'a\u0000' -- then\nfrom public.users", + 'SELECT 1\u0000FROM public.users', + '\u0000SELECT 1', + 'SELECT "a\u0000b" FROM notes', + ]) { + expect(() => tokenize(sql), JSON.stringify(sql)).toThrow(PLUGIN_SQL_NUL_REFUSAL); + expect(() => check(sql), JSON.stringify(sql)).toThrow(PLUGIN_SQL_NUL_REFUSAL); + } + }); + it('reads nested block comments as Postgres does, so code after the inner close is still a comment', () => { expect(() => check('SELECT 1 /* outer /* inner */ public.users */')).not.toThrow(); expect(() => check('SELECT 1 /* one */ FROM public.users')).toThrow(/another schema/); diff --git a/server/tests/unit/eslint-rules/restricted-imports-xml.test.ts b/server/tests/unit/eslint-rules/restricted-imports-xml.test.ts index cc76f5433..592de6c77 100644 --- a/server/tests/unit/eslint-rules/restricted-imports-xml.test.ts +++ b/server/tests/unit/eslint-rules/restricted-imports-xml.test.ts @@ -48,8 +48,8 @@ describe('fast-xml-parser stays in place-import and the WebDAV client', () => { 'src/nest/tours/tours.service.ts', 'src/nest/maps/maps.service.ts', 'src/db/repositories/Places.repository.ts', - // The driver's own homes and the plugin sandbox's database, which the - // driver wall leaves out, and the RPC kit, which restates the rule. + // The driver's own homes, which the driver wall leaves out, the plugin + // data directories, which it holds again, and the RPC kit, which restates the rule. 'src/db/database.ts', 'src/db/connection.ts', 'src/db/orm-driver.ts', @@ -72,8 +72,9 @@ describe('fast-xml-parser stays in place-import and the WebDAV client', () => { }); it('keeps the services wall in the driver-exempt files and the driver wall in the RPC kit', async () => { - expect(await flags('src/nest/plugins/host/plugin-data.service.ts', SERVICES)).toBe(true); - expect(await flags('src/nest/plugins/host/plugin-data.service.ts', DRIVER)).toBe(false); + expect(await flags('src/db/connection.ts', SERVICES)).toBe(true); + expect(await flags('src/db/connection.ts', DRIVER)).toBe(false); + expect(await flags('src/nest/plugins/host/plugin-data.service.ts', DRIVER)).toBe(true); expect(await flags('src/nest-rpc/rpc-kit/registry.ts', DRIVER)).toBe(true); }); diff --git a/server/tests/unit/plugins/plugin-data-backend.test.ts b/server/tests/unit/plugins/plugin-data-backend.test.ts index 26d278bb8..4f8bad9a5 100644 --- a/server/tests/unit/plugins/plugin-data-backend.test.ts +++ b/server/tests/unit/plugins/plugin-data-backend.test.ts @@ -10,13 +10,13 @@ import type { DatabaseEngine } from '../../../src/db/engine'; import { PRIVILEGED_ROLE_PLUGIN_REFUSAL } from '../../../src/db/engine/postgres/plugin-data-checks'; import { PostgresPluginDataStore } from '../../../src/db/engine/postgres/plugin-data-store'; import { PRIVILEGED_ROLE_DESCRIPTION } from '../../../src/db/engine/postgres/server-checks'; +import { SqlitePluginDataStore } from '../../../src/db/engine/sqlite/plugin-data-store'; import { PLUGIN_DATA, PLUGIN_DATA_PROVIDERS, PostgresPluginDataBackend, SqlitePluginDataBackend, } from '../../../src/nest/plugins/host/plugin-data.backend'; -import { PluginDataDb } from '../../../src/nest/plugins/host/plugin-data.service'; import type { MikroORM } from '@mikro-orm/core'; import type { FactoryProvider } from '@nestjs/common'; @@ -88,7 +88,7 @@ describe('plugin data port', () => { it("PLUGDATA-002: on SQLite a plugin's data is a file of its own, and erasing removes its directory", async () => { const backend = new SqlitePluginDataBackend(); const store = backend.open('sqlite-one'); - expect(store).toBeInstanceOf(PluginDataDb); + expect(store).toBeInstanceOf(SqlitePluginDataStore); expect(store.engine).toBe('sqlite'); store.close(); expect(fs.existsSync(path.join(tmp, 'sqlite-one', 'plugin.db'))).toBe(true); diff --git a/server/tests/unit/plugins/plugin-data.test.ts b/server/tests/unit/plugins/plugin-data.test.ts index 71eda8c29..805172602 100644 --- a/server/tests/unit/plugins/plugin-data.test.ts +++ b/server/tests/unit/plugins/plugin-data.test.ts @@ -1,13 +1,16 @@ /** - * The per-plugin sqlite file (#plugins, M1, db:own). Proves migrations are + * The per-plugin SQLite store (src/db/engine/sqlite/plugin-data-store.ts, db:own). Proves migrations are * idempotent, reads/writes work against the plugin's OWN file, and the guard * blocks statements that would let a plugin escape its file (ATTACH/PRAGMA). */ import { - PluginDataDb, - removePluginData, - snapshotAllPluginDataDbs, -} from '../../../src/nest/plugins/host/plugin-data.service'; + PLUGIN_DATA_ERRORS, + PLUGIN_SQL_NUL_REFUSAL, + PLUGIN_VALUE_NUL_REFUSAL, +} from '../../../src/db/engine/plugin-data-contract'; +import { SqlitePluginDataStore } from '../../../src/db/engine/sqlite/plugin-data-store'; +import { removePluginData, snapshotAllPluginDataDbs } from '../../../src/nest/plugins/host/plugin-data.service'; +import { pluginDbFile } from '../../../src/nest/plugins/paths'; import fs from 'node:fs'; import os from 'node:os'; @@ -24,9 +27,14 @@ afterAll(() => { fs.rmSync(tmp, { recursive: true, force: true }); }); -describe('PluginDataDb', () => { +/** The plugin `id`'s store, at the file the host opens it at. */ +function openStore(id: string, limits: ConstructorParameters[2] = {}) { + return new SqlitePluginDataStore(id, pluginDbFile(id), limits); +} + +describe('SqlitePluginDataStore', () => { it('migrates once (idempotent by id), then reads and writes its own data', () => { - const db = new PluginDataDb('notes'); + const db = openStore('notes'); expect(db.migrate('001', 'CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT)').applied).toBe(true); expect(db.migrate('001', 'CREATE TABLE notes (x)').applied).toBe(false); // same id -> skipped @@ -42,10 +50,10 @@ describe('PluginDataDb', () => { }); it('rejects statements that would escape the plugin file, DoS, or exceed limits', () => { - const db = new PluginDataDb('guard'); + const db = openStore('guard'); expect(() => db.exec("ATTACH DATABASE 'trek.db' AS core")).toThrow(/not allowed/); expect(() => db.query('PRAGMA table_info(x)')).toThrow(/not allowed/); - // WITH RECURSIVE is the unbounded-CPU vector on the synchronous host — refused + // WITH RECURSIVE is the unbounded-CPU vector on the synchronous host, so it is refused expect(() => db.query('WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM r) SELECT x FROM r')).toThrow( /not allowed/, ); @@ -58,7 +66,7 @@ describe('PluginDataDb', () => { }); it('row-caps a query so it cannot materialize an unbounded result set', () => { - const db = new PluginDataDb('rowcap'); + const db = openStore('rowcap'); db.exec('CREATE TABLE seq (n INTEGER)'); // Insert a modest table and a self-cross-join that would explode past the cap. const insert = 'INSERT INTO seq (n) VALUES ' + Array.from({ length: 400 }, (_, i) => `(${i})`).join(','); @@ -68,8 +76,8 @@ describe('PluginDataDb', () => { db.close(); }); - it('tx runs a batch atomically — reads see earlier writes, and any error rolls the whole batch back', () => { - const db = new PluginDataDb('txn'); + it('tx runs a batch atomically: reads see earlier writes, and any error rolls the whole batch back', () => { + const db = openStore('txn'); db.migrate('001', 'CREATE TABLE acct (id INTEGER PRIMARY KEY, bal INTEGER)'); db.exec('INSERT INTO acct (id, bal) VALUES (1, 100), (2, 0)'); @@ -135,7 +143,7 @@ describe('PluginDataDb', () => { }; it('stops a query at the first row past its time budget', () => { - const db = new PluginDataDb('budget-query', { timeBudgetMs: 100, now: steppingClock(30) }); + const db = openStore('budget-query', { timeBudgetMs: 100, now: steppingClock(30) }); db.exec('CREATE TABLE seq (n INTEGER)'); db.exec('INSERT INTO seq (n) VALUES ' + Array.from({ length: 50 }, (_, i) => `(${i})`).join(',')); // The start reads the clock and every row after the first reads it again, 30 ms @@ -147,7 +155,7 @@ describe('PluginDataDb', () => { }); it('holds a whole tx batch to one time budget and rolls it back when it runs out', () => { - const db = new PluginDataDb('budget-tx', { timeBudgetMs: 80, now: steppingClock(30) }); + const db = openStore('budget-tx', { timeBudgetMs: 80, now: steppingClock(30) }); db.exec('CREATE TABLE acct (id INTEGER PRIMARY KEY, bal INTEGER)'); db.exec('INSERT INTO acct (id, bal) VALUES (1, 100)'); // The clock is read at the start and after each statement: 30 ms, 60 ms, then @@ -167,7 +175,7 @@ describe('PluginDataDb', () => { }); it('holds an exec script to the time budget between its statements', () => { - const db = new PluginDataDb('budget-exec', { timeBudgetMs: 80, now: steppingClock(30) }); + const db = openStore('budget-exec', { timeBudgetMs: 80, now: steppingClock(30) }); db.exec('CREATE TABLE log (n INTEGER)'); // Start, then one read before each statement after the first: 30, 60, then 90 ms // before the fourth insert. The script stops there and the fourth insert never @@ -180,11 +188,10 @@ describe('PluginDataDb', () => { ).toThrow('exec exceeded its 80 ms time budget'); // One row back, so the check itself reads the clock only at its start. expect(db.query('SELECT group_concat(n) AS ns FROM (SELECT n FROM log ORDER BY n)')).toEqual([{ ns: '1,2,3' }]); - // A script that opened its own transaction and ran out of time inside it is rolled - // back, so the connection is not left in that transaction. + // A script that would open its own transaction is refused before any of it runs. expect(() => - db.exec('BEGIN; INSERT INTO log VALUES (5); INSERT INTO log VALUES (6); INSERT INTO log VALUES (7); COMMIT;'), - ).toThrow(/time budget/); + db.exec('INSERT INTO log VALUES (5); BEGIN; INSERT INTO log VALUES (6); INSERT INTO log VALUES (7); COMMIT;'), + ).toThrow(PLUGIN_DATA_ERRORS.txControl); expect(db.query('SELECT count(*) AS c FROM log WHERE n > 4')).toEqual([{ c: 0 }]); db.exec('INSERT INTO log VALUES (8)'); expect(db.query('SELECT count(*) AS c FROM log WHERE n > 4')).toEqual([{ c: 1 }]); @@ -193,22 +200,24 @@ describe('PluginDataDb', () => { it('reports a call whose last statement or row ran past the budget as done', () => { const clock = scriptedClock(); - const db = new PluginDataDb('budget-last', { timeBudgetMs: 100, now: clock.now }); + const db = openStore('budget-last', { timeBudgetMs: 100, now: clock.now }); db.exec('CREATE TABLE log (n INTEGER)'); // A single statement without args: the clock is read at the start only, so a // slow insert that has run and committed is not reported as failed. clock.reads = [0, 1000]; expect(db.exec('INSERT INTO log VALUES (1)')).toEqual({ changes: 0 }); - // A script whose closing COMMIT is what takes the time: the reads before the - // INSERT and before the COMMIT are in budget, and nothing is read after it. - clock.reads = [0, 0, 0, 1000]; - expect(db.exec('BEGIN; INSERT INTO log VALUES (2); COMMIT;')).toEqual({ changes: 0 }); + // A script whose last statement is what takes the time: the reads at the start + // and before the last statement are in budget, and nothing is read after it. + clock.reads = [0, 0, 1000]; + expect(db.exec('INSERT INTO log VALUES (2); DELETE FROM log WHERE n > 2;')).toEqual({ changes: 0 }); clock.reads = [0]; expect(db.query('SELECT n FROM log ORDER BY n')).toEqual([{ n: 1 }, { n: 2 }]); - // The same budget running out before the COMMIT still stops the script and rolls - // the transaction back. - clock.reads = [0, 0, 1000]; - expect(() => db.exec('BEGIN; INSERT INTO log VALUES (3); COMMIT;')).toThrow('exec exceeded its 100 ms time budget'); + // The same budget running out before the last statement still stops the script + // there: the insert never runs. + clock.reads = [0, 1000]; + expect(() => db.exec('UPDATE log SET n = n WHERE n = 1; INSERT INTO log VALUES (3);')).toThrow( + 'exec exceeded its 100 ms time budget', + ); clock.reads = [0]; expect(db.query('SELECT count(*) AS c FROM log')).toEqual([{ c: 2 }]); // A query read to its end is returned, even when its last row came in late: the @@ -226,7 +235,7 @@ describe('PluginDataDb', () => { }); it('runs an exec script statement by statement with the semantics Database#exec had', () => { - const db = new PluginDataDb('exec-script'); + const db = openStore('exec-script'); db.exec(` -- a setup script: comments, a semicolon in a string, and a trigger body CREATE TABLE notes (id INTEGER PRIMARY KEY, body TEXT); /* one; two */ @@ -251,11 +260,20 @@ describe('PluginDataDb', () => { expect(db.query('SELECT count(*) AS n FROM notes')).toEqual([{ n: 3 }]); // A statement left open at the end reports what Database#exec reported. expect(() => db.exec('CREATE TRIGGER t AFTER INSERT ON notes BEGIN SELECT 1;')).toThrow('incomplete input'); + // The END that closes a trigger body (after a CASE ... END too) is no transaction + // control, a COMMIT after the trigger is, and refusing it runs nothing. + db.exec('CREATE TEMP TRIGGER t2 AFTER UPDATE ON notes BEGIN SELECT CASE WHEN 1 THEN 1 END; END; SELECT 1;'); + expect(() => + db.exec( + 'CREATE TRIGGER t3 AFTER DELETE ON notes BEGIN UPDATE audit SET what = CASE WHEN 1 THEN what END; END; COMMIT;', + ), + ).toThrow(PLUGIN_DATA_ERRORS.txControl); + expect(db.query("SELECT count(*) AS n FROM sqlite_master WHERE name = 't3'")).toEqual([{ n: 0 }]); db.close(); }); it('applies an overridden row cap to query and tx alike', () => { - const db = new PluginDataDb('budget-rows', { maxRows: 3 }); + const db = openStore('budget-rows', { maxRows: 3 }); db.exec('CREATE TABLE seq (n INTEGER)'); db.exec('INSERT INTO seq (n) VALUES (1), (2), (3), (4)'); expect(() => db.query('SELECT n FROM seq')).toThrow('query returned more than 3 rows'); @@ -264,34 +282,40 @@ describe('PluginDataDb', () => { db.close(); }); - it("binds a plugin's NUL exactly: stored rows stay findable and NUL-separated keys never collide", () => { - const db = new PluginDataDb('nul-keys'); + it("refuses NUL in SQL, an argument or a migration id with the Postgres store's error, and stores nothing", () => { + const db = openStore('nul-keys'); db.exec('CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT)'); - // A row a plugin wrote with a NUL in its key, through its own SQL as well as through a bound arg. - db.exec("INSERT INTO kv VALUES ('q' || char(0) || 'r', 'spelled')"); - db.exec('INSERT INTO kv VALUES (?, ?)', ['trip-1\u0000slug', 'bound']); - // Keys that differ only in where the NUL sits are two keys. - db.exec('INSERT INTO kv VALUES (?, ?)', ['a\u0000bc', 'one']); - db.exec('INSERT INTO kv VALUES (?, ?)', ['ab\u0000c', 'two']); - db.tx([{ sql: 'INSERT INTO kv VALUES (?, ?)', args: ['x\u0000', 'tx'] }]); - - expect(db.query('SELECT v FROM kv WHERE k = ?', ['q\u0000r'])).toEqual([{ v: 'spelled' }]); - expect(db.query('SELECT v FROM kv WHERE k = ?', ['trip-1\u0000slug'])).toEqual([{ v: 'bound' }]); - expect(db.query('SELECT v FROM kv WHERE k = ?', ['trip-1slug'])).toEqual([]); - expect(db.query('SELECT v FROM kv WHERE k IN (?, ?) ORDER BY v', ['a\u0000bc', 'ab\u0000c'])).toEqual([ - { v: 'one' }, - { v: 'two' }, - ]); - expect(db.tx([{ sql: 'SELECT v FROM kv WHERE k = ?', args: ['x\u0000'] }])).toEqual({ - results: [{ rows: [{ v: 'tx' }] }], - }); - // Stored whole: SQLite's length() of text stops at a NUL, the bytes do not. - expect(db.query('SELECT length(CAST(k AS BLOB)) AS n FROM kv WHERE v = ?', ['bound'])).toEqual([{ n: 11 }]); + const refusals: Array<[() => unknown, string]> = [ + [() => db.exec('INSERT INTO kv VALUES (?, ?)', ['trip-1\u0000slug', 'bound']), PLUGIN_VALUE_NUL_REFUSAL], + [() => db.query('SELECT v FROM kv WHERE k = ?', ['a\u0000bc']), PLUGIN_VALUE_NUL_REFUSAL], + [() => db.query('SELECT v FROM kv WHERE k = :k', [{ k: 'x\u0000' }]), PLUGIN_VALUE_NUL_REFUSAL], + [() => db.exec('INSERT INTO kv VALUES (?, ?)', [['a', '\u0000'], 'nested']), PLUGIN_VALUE_NUL_REFUSAL], + [ + () => + db.tx([ + { sql: "INSERT INTO kv VALUES ('first', 'tx')" }, + { sql: 'INSERT INTO kv VALUES (?, ?)', args: ['x\u0000', 'tx'] }, + ]), + PLUGIN_VALUE_NUL_REFUSAL, + ], + [() => db.migrate('002\u0000', 'CREATE TABLE more (x INTEGER)'), PLUGIN_VALUE_NUL_REFUSAL], + [() => db.exec("INSERT INTO kv VALUES ('a\u0000b', 'literal')"), PLUGIN_SQL_NUL_REFUSAL], + [() => db.query("SELECT v FROM kv WHERE k = 'q\u0000'"), PLUGIN_SQL_NUL_REFUSAL], + ]; + for (const [call, reason] of refusals) expect(call).toThrow(reason); + expect(db.query('SELECT count(*) AS n FROM kv')).toEqual([{ n: 0 }]); + expect(db.query("SELECT count(*) AS n FROM sqlite_master WHERE name = 'more'")).toEqual([{ n: 0 }]); + // Binary data with a zero byte is a BLOB, and a row an older TREK stored with a NUL is still read. + db.exec('CREATE TABLE raw (b BLOB)'); + db.exec('INSERT INTO raw VALUES (?)', [Buffer.from([1, 0, 2])]); + expect(db.query('SELECT length(b) AS n FROM raw')).toEqual([{ n: 3 }]); + db.exec("INSERT INTO kv VALUES ('q' || char(0) || 'r', 'older')"); + expect(db.query('SELECT length(CAST(k AS BLOB)) AS n FROM kv WHERE v = ?', ['older'])).toEqual([{ n: 3 }]); db.close(); }); it('removePluginData deletes the whole data dir', () => { - const db = new PluginDataDb('temp'); + const db = openStore('temp'); db.migrate('001', 'CREATE TABLE t (id INTEGER)'); db.close(); expect(fs.existsSync(path.join(tmp, 'temp'))).toBe(true); @@ -300,7 +324,7 @@ describe('PluginDataDb', () => { }); it('snapshots a closed plugin.db together with its -wal/-shm sidecars (no data loss after an unclean shutdown)', () => { - const srcDir = path.join(tmp, 'wal-plugin'); // no open PluginDataDb handle → treated as closed + const srcDir = path.join(tmp, 'wal-plugin'); // no open store → treated as closed fs.mkdirSync(srcDir, { recursive: true }); fs.writeFileSync(path.join(srcDir, 'plugin.db'), 'DB'); fs.writeFileSync(path.join(srcDir, 'plugin.db-wal'), 'WAL-committed'); @@ -310,7 +334,7 @@ describe('PluginDataDb', () => { snapshotAllPluginDataDbs(dest); const outDir = path.join(dest, 'wal-plugin'); expect(fs.existsSync(path.join(outDir, 'plugin.db'))).toBe(true); - // No writer, so the WAL is a consistent set with the .db and must be copied — + // No writer, so the WAL is a consistent set with the .db and must be copied: // otherwise committed-but-uncheckpointed rows in the WAL are lost from the backup. expect(fs.readFileSync(path.join(outDir, 'plugin.db-wal'), 'utf8')).toBe('WAL-committed'); expect(fs.existsSync(path.join(outDir, 'plugin.db-shm'))).toBe(true); diff --git a/server/tests/unit/plugins/sql-script.test.ts b/server/tests/unit/plugins/sql-script.test.ts index 87be38412..2f0bc60a8 100644 --- a/server/tests/unit/plugins/sql-script.test.ts +++ b/server/tests/unit/plugins/sql-script.test.ts @@ -1,8 +1,8 @@ /** * splitAfterSemicolons: where a plugin's exec() script is cut into statements, so - * the time budget can be read between them (plugin-data.service.ts). + * the time budget can be read between them (src/db/engine/sqlite/plugin-data-store.ts). */ -import { splitAfterSemicolons } from '../../../src/nest/plugins/host/sql-script'; +import { splitAfterSemicolons } from '../../../src/db/engine/sqlite/sql-script'; import { describe, expect, it } from 'vitest';