From a884539a2b3b0fbca1a7de601bb6dcad663fe482 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 21:50:30 +0100 Subject: [PATCH] fix(db): run plugin SQL on a pool client of its own instead of patching the pool Plugin statements on Postgres went over the extended query protocol through a marker parameter and a hook that replaced query() on every client of TREK's pool. That kept a plugin to one statement per round trip, but it changed TREK's own hot path for a concern only plugin data has. The plugin data store now checks a pg client out of the ORM's pool for each transaction, through MikroORM's getNativeClient, and runs the transaction on it directly. Every plugin write goes as a query with queryMode 'extended' and every read through a pg-cursor on that client, so Postgres parses exactly one statement either way. The settings the transaction makes are local, and before the client goes back to the pool the store drops temporary tables, cursors, listeners, session advisory locks and sequence state, or closes a client it cannot bring back to that state. The row cap, quota, time budget, untimed migrate, per-plugin queue, pool share and script commits are as before, and plugin args are still sent without NUL. The marker, the hook and its install in the Postgres driver are gone. --- package-lock.json | 1 + server/package.json | 1 + .../src/db/engine/postgres/extended-query.ts | 55 ---- .../db/engine/postgres/plugin-data-store.ts | 295 ++++++++++++------ .../src/db/engine/postgres/postgres-driver.ts | 19 +- server/src/db/nul-free.ts | 6 +- .../nest/plugins/host/plugin-data.backend.ts | 8 +- server/src/typings/pg-cursor.d.ts | 14 + server/tests/postgres/plugin-data.test.ts | 151 ++++++--- server/tests/postgres/sqlite-move.test.ts | 6 +- .../db/engine/postgres/extended-query.test.ts | 59 ---- .../engine/postgres/plugin-data-store.test.ts | 224 +++++++++---- server/tests/unit/db/nul-free.test.ts | 9 +- 13 files changed, 505 insertions(+), 343 deletions(-) delete mode 100644 server/src/db/engine/postgres/extended-query.ts create mode 100644 server/src/typings/pg-cursor.d.ts delete mode 100644 server/tests/unit/db/engine/postgres/extended-query.test.ts diff --git a/package-lock.json b/package-lock.json index 0921101ff..04334ea42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25952,6 +25952,7 @@ "nodemailer": "^10.5.0", "otplib": "^12.0.1", "pdf-parse": "^2.4.5", + "pg-cursor": "^2.22.1", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", diff --git a/server/package.json b/server/package.json index 6e5a8244c..74a8f290d 100644 --- a/server/package.json +++ b/server/package.json @@ -106,6 +106,7 @@ "nodemailer": "^10.5.0", "otplib": "^12.0.1", "pdf-parse": "^2.4.5", + "pg-cursor": "^2.22.1", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", diff --git a/server/src/db/engine/postgres/extended-query.ts b/server/src/db/engine/postgres/extended-query.ts deleted file mode 100644 index 8af8702c5..000000000 --- a/server/src/db/engine/postgres/extended-query.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { CompiledQuery } from 'kysely'; - -/** - * Statements that must run alone: sent over Postgres's extended query - * protocol, whose Parse message takes exactly one statement, so a second one - * in the text is a syntax error instead of a statement that runs. - * - * `pg` picks the protocol by the values it is handed: with none it sends the - * text over the simple protocol, which runs every statement in it. Kysely - * hands it a query's parameters as they are, so {@link extendedQuery} adds a - * marker as the last parameter, and the connection hook - * {@link sendMarkedQueriesExtended} (installed on every client of TREK's pool - * by `postgres-driver.ts`) takes the marker off again and asks `pg` for the - * extended protocol. A query that reached a client without the hook still goes - * over the extended protocol, with one parameter too many, and Postgres - * refuses it: the marker fails closed. - * - * Plugin SQL (`plugin-data-store.ts`) runs through here. A read streamed - * through a cursor needs none of this: a cursor always uses the extended - * protocol. - */ - -/** The last parameter of a query {@link extendedQuery} built. */ -const EXTENDED_ONLY = Object.freeze({ trekExtendedProtocol: true }); - -/** `sql` with `args` as a query sent over the extended protocol, so it runs as one statement or not at all. */ -export function extendedQuery(sql: string, args: readonly unknown[]): CompiledQuery { - return CompiledQuery.raw(sql, [...args, EXTENDED_ONLY]); -} - -/** The values `pg` was handed, without the marker, when {@link extendedQuery} built them; otherwise null. */ -export function unmarkedValues(values: unknown): unknown[] | null { - if (!Array.isArray(values) || values.length === 0 || values[values.length - 1] !== EXTENDED_ONLY) return null; - return values.slice(0, -1); -} - -/** The part of a `pg` client this hook replaces. */ -interface QueryClient { - query: (...args: unknown[]) => unknown; -} - -/** - * Makes `client` send a query {@link extendedQuery} built over the extended - * protocol (`queryMode: 'extended'`, which `pg` honours even without values), - * and every other query as before. - */ -export function sendMarkedQueriesExtended(client: QueryClient): void { - const query = client.query.bind(client); - client.query = (...args: unknown[]) => { - const [text, values, ...rest] = args; - const unmarked = unmarkedValues(values); - if (typeof text !== 'string' || unmarked === null) return query(...args); - return query({ text, values: unmarked, queryMode: 'extended' }, ...rest); - }; -} diff --git a/server/src/db/engine/postgres/plugin-data-store.ts b/server/src/db/engine/postgres/plugin-data-store.ts index e392dbb39..3ea7eb25c 100644 --- a/server/src/db/engine/postgres/plugin-data-store.ts +++ b/server/src/db/engine/postgres/plugin-data-store.ts @@ -1,25 +1,33 @@ +import { nulFreeParameter } from '../../nul-free'; import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data'; -import { extendedQuery } from './extended-query'; import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema'; import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard'; -import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; +import { CompiledQuery, type Kysely } from 'kysely'; +import Cursor from 'pg-cursor'; /** * A plugin's own data on Postgres (`db:own`): a schema of its own in TREK's * database (`plugin-schema.ts`), reached through TREK's pool. * - * Every call runs in one transaction that first makes sure the schema and its - * migration table exist, then puts only that schema on the `search_path`, with - * `pg_temp` explicitly last so a temporary table is never found before the - * plugin's own, and sets `statement_timeout` to the call's time budget. The - * plugin's SQL is checked by `plugin-sql-guard.ts` before it runs, so it cannot - * name another schema, and each statement goes to Postgres on its own over the - * extended query protocol (`extended-query.ts`), which runs one statement and - * no more. Reads stream through a cursor and stop at the row cap. A call that - * may write measures the schema's size before and after, and rolls back one - * that leaves it past the quota and larger than it found it, so a plugin at - * its quota can still delete. + * Every call checks a `pg` client out of the pool for itself + * ({@link poolClients}) and runs one transaction on it directly, not through + * Kysely. The transaction first makes sure the schema and its migration table + * exist, then puts only that schema on the `search_path`, with `pg_temp` + * explicitly last so a temporary table is never found before the plugin's + * own, and sets `statement_timeout` to the call's time budget; every setting + * is `local` and ends with the transaction. The plugin's SQL is checked by + * `plugin-sql-guard.ts` before it runs, so it cannot name another schema, and + * each statement goes to Postgres on its own over the extended query + * protocol, which runs one statement and no more: a write as a query with + * `queryMode: 'extended'`, which `pg` honours even without values, and a read + * through a `pg-cursor`, which always uses that protocol. Reads stop at the + * row cap. A call that may write measures the schema's size before and + * after, and rolls back one that leaves it past the quota and larger than it + * found it, so a plugin at its quota can still delete. Before the client goes + * back to the pool, whatever session state a statement could have left on it + * is dropped ({@link END_SESSION}); a client that cannot be brought back to a + * clean state is closed instead of reused. * * The calls of one plugin run one at a time, as they do on its SQLite file, * and all plugins together hold at most half of TREK's pool @@ -57,6 +65,54 @@ const CHUNK_ROWS = 500; /** What better-sqlite3 says about a closed handle, which the host recognizes. */ const CLOSED = 'The database connection is not open'; +/** + * What a client runs before it goes back to the pool, outside any + * transaction: `DISCARD ALL` without the two parts that would undo the pool's + * own session state. `RESET ALL` would drop the `search_path` every pool + * connection sets with `pg_temp` last (`session-search-path.ts`), and + * `DEALLOCATE ALL` the statements `pg` prepared by name for other callers. + * The plugin's settings need neither: they are all `local`. The guard + * refuses every statement that could leave anything this drops, so this is + * the layer behind it. + */ +export const END_SESSION = 'close all; unlisten *; select pg_advisory_unlock_all(); discard temp; discard sequences'; + +/** A query result, as far as the store reads it. */ +interface PgResult { + rows: unknown[]; + rowCount: number | null; +} + +/** The part of a `pg` client the store uses. It is the store's alone from checkout to release. */ +export interface PluginSqlClient { + /** SQL of TREK's own, without values: over the simple protocol. */ + query(text: string): Promise; + /** A query with values, or a plugin statement with `queryMode: 'extended'`. */ + query(config: { text: string; values: unknown[]; queryMode?: 'extended' }): Promise; + /** Submits a cursor and hands it back. */ + query(cursor: Cursor): Cursor; + /** Back to the pool, or closed when `destroy` is true. */ + release(destroy?: boolean): void; +} + +/** Checks a client out of the pool; resolves once one is free. */ +export type ConnectPluginClient = () => Promise; + +/** The part of `pg`'s pool the store checks clients out of. */ +interface PgPool { + connect(): Promise; +} + +/** + * Clients of the `pg` pool behind MikroORM's Postgres connection, which its + * `getNativeClient` hands out. Its options apply to them as to every other + * client of the pool: the TLS settings, and the `int8` and `numeric` parsers + * (`postgres-driver.ts`). Nothing of the pool is changed for them. + */ +export function poolClients(connection: { getNativeClient(): Promise }): ConnectPluginClient { + return async () => ((await connection.getNativeClient()) as PgPool).connect(); +} + /** How many plugin transactions may hold a pool connection at once: half of TREK's pool, at least one. */ export function pluginDataSlots(poolMax: number): number { return Math.max(1, Math.floor(poolMax / 2)); @@ -83,8 +139,6 @@ export class PluginDataSlots { } } -type Trx = Transaction; - /** The rows one call has returned so far, against the cap they share. */ interface RowBudget { count: number; @@ -109,7 +163,7 @@ export class PostgresPluginDataStore implements PluginDataStore { constructor( readonly pluginId: string, - private readonly db: Kysely, + private readonly connect: ConnectPluginClient, private readonly slots: PluginDataSlots, limits: Partial = {}, ) { @@ -122,7 +176,7 @@ export class PostgresPluginDataStore implements PluginDataStore { return this.open; } - /** Stops new calls. One already running finishes: it holds a pool connection of its own, not this store. */ + /** Stops new calls. One already running finishes: it holds a pool client of its own, not this store. */ close(): void { this.open = false; } @@ -131,9 +185,9 @@ export class PostgresPluginDataStore implements PluginDataStore { const statement = this.single(sql, args); const { maxRows } = this.limits; return this.call('query', (budget) => - this.transaction(budget.ms, [statement], (trx) => { + this.transaction(budget.ms, [statement], (client) => { const rows = { count: 0, max: maxRows, message: `query returned more than ${maxRows} rows` }; - return this.stream(trx, statement, args, rows, budget.check); + return this.stream(client, statement, args, rows, budget.check); }), ); } @@ -142,7 +196,9 @@ export class PostgresPluginDataStore implements PluginDataStore { if (args.length > 0) { const statement = this.single(sql, args); return this.call('exec', (budget) => - this.transaction(budget.ms, [statement], async (trx) => ({ changes: await this.run(trx, statement, args) })), + this.transaction(budget.ms, [statement], async (client) => ({ + changes: await this.run(client, statement, args), + })), ); } const statements = this.parse(sql); @@ -151,7 +207,7 @@ export class PostgresPluginDataStore implements PluginDataStore { // the clock is read before every statement but the first. await inOrder(statements, (statement, index) => { if (index > 0) budget.check(); - return this.transaction(budget.remaining(), [statement], (trx) => this.run(trx, statement, [])); + return this.transaction(budget.remaining(), [statement], (client) => this.run(client, statement, [])); }); return { changes: 0 }; }); @@ -166,15 +222,15 @@ export class PostgresPluginDataStore implements PluginDataStore { const statements = ops.map((op) => this.single(op?.sql, op?.args ?? [])); const { maxRows } = this.limits; return this.call('tx', (budget) => - this.transaction(budget.ms, statements, async (trx) => { + this.transaction(budget.ms, statements, async (client) => { const rows = { count: 0, max: maxRows, message: `tx returned more than ${maxRows} rows in total` }; const results = await inOrder(statements, async (statement, index) => { const args = ops[index].args ?? []; // One budget for the whole batch: each statement after the first gets what is left of it. - if (index > 0) await this.setTimeout(trx, budget.remaining()); + if (index > 0) await this.setTimeout(client, budget.remaining()); const result = statement.returnsRows - ? { rows: await this.stream(trx, statement, args, rows, budget.check) } - : { changes: await this.run(trx, statement, args) }; + ? { rows: await this.stream(client, statement, args, rows, budget.check) } + : { changes: await this.run(client, statement, args) }; budget.check(); return result; }); @@ -187,18 +243,17 @@ export class PostgresPluginDataStore implements PluginDataStore { async migrate(id: string, sql: string): Promise<{ applied: boolean }> { const statements = this.parse(sql); return this.enqueue(() => - this.transaction(null, statements, async (trx) => { - const seen = await trx.executeQuery( - CompiledQuery.raw(`select 1 from ${this.quoted}._plugin_migrations where id = $1`, [id]), - ); + this.transaction(null, statements, async (client) => { + const seen = await client.query({ + text: `select 1 from ${this.quoted}._plugin_migrations where id = $1`, + values: [id], + }); if (seen.rows.length > 0) return { applied: false }; - await inOrder(statements, (statement) => this.run(trx, statement, [])); - await trx.executeQuery( - CompiledQuery.raw(`insert into ${this.quoted}._plugin_migrations (id, applied_at) values ($1, $2)`, [ - id, - Date.now(), - ]), - ); + await inOrder(statements, (statement) => this.run(client, statement, [])); + await client.query({ + text: `insert into ${this.quoted}._plugin_migrations (id, applied_at) values ($1, $2)`, + values: [id, Date.now()], + }); return { applied: true }; }), ); @@ -257,81 +312,111 @@ export class PostgresPluginDataStore implements PluginDataStore { } /** - * One transaction on this plugin's schema: created if missing, alone on the - * `search_path`, with `statement_timeout` at `timeoutMs` (none for null). - * The statements' qualified names are checked against the database's - * schemas first, and a call that may write is held to the quota. + * One transaction on this plugin's schema, on a client of its own: created + * if missing, alone on the `search_path`, with `statement_timeout` at + * `timeoutMs` (none for null). The statements' qualified names are checked + * against the database's schemas first, and a call that may write is held + * to the quota. The client goes back to the pool with its session state + * dropped, or is closed when the rollback or that cleanup fails. */ - private transaction( + private async transaction( timeoutMs: number | null, statements: readonly PluginStatement[], - work: (trx: Trx) => Promise, + work: (client: PluginSqlClient) => Promise, ): Promise { - return this.db.transaction().execute(async (trx) => { - const setup = [ - `create schema if not exists ${this.quoted}`, - `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, - `set local search_path to ${this.quoted}, pg_temp`, - 'set local standard_conforming_strings to on', - ]; - if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); - await trx.executeQuery(CompiledQuery.raw(setup.join(';\n'))); - if (statements.some((s) => s.qualifies)) { - const schemas = await this.schemaNames(trx); - for (const statement of statements) checkQualifiers(statement, this.schema, schemas); + const client = await this.connect(); + let reusable = true; + try { + await client.query('begin'); + try { + const result = await this.inTransaction(client, timeoutMs, statements, work); + await client.query('commit'); + return result; + } catch (err) { + await client.query('rollback').catch(() => { + reusable = false; + }); + throw err; } - const writes = statements.some((s) => s.mayWrite); - const before = writes ? await this.sizeBytes(trx) : 0; - const result = await work(trx); - if (writes) { - const after = await this.sizeBytes(trx); - if (after > this.limits.quotaBytes && after > before) { - throw new Error(`the plugin database is past its ${this.limits.quotaBytes / 1024 / 1024} MB quota`); - } + } finally { + if (reusable) { + await client.query(END_SESSION).catch(() => { + reusable = false; + }); } - return result; + client.release(!reusable); + } + } + + /** What {@link transaction} runs between `begin` and `commit`. */ + private async inTransaction( + client: PluginSqlClient, + timeoutMs: number | null, + statements: readonly PluginStatement[], + work: (client: PluginSqlClient) => Promise, + ): Promise { + const setup = [ + `create schema if not exists ${this.quoted}`, + `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, + `set local search_path to ${this.quoted}, pg_temp`, + 'set local standard_conforming_strings to on', + ]; + if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); + await client.query(setup.join(';\n')); + if (statements.some((s) => s.qualifies)) { + const schemas = await this.schemaNames(client); + for (const statement of statements) checkQualifiers(statement, this.schema, schemas); + } + const writes = statements.some((s) => s.mayWrite); + const before = writes ? await this.sizeBytes(client) : 0; + const result = await work(client); + if (writes) { + const after = await this.sizeBytes(client); + if (after > this.limits.quotaBytes && after > before) { + throw new Error(`the plugin database is past its ${this.limits.quotaBytes / 1024 / 1024} MB quota`); + } + } + return result; + } + + private async setTimeout(client: PluginSqlClient, ms: number): Promise { + await client.query({ + text: "select set_config('statement_timeout', $1, true)", + values: [String(Math.max(1, Math.floor(ms)))], }); } - private async setTimeout(trx: Trx, ms: number): Promise { - await trx.executeQuery( - CompiledQuery.raw("select set_config('statement_timeout', $1, true)", [String(Math.max(1, Math.floor(ms)))]), - ); - } - /** Every schema name of the database, which a qualified name in plugin SQL may not use. */ - private async schemaNames(trx: Trx): Promise> { - const result = await trx.executeQuery<{ nspname: string }>(CompiledQuery.raw('select nspname from pg_namespace')); - return new Set(result.rows.map((row) => row.nspname)); + private async schemaNames(client: PluginSqlClient): Promise> { + const result = await client.query('select nspname from pg_namespace'); + return new Set((result.rows as Array<{ nspname: string }>).map((row) => row.nspname)); } /** What this plugin's tables, indexes and materialized views take on disk. */ - private async sizeBytes(trx: Trx): Promise { - const result = await trx.executeQuery<{ bytes: number | string }>( - CompiledQuery.raw( + private async sizeBytes(client: PluginSqlClient): Promise { + const result = await client.query({ + text: 'select coalesce(sum(pg_total_relation_size(c.oid)), 0) as bytes from pg_class c ' + - "join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'm')", - [this.schema], - ), - ); - return Number(result.rows[0]?.bytes ?? 0); + "join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'm')", + values: [this.schema], + }); + const [row] = result.rows as Array<{ bytes: number | string }>; + return Number(row?.bytes ?? 0); } /** * Runs a statement for its effect: the rows it changed, or 0 for a read, - * whose rows are streamed past and dropped. Both ways use the extended - * protocol: a cursor always does, and a write goes through `extendedQuery`. + * whose rows are read past and dropped. Both go over the extended protocol: + * a write as an extended query, a read through a cursor. */ - private async run(trx: Trx, statement: PluginStatement, args: readonly unknown[]): Promise { + private async run(client: PluginSqlClient, statement: PluginStatement, args: readonly unknown[]): Promise { if (!statement.mayWrite) { - const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); - for await (const _chunk of chunks) { - // Read to the end so the statement runs whole; its rows are not wanted. - } + // Read to the end so the statement runs whole; its rows are not wanted. + await readCursor(client.query(new Cursor(statement.text, boundValues(args))), () => undefined); return 0; } - const result = await trx.executeQuery(extendedQuery(statement.text, args)); - return Number(result.numAffectedRows ?? 0); + const result = await client.query({ text: statement.text, values: boundValues(args), queryMode: 'extended' }); + return result.rowCount ?? 0; } /** @@ -339,26 +424,48 @@ export class PostgresPluginDataStore implements PluginDataStore { * row cap the read stops and throws, and the clock is read between chunks. */ private async stream( - trx: Trx, + client: PluginSqlClient, statement: PluginStatement, args: readonly unknown[], budget: RowBudget, checkTime: () => void, ): Promise { const rows: unknown[] = []; - const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); - for await (const chunk of chunks) { + await readCursor(client.query(new Cursor(statement.text, boundValues(args))), (chunk) => { if (rows.length > 0) checkTime(); - for (const row of chunk.rows) { + for (const row of chunk) { rows.push(row); budget.count++; if (budget.count > budget.max) throw new Error(budget.message); } - } + }); return rows; } } +/** A plugin's args as they are bound: without NUL, which Postgres text cannot hold (`nul-free.ts`). */ +function boundValues(args: readonly unknown[]): unknown[] { + return args.map(nulFreeParameter); +} + +/** + * Reads `cursor` to its end a chunk at a time, handing each chunk to `take` + * (which may throw to stop the read), and closes it however the read ends. + */ +async function readCursor(cursor: Cursor, take: (rows: unknown[]) => void): Promise { + const next = async (): Promise => { + const rows = await cursor.read(CHUNK_ROWS); + if (rows.length === 0) return; + take(rows); + return next(); + }; + try { + await next(); + } finally { + await cursor.close(); + } +} + /** `work` for each item, one after the other, each started once the one before it has finished. */ async function inOrder(items: readonly T[], work: (item: T, index: number) => Promise): Promise { const results: R[] = []; diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index 2c8029cd8..0922a48e5 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,14 +1,8 @@ import { nulFreeJson, stripNul, withNulFreeParameters } from '../../nul-free'; -import { sendMarkedQueriesExtended } from './extended-query'; import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; import { type Configuration, type TransformContext } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; -/** The `pg` pool event a new client is handed to before anyone uses it. */ -interface ClientEvents { - on(event: 'connect', listener: (client: { query: (...args: unknown[]) => unknown }) => void): void; -} - /** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */ const INT8_OID = 20; /** `numeric`: what `sum()` of a bigint and `avg()` answer. */ @@ -60,18 +54,11 @@ class NumberTypesConnection extends PostgreSqlConnection { /** * The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL - * position (`null-order.ts`), sending every parameter without NUL - * (`nul-free.ts`), on a pool whose every client sends the queries - * `extendedQuery` builds over the extended protocol (`extended-query.ts`). + * position (`null-order.ts`) and sending every parameter without NUL + * (`nul-free.ts`). */ override createKyselyDialect(overrides: Parameters[0]) { - const given = (overrides ?? {}) as { onPoolCreated?: (pool: unknown) => unknown }; - const onPoolCreated = (pool: unknown) => { - (pool as ClientEvents).on('connect', sendMarkedQueriesExtended); - return given.onPoolCreated?.(pool); - }; - const withHook = { ...given, onPoolCreated } as Parameters[0]; - return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(withHook))) as ReturnType< + return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(overrides))) as ReturnType< PostgreSqlConnection['createKyselyDialect'] >; } diff --git a/server/src/db/nul-free.ts b/server/src/db/nul-free.ts index 32f1a770c..9305cfbb6 100644 --- a/server/src/db/nul-free.ts +++ b/server/src/db/nul-free.ts @@ -20,9 +20,11 @@ import type { AbortableOperationOptions, CompiledQuery, DatabaseConnection, Dial * - the Kysely dialects of both engines (`orm-driver.ts`, * `engine/postgres/postgres-driver.ts`) send every query's parameters * through {@link withNulFreeParameters}, which covers the Kysely - * statements of the repositories and a plugin's own SQL on Postgres; + * statements of the repositories; * - a plugin's own SQLite file (`nest/plugins/host/plugin-data.service.ts`) - * binds its arguments through {@link nulFreeParameter}; + * and its schema on Postgres (`engine/postgres/plugin-data-store.ts`, + * which sends on a `pg` client of its own) bind its arguments through + * {@link nulFreeParameter}; * - the move from SQLite to Postgres (`engine/postgres/sqlite-move/values.ts`) * copies text the same way. * diff --git a/server/src/nest/plugins/host/plugin-data.backend.ts b/server/src/nest/plugins/host/plugin-data.backend.ts index e632fc310..d93b3305b 100644 --- a/server/src/nest/plugins/host/plugin-data.backend.ts +++ b/server/src/nest/plugins/host/plugin-data.backend.ts @@ -6,6 +6,7 @@ import { dropPluginSchema, PluginDataSlots, pluginDataSlots, + poolClients, PostgresPluginDataStore, } from '../../../db/engine/postgres/plugin-data-store'; import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../../database/database-engine.module'; @@ -52,8 +53,9 @@ export class SqlitePluginDataBackend implements PluginDataBackend { /** * Each plugin's own schema in TREK's Postgres database - * (`db/engine/postgres/plugin-data-store.ts`), reached through the ORM's - * pool. The plugin's data directory is still created, for the files a plugin + * (`db/engine/postgres/plugin-data-store.ts`). Each call checks a client + * out of the ORM's pool for itself; erasing a plugin runs on the ORM's Kysely + * client. The plugin's data directory is still created, for the files a plugin * keeps beside its database, and it is what marks a plugin uninstalled with * its data kept (the erasure queue reads it). */ @@ -70,7 +72,7 @@ export class PostgresPluginDataBackend implements PluginDataBackend { open(pluginId: string): PluginDataStore { fs.mkdirSync(pluginDataDir(pluginId), { recursive: true }); - return new PostgresPluginDataStore(pluginId, this.client(), this.sharedSlots()); + return new PostgresPluginDataStore(pluginId, poolClients(this.orm.em.getConnection()), this.sharedSlots()); } async erase(pluginId: string): Promise { diff --git a/server/src/typings/pg-cursor.d.ts b/server/src/typings/pg-cursor.d.ts new file mode 100644 index 000000000..993031deb --- /dev/null +++ b/server/src/typings/pg-cursor.d.ts @@ -0,0 +1,14 @@ +// `pg-cursor` ships no type definitions. These are the parts the Postgres +// plugin data store uses (`db/engine/postgres/plugin-data-store.ts`): a cursor +// is handed to a `pg` client's `query`, which submits it over the extended +// query protocol, and is then read a number of rows at a time. + +declare module 'pg-cursor' { + export default class Cursor { + constructor(text: string, values?: unknown[]); + /** The next `rows` rows, or none once the statement has returned them all. */ + read(rows: number): Promise; + /** Closes the portal; safe after the last read and after an error. */ + close(): Promise; + } +} diff --git a/server/tests/postgres/plugin-data.test.ts b/server/tests/postgres/plugin-data.test.ts index 8d70d35fb..df9b443f5 100644 --- a/server/tests/postgres/plugin-data.test.ts +++ b/server/tests/postgres/plugin-data.test.ts @@ -1,27 +1,32 @@ /** - * PGPLUG-001..011 — a plugin's own data (`db:own`) on Postgres, against the + * PGPLUG-001..012 — a plugin's own data (`db:own`) on Postgres, against the * real app on the lane's database: each plugin gets a schema of its own and no * SQLite file, its SQL keeps the SQLite store's semantics (migrate once per * id, a script's statements committed one by one, an atomic tx, the row cap, * the time budget, the quota), it cannot reach TREK's tables or another - * plugin's schema however it spells the name, and erasing a plugin drops its - * schema and nothing else. + * plugin's schema however it spells the name, every statement runs alone, + * the pool client a call used goes back without anything the call set, and + * erasing a plugin drops its schema and nothing else. */ import { buildApp } from '../../src/bootstrap'; -import { extendedQuery } from '../../src/db/engine/postgres/extended-query'; -import { PluginDataSlots, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; +import { + type ConnectPluginClient, + PluginDataSlots, + type PluginSqlClient, + poolClients, + PostgresPluginDataStore, +} from '../../src/db/engine/postgres/plugin-data-store'; import { withRequestContext } from '../../src/nest/database/request-context'; import { PLUGIN_DATA, type PluginDataBackend } from '../../src/nest/plugins/host/plugin-data.backend'; import { closePluginDataDb } from '../../src/nest/plugins/host/plugin-host-state'; import { PluginRpcHostFactory } from '../../src/nest/plugins/host/plugin-rpc-host.factory'; import { MikroORM } from '@mikro-orm/core'; -import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { INestApplication } from '@nestjs/common'; -import { CompiledQuery } from 'kysely'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import Cursor from 'pg-cursor'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; let app: INestApplication; @@ -57,10 +62,48 @@ async function pluginSchemas(): Promise { return rows.map((row) => row.nspname); } +/** Clients of the app's pool, as the backend hands them to a store. */ +function appClients(): ConnectPluginClient { + return poolClients(orm.em.getConnection()); +} + /** A store on the app's pool with limits small enough to reach in a test. */ function cappedStore(id: string, limits: { maxRows?: number; timeBudgetMs?: number; quotaBytes?: number }) { - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); - return new PostgresPluginDataStore(id, client, new PluginDataSlots(1), limits); + return new PostgresPluginDataStore(id, appClients(), new PluginDataSlots(1), limits); +} + +/** `client` for every call of a store, kept checked out by the test: a release only records whether it would close it. */ +function keep(client: PluginSqlClient, released: boolean[] = []): ConnectPluginClient { + return async () => + ({ + query: (arg: never) => client.query(arg), + release: (destroy?: boolean) => released.push(destroy === true), + }) as unknown as PluginSqlClient; +} + +/** + * Pool clients that hand Postgres `swap[text]` in place of a plugin statement + * whose text is `text`, as a store would if the guard let a second statement + * through: as an extended query or as a cursor, whichever the store sends. + */ +function smuggling(swap: Record): ConnectPluginClient { + const connect = appClients(); + return async () => { + const client = await connect(); + const query = (arg: unknown): unknown => { + if (arg instanceof Cursor) { + const cursor = arg as unknown as { text: string }; + cursor.text = swap[cursor.text] ?? cursor.text; + return client.query(arg); + } + const config = arg as { text?: unknown; values: unknown[] }; + if (typeof config.text === 'string' && swap[config.text]) { + return client.query({ ...config, text: swap[config.text] }); + } + return client.query(arg as never); + }; + return { query, release: (destroy?: boolean) => client.release(destroy) } as unknown as PluginSqlClient; + }; } describe('plugin data on Postgres', () => { @@ -275,24 +318,36 @@ describe('plugin data on Postgres', () => { expect(await store.query('SELECT body FROM notes')).toEqual([{ body: 'x' }]); }); - it('PGPLUG-010: a statement extendedQuery builds runs alone: a second one in its text is refused by Postgres, not run', async () => { + it('PGPLUG-010: every plugin statement runs alone: a second one smuggled into its text is refused by Postgres, not run', async () => { const store = data.open('pg-proto'); await store.migrate('001', 'CREATE TABLE t (x int)'); - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); const two = 'insert into trek_plugin_pg_proto.t values (1); insert into trek_plugin_pg_proto.t values (2)'; - // What a guard that let a second statement through would hand the pool: Postgres parses one statement only. - await expect(client.executeQuery(extendedQuery(two, []))).rejects.toThrow( + // What the store would hand Postgres if the guard let a second statement through, as a write and as a read. + const fooled = new PostgresPluginDataStore( + 'pg-proto', + smuggling({ 'INSERT INTO t VALUES (7)': two, 'SELECT x FROM t': `select x from t; ${two}` }), + new PluginDataSlots(1), + ); + await expect(fooled.exec('INSERT INTO t VALUES (7)')).rejects.toThrow( /cannot insert multiple commands into a prepared statement/, ); + await expect(fooled.exec('INSERT INTO t VALUES (7)', [])).rejects.toThrow(/cannot insert multiple commands/); + await expect(fooled.tx([{ sql: 'INSERT INTO t VALUES (7)' }])).rejects.toThrow(/cannot insert multiple commands/); + await expect(fooled.query('SELECT x FROM t')).rejects.toThrow(/cannot insert multiple commands/); expect(await store.query('SELECT count(*) AS n FROM t')).toEqual([{ n: 0 }]); - // The same text without the marker goes over the simple protocol and runs both, which is what the marker prevents. - await client.executeQuery(CompiledQuery.raw(two, [])); + + // The same text over the simple protocol runs both, which is what sending each statement extended prevents. + const client = await appClients()(); + try { + await client.query(two); + } finally { + client.release(); + } expect(await store.query('SELECT count(*) AS n FROM t')).toEqual([{ n: 2 }]); // A single statement runs and reports its rows, with or without args. - const one = await client.executeQuery(extendedQuery('delete from trek_plugin_pg_proto.t where x = $1', [1])); - expect(one.numAffectedRows).toBe(1n); + expect(await store.exec('DELETE FROM t WHERE x = $1', [1])).toEqual({ changes: 1 }); expect(await store.exec('DELETE FROM t')).toEqual({ changes: 0 }); expect(await store.exec('INSERT INTO t VALUES (5)', [])).toEqual({ changes: 0 }); expect(await store.tx([{ sql: 'INSERT INTO t VALUES (6)' }, { sql: 'DELETE FROM t' }])).toEqual({ @@ -316,23 +371,49 @@ describe('plugin data on Postgres', () => { const store = data.open('pg-shadow'); await store.migrate('001', 'CREATE TABLE notes (body text)'); await store.exec('INSERT INTO notes VALUES ($1)', ['real']); - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); - // One pool connection holding temporary tables named like TREK's and the plugin's, as one left behind would. - await client.connection().execute(async (conn) => { - try { - await conn.executeQuery(CompiledQuery.raw("create temp table users as select 'forged'::text as email")); - await conn.executeQuery(CompiledQuery.raw("create temp table notes as select 'forged'::text as body")); - const resolved = await conn.executeQuery<{ ns: string }>( - CompiledQuery.raw( - "select relnamespace::regnamespace::text as ns from pg_class where oid = 'users'::regclass", - ), - ); - expect(resolved.rows).toEqual([{ ns: 'public' }]); - const onConnection = new PostgresPluginDataStore('pg-shadow', conn, new PluginDataSlots(1)); - expect(await onConnection.query('SELECT body FROM notes')).toEqual([{ body: 'real' }]); - } finally { - await conn.executeQuery(CompiledQuery.raw('drop table if exists pg_temp.users, pg_temp.notes')); - } - }); + // A pool client holding temporary tables named like TREK's and the plugin's, as one left behind would. + const client = await appClients()(); + try { + await client.query("create temp table users as select 'forged'::text as email"); + await client.query("create temp table notes as select 'forged'::text as body"); + const released: boolean[] = []; + const onClient = new PostgresPluginDataStore('pg-shadow', keep(client, released), new PluginDataSlots(1)); + expect(await onClient.query('SELECT body FROM notes')).toEqual([{ body: 'real' }]); + // The call dropped them with the rest of the session's state before it gave the client back. + const temps = await client.query( + "select count(*)::int as n from pg_class where relpersistence = 't' and relnamespace = pg_my_temp_schema()", + ); + expect(temps.rows).toEqual([{ n: 0 }]); + expect(released).toEqual([false]); + } finally { + await client.query('drop table if exists pg_temp.users, pg_temp.notes'); + client.release(); + } + }); + + it('PGPLUG-012: a call leaves nothing it set on the pool client it used, a cancelled one neither', async () => { + const store = data.open('pg-session'); + await store.migrate('001', 'CREATE TABLE notes (body text)'); + const settings = + "select current_setting('search_path') as path, current_setting('statement_timeout') as timeout, " + + "current_setting('standard_conforming_strings') as strings, now() = statement_timestamp() as outside"; + const client = await appClients()(); + try { + const before = await client.query(settings); + const released: boolean[] = []; + const onClient = new PostgresPluginDataStore('pg-session', keep(client, released), new PluginDataSlots(1), { + timeBudgetMs: 300, + }); + await onClient.tx([{ sql: "INSERT INTO notes VALUES ('a')" }, { sql: 'SELECT body FROM notes' }]); + await expect(onClient.query('SELECT count(*) FROM generate_series(1, 2000000000)')).rejects.toThrow( + 'query exceeded its 300 ms time budget', + ); + await onClient.migrate('002', 'CREATE INDEX notes_body ON notes (body)'); + // Same search_path and settings as before, and no transaction left open (now() moves with each statement). + expect((await client.query(settings)).rows).toEqual(before.rows); + expect(released).toEqual([false, false, false]); + } finally { + client.release(); + } }); }); diff --git a/server/tests/postgres/sqlite-move.test.ts b/server/tests/postgres/sqlite-move.test.ts index 2b6c05cac..40d51ef56 100644 --- a/server/tests/postgres/sqlite-move.test.ts +++ b/server/tests/postgres/sqlite-move.test.ts @@ -14,7 +14,7 @@ import { buildApp } from '../../src/bootstrap'; import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; -import { PluginDataSlots, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; +import { PluginDataSlots, poolClients, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; import { AppSettings } from '../../src/db/entities/AppSettings.entity'; @@ -45,7 +45,6 @@ import { makeTrip, readTripDays } from '../helpers/factories/trips'; import { makeUser } from '../helpers/factories/users'; import { databaseUrl, withAdmin } from './databases'; import { MikroORM, type EntityClass, type Options } from '@mikro-orm/core'; -import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { INestApplication } from '@nestjs/common'; import Database from 'better-sqlite3'; @@ -270,8 +269,7 @@ describe('moving a SQLite install to Postgres', () => { const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id)); expect(user.id).toBeGreaterThan(highest); - const client = (pgOrm.em.getConnection() as AbstractSqlConnection).getClient(); - const store = new PostgresPluginDataStore(PLUGIN_ID, client, new PluginDataSlots(1)); + const store = new PostgresPluginDataStore(PLUGIN_ID, poolClients(pgOrm.em.getConnection()), new PluginDataSlots(1)); expect(await store.query('select id, body, score from notes order by id')).toEqual([ { id: 1, body: 'first', score: 1.5 }, { id: 2, body: 'second', score: null }, diff --git a/server/tests/unit/db/engine/postgres/extended-query.test.ts b/server/tests/unit/db/engine/postgres/extended-query.test.ts deleted file mode 100644 index cad2e542e..000000000 --- a/server/tests/unit/db/engine/postgres/extended-query.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * The extended-protocol marker of plugin SQL - * (src/db/engine/postgres/extended-query.ts), against a stand-in `pg` client - * that records what it is handed. That Postgres then runs one statement and - * no more is PGPLUG-010 in the Postgres lane. - */ -import { - extendedQuery, - sendMarkedQueriesExtended, - unmarkedValues, -} from '../../../../../src/db/engine/postgres/extended-query'; - -import { describe, expect, it } from 'vitest'; - -/** A `pg` client that records the arguments of every `query` call. */ -function recordingClient() { - const calls: unknown[][] = []; - const client = { - query: (...args: unknown[]): unknown => { - calls.push(args); - return Promise.resolve({ rows: [] }); - }, - }; - return { client, calls }; -} - -describe('extended query', () => { - it('EXTQ-001: a marked query reaches pg as one extended-protocol query with its own values, even with none', async () => { - const { client, calls } = recordingClient(); - sendMarkedQueriesExtended(client); - const none = extendedQuery('insert into t values (1)', []); - const some = extendedQuery('insert into t values ($1, $2)', ['a', 2]); - await client.query(none.sql, none.parameters); - await client.query(some.sql, some.parameters); - expect(calls).toEqual([ - [{ text: 'insert into t values (1)', values: [], queryMode: 'extended' }], - [{ text: 'insert into t values ($1, $2)', values: ['a', 2], queryMode: 'extended' }], - ]); - }); - - it('EXTQ-002: any other query, a cursor included, reaches pg unchanged', async () => { - const { client, calls } = recordingClient(); - sendMarkedQueriesExtended(client); - const cursor = { submit: () => undefined }; - await client.query('select 1; select 2', []); - await client.query('select $1', [1]); - await client.query(cursor); - expect(calls).toEqual([['select 1; select 2', []], ['select $1', [1]], [cursor]]); - }); - - it('EXTQ-003: without the hook the marker stays a parameter, so pg still binds and Postgres refuses the count', () => { - const query = extendedQuery('select 1', []); - expect(query.parameters).toHaveLength(1); - expect(unmarkedValues(query.parameters)).toEqual([]); - expect(unmarkedValues([])).toBeNull(); - expect(unmarkedValues(['a'])).toBeNull(); - expect(unmarkedValues(undefined)).toBeNull(); - }); -}); diff --git a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts index ff6dede07..4693d9e91 100644 --- a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts @@ -1,20 +1,26 @@ /** * The Postgres store of a plugin's own data - * (src/db/engine/postgres/plugin-data-store.ts) against a stand-in Kysely - * client that records what it is handed: the transaction every call opens and - * what it sets first, the schema check before plugin SQL runs, the row cap, - * the time budget, the quota, a script committed statement by statement, the - * tx and migrate shapes, and one call at a time per plugin. The same store on - * a real database is the Postgres lane's tests/postgres/plugin-data.test.ts. + * (src/db/engine/postgres/plugin-data-store.ts) against a stand-in `pg` + * client that records what it is handed: the client every call checks out + * and releases, the transaction it opens and what it sets first, the schema + * check before plugin SQL runs, the row cap, the time budget, the quota, a + * script committed statement by statement, the tx and migrate shapes, one + * call at a time per plugin, the protocol every plugin statement goes over, + * and the session cleanup before a client goes back to the pool. The same + * store on a real database is the Postgres lane's + * tests/postgres/plugin-data.test.ts. */ -import { unmarkedValues } from '../../../../../src/db/engine/postgres/extended-query'; import { dropPluginSchema, + END_SESSION, PluginDataSlots, + poolClients, PostgresPluginDataStore, + type PluginSqlClient, } from '../../../../../src/db/engine/postgres/plugin-data-store'; import type { Kysely } from 'kysely'; +import Cursor from 'pg-cursor'; import { describe, expect, it } from 'vitest'; interface FakeOptions { @@ -22,7 +28,7 @@ interface FakeOptions { schemas?: string[]; /** The sizes the quota reads answer, in order. */ sizes?: number[]; - /** Rows a streamed statement yields, by its text; streamed two at a time. */ + /** Rows a cursor yields, by its statement's text; read two at a time. */ rows?: Record; /** What a write reports as changed. */ changes?: number; @@ -30,67 +36,87 @@ interface FakeOptions { migrated?: boolean; /** An error for a statement, or undefined to run it. */ fail?: (sql: string) => Error | undefined; - /** Called before every streamed chunk, so a test can move its clock. */ + /** Called before every chunk a cursor reads, so a test can move its clock. */ onChunk?: () => void; - /** Awaited before a stream starts. */ + /** Awaited before a cursor's first read. */ beforeStream?: (sql: string) => Promise; } -/** A Kysely client that records each statement, and `begin`/`commit`/`rollback` around each transaction. */ -function fakeDb(options: FakeOptions = {}) { +/** What a plugin statement or a query with values was handed to the client as. */ +interface QueryConfig { + text: string; + values: unknown[]; + queryMode?: 'extended'; +} + +/** + * A pool of stand-in `pg` clients. `log` holds every statement's text in the + * order the clients ran them (`begin`, `commit`, `rollback` and the session + * cleanup among them), `configs` every query handed over as a config object, + * `cursors` every cursor submitted, `releases` whether each client was + * released to be closed. + */ +function fakePool(options: FakeOptions = {}) { const log: string[] = []; - /** Every statement handed to executeQuery, with its parameters as Kysely passes them to pg. */ - const executed: Array<{ sql: string; parameters: readonly unknown[] }> = []; + const configs: QueryConfig[] = []; + const cursors: Array<{ text: string; values: unknown[] | null; closed: boolean }> = []; + const releases: boolean[] = []; let size = 0; const run = (sql: string) => { log.push(sql); const err = options.fail?.(sql); if (err) throw err; }; - const trx = { - executeQuery: async (query: { sql: string; parameters: readonly unknown[] }) => { - executed.push(query); - run(query.sql); - if (query.sql.startsWith('select nspname')) - return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; - if (query.sql.startsWith('select coalesce(sum(')) return { rows: [{ bytes: options.sizes?.[size++] ?? 0 }] }; - if (query.sql.startsWith('select 1 from')) return { rows: options.migrated ? [{ '?column?': 1 }] : [] }; - return { rows: [], numAffectedRows: BigInt(options.changes ?? 0) }; + const answer = (sql: string) => { + if (sql.startsWith('select nspname')) return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; + if (sql.startsWith('select coalesce(sum(')) return { rows: [{ bytes: options.sizes?.[size++] ?? 0 }], rowCount: 1 }; + if (sql.startsWith('select 1 from')) return { rows: options.migrated ? [{ '?column?': 1 }] : [], rowCount: 0 }; + return { rows: [], rowCount: options.changes ?? 0 }; + }; + const cursorOf = (cursor: Cursor) => { + const { text, values } = cursor as unknown as { text: string; values: unknown[] | null }; + const seen = { text, values, closed: false }; + cursors.push(seen); + let next: number | null = null; + return { + async read(): Promise { + if (next === null) { + await options.beforeStream?.(text); + run(text); + next = 0; + } + const rows = options.rows?.[text] ?? []; + if (next >= rows.length) return []; + options.onChunk?.(); + next += 2; + return rows.slice(next - 2, next); + }, + async close(): Promise { + seen.closed = true; + }, + }; + }; + const client = { + query(arg: string | QueryConfig | Cursor): unknown { + if (arg instanceof Cursor) return cursorOf(arg); + const sql = typeof arg === 'string' ? arg : arg.text; + if (typeof arg !== 'string') configs.push(arg); + return (async () => { + run(sql); + return answer(sql); + })(); + }, + release(destroy?: boolean) { + releases.push(destroy === true); }, - getExecutor: () => ({ - async *stream(query: { sql: string }) { - await options.beforeStream?.(query.sql); - run(query.sql); - const rows = options.rows?.[query.sql] ?? []; - for (let i = 0; i < rows.length; i += 2) { - options.onChunk?.(); - yield { rows: rows.slice(i, i + 2) }; - } - }, - }), }; - const db = { - transaction: () => ({ - execute: async (work: (t: typeof trx) => Promise): Promise => { - log.push('begin'); - try { - const result = await work(trx); - log.push('commit'); - return result; - } catch (err) { - log.push('rollback'); - throw err; - } - }, - }), - executeQuery: trx.executeQuery, - }; - return { db: db as unknown as Kysely, log, executed }; + const connect = async () => client as unknown as PluginSqlClient; + return { connect, log, configs, cursors, releases }; } function store(options: FakeOptions = {}, limits = {}) { - const fake = fakeDb(options); - return { ...fake, store: new PostgresPluginDataStore('notes', fake.db, new PluginDataSlots(4), limits) }; + const pool = fakePool(options); + return { ...pool, store: new PostgresPluginDataStore('notes', pool.connect, new PluginDataSlots(4), limits) }; } /** A pg error with its SQLSTATE. */ @@ -116,6 +142,7 @@ describe('Postgres plugin data store', () => { ].join(';\n'), 'SELECT body FROM notes', 'commit', + END_SESSION, ]); }); @@ -123,7 +150,7 @@ describe('Postgres plugin data store', () => { const { store: s, log } = store({ schemas: ['public', 'auth', 'trek_plugin_notes'] }); await expect(s.query('SELECT x FROM auth.secret')).rejects.toThrow(/"auth" is another schema/); expect(log).not.toContain('SELECT x FROM auth.secret'); - expect(log.at(-1)).toBe('rollback'); + expect(log.slice(-2)).toEqual(['rollback', END_SESSION]); await expect(s.query('SELECT n.body FROM trek_plugin_notes.notes n')).resolves.toEqual([]); await expect(s.query('SELECT * FROM public.users')).rejects.toThrow(/another schema/); }); @@ -165,7 +192,7 @@ describe('Postgres plugin data store', () => { await expect(grows.store.exec('INSERT INTO t VALUES ($1)', ['x'])).rejects.toThrow( 'the plugin database is past its 0.0009765625 MB quota', ); - expect(grows.log.at(-1)).toBe('rollback'); + expect(grows.log.slice(-2)).toEqual(['rollback', END_SESSION]); const shrinks = store({ sizes: [4096, 4096], changes: 3 }, { quotaBytes: 1024 }); expect(await shrinks.store.exec('DELETE FROM t WHERE x = $1', ['x'])).toEqual({ changes: 3 }); @@ -230,13 +257,14 @@ describe('Postgres plugin data store', () => { }); it('PGSTORE-009: the SQL is checked before anything runs, and a closed store takes no new call', async () => { - const { store: s, log } = store(); + const { store: s, log, releases } = store(); await expect(s.query(42 as never)).rejects.toThrow('sql must be a string'); await expect(s.query('x'.repeat(100_001))).rejects.toThrow('sql too long'); await expect(s.query('-- only a comment')).rejects.toThrow('contains no statements'); await expect(s.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); await expect(s.exec('DO $$ BEGIN END $$')).rejects.toThrow(/dollar-quoted/); expect(log).toEqual([]); + expect(releases).toEqual([]); s.close(); expect(s.isOpen()).toBe(false); @@ -256,25 +284,87 @@ describe('Postgres plugin data store', () => { expect(log.filter((sql) => sql === 'begin' || sql === 'commit')).toEqual(['begin', 'commit', 'begin', 'commit']); }); - it('PGSTORE-012: every plugin statement it executes goes over the extended protocol, alone and with its own args', async () => { - const { store: s, executed } = store({ changes: 1 }); + it('PGSTORE-012: every plugin write goes to pg as one extended-protocol query with its own args, even with none', async () => { + // EXTQ-001 before the store checked out its own clients. + const { store: s, configs } = store({ changes: 1 }); await s.exec('INSERT INTO a VALUES (1); UPDATE a SET x = 2 -- note\rWHERE x = 1'); await s.exec('INSERT INTO a VALUES ($1)', ['x']); await s.tx([{ sql: 'DELETE FROM a WHERE x = $1', args: [3] }]); await s.migrate('002', 'CREATE TABLE b (y int); CREATE INDEX b_y ON b (y)'); - const plugin = executed.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.sql)); - expect(plugin.map((q) => [q.sql, unmarkedValues(q.parameters)])).toEqual([ - ['INSERT INTO a VALUES (1)', []], - ['UPDATE a SET x = 2 -- note\rWHERE x = 1', []], - ['INSERT INTO a VALUES ($1)', ['x']], - ['DELETE FROM a WHERE x = $1', [3]], - ['CREATE TABLE b (y int)', []], - ['CREATE INDEX b_y ON b (y)', []], + const plugin = configs.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.text)); + expect(plugin).toEqual([ + { text: 'INSERT INTO a VALUES (1)', values: [], queryMode: 'extended' }, + { text: 'UPDATE a SET x = 2 -- note\rWHERE x = 1', values: [], queryMode: 'extended' }, + { text: 'INSERT INTO a VALUES ($1)', values: ['x'], queryMode: 'extended' }, + { text: 'DELETE FROM a WHERE x = $1', values: [3], queryMode: 'extended' }, + { text: 'CREATE TABLE b (y int)', values: [], queryMode: 'extended' }, + { text: 'CREATE INDEX b_y ON b (y)', values: [], queryMode: 'extended' }, ]); + // TREK's own statements with values are never sent as a plugin's are. + expect(configs.filter((q) => q.queryMode === 'extended')).toHaveLength(plugin.length); + }); + + it('PGSTORE-013: every read goes through a cursor on the same client, closed however the read ends, args without NUL', async () => { + // EXTQ-002 before the store checked out its own clients: what is not a plugin write reaches pg as it was. + const rows = Array.from({ length: 7 }, (_, i) => ({ i })); + const { + store: s, + cursors, + configs, + } = store({ rows: { 'SELECT i FROM t WHERE x = $1': rows, 'SELECT 1 FROM t': [{}] } }, { maxRows: 5 }); + await expect(s.query('SELECT i FROM t WHERE x = $1', ['a\u0000b'])).rejects.toThrow('more than 5 rows'); + await s.exec('SELECT 1 FROM t', ['x']); + await s.exec('INSERT INTO t VALUES ($1)', ['c\u0000d']); + expect(cursors).toEqual([ + { text: 'SELECT i FROM t WHERE x = $1', values: ['ab'], closed: true }, + { text: 'SELECT 1 FROM t', values: ['x'], closed: true }, + ]); + expect(configs.find((q) => q.text.startsWith('INSERT'))?.values).toEqual(['cd']); + }); + + it('PGSTORE-014: each transaction checks out a client of its own and gives it back clean, or closes it', async () => { + const ok = store({ changes: 1 }); + await ok.store.exec('INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)'); + await expect(ok.store.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); + // One client per statement of the script, each after its session state was dropped; the refused read took none. + expect(ok.releases).toEqual([false, false]); + expect(ok.log.filter((sql) => sql === END_SESSION)).toHaveLength(2); + + const failed = store({ fail: (sql) => (sql.startsWith('INSERT') ? new Error('boom') : undefined) }); + await expect(failed.store.exec('INSERT INTO a VALUES ($1)', [1])).rejects.toThrow('boom'); + expect(failed.log.slice(-2)).toEqual(['rollback', END_SESSION]); + expect(failed.releases).toEqual([false]); + + // A client whose rollback fails is in a state nobody knows: it is closed, and the call reports its own error. + const stuck = store({ + fail: (sql) => + sql.startsWith('INSERT') ? new Error('boom') : sql === 'rollback' ? new Error('gone') : undefined, + }); + await expect(stuck.store.exec('INSERT INTO a VALUES ($1)', [1])).rejects.toThrow('boom'); + expect(stuck.log).not.toContain(END_SESSION); + expect(stuck.releases).toEqual([true]); + + // So is one whose session state cannot be dropped, after a call that succeeded. + const dirty = store({ fail: (sql) => (sql === END_SESSION ? new Error('gone') : undefined) }); + expect(await dirty.store.query('SELECT 1')).toEqual([]); + expect(dirty.releases).toEqual([true]); + }); + + it("PGSTORE-015: poolClients checks a client out of the pool behind the ORM's connection", async () => { + const client = { query: async () => ({ rows: [], rowCount: 0 }), release: () => undefined }; + const pool = { connect: async () => client }; + const connect = poolClients({ getNativeClient: async () => pool }); + expect(await connect()).toBe(client); }); it('PGSTORE-011: dropping a plugin schema names it quoted, and an id no schema comes from drops nothing', async () => { - const { db, log } = fakeDb(); + const log: string[] = []; + const db = { + executeQuery: async (query: { sql: string }) => { + log.push(query.sql); + return { rows: [] }; + }, + } as unknown as Kysely; await dropPluginSchema(db, 'trip-notes'); await dropPluginSchema(db, '../etc'); expect(log).toEqual(['drop schema if exists "trek_plugin_trip_notes" cascade']); diff --git a/server/tests/unit/db/nul-free.test.ts b/server/tests/unit/db/nul-free.test.ts index 311a99293..f6b74faae 100644 --- a/server/tests/unit/db/nul-free.test.ts +++ b/server/tests/unit/db/nul-free.test.ts @@ -4,7 +4,6 @@ * tests/integration/places-nul.test.ts runs a write and a search through the * whole app on SQLite and on Postgres. */ -import { extendedQuery, unmarkedValues } from '../../../src/db/engine/postgres/extended-query'; import { TrekPostgreSqlPlatform } from '../../../src/db/engine/postgres/postgres-driver'; import { nulFreeJson, nulFreeParameter, stripNul, withNulFreeParameters } from '../../../src/db/nul-free'; @@ -86,8 +85,6 @@ describe('NUL-free values', () => { expect(nulFreeParameter(numbers)).toBe(numbers); const bytes = Buffer.from([0, 1]); expect(nulFreeParameter(bytes)).toBe(bytes); - const marker = { trekExtendedProtocol: true }; - expect(nulFreeParameter(marker)).toBe(marker); expect(nulFreeParameter(null)).toBeNull(); }); @@ -118,13 +115,9 @@ describe('NUL-free values', () => { } expect(connection.sent[2].parameters).toEqual(['s']); - // The extended-protocol marker survives by identity. - await acquired.executeQuery(extendedQuery('select $1', ['m\u0000'])); - expect(unmarkedValues(connection.sent[3].parameters)).toEqual(['m']); - // The driver's own methods get the wrapped connection and still reach its private state. await driver.beginTransaction(acquired, {}); - expect(connection.sent[4].sql).toBe('begin'); + expect(connection.sent[3].sql).toBe('begin'); await driver.releaseConnection(acquired); expect(connection.released).toBe(true); });