diff --git a/package-lock.json b/package-lock.json index 0921101ff..04334ea42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25952,6 +25952,7 @@ "nodemailer": "^10.5.0", "otplib": "^12.0.1", "pdf-parse": "^2.4.5", + "pg-cursor": "^2.22.1", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", diff --git a/server/package.json b/server/package.json index 6e5a8244c..74a8f290d 100644 --- a/server/package.json +++ b/server/package.json @@ -106,6 +106,7 @@ "nodemailer": "^10.5.0", "otplib": "^12.0.1", "pdf-parse": "^2.4.5", + "pg-cursor": "^2.22.1", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", diff --git a/server/src/db/engine/postgres/extended-query.ts b/server/src/db/engine/postgres/extended-query.ts deleted file mode 100644 index 8af8702c5..000000000 --- a/server/src/db/engine/postgres/extended-query.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { CompiledQuery } from 'kysely'; - -/** - * Statements that must run alone: sent over Postgres's extended query - * protocol, whose Parse message takes exactly one statement, so a second one - * in the text is a syntax error instead of a statement that runs. - * - * `pg` picks the protocol by the values it is handed: with none it sends the - * text over the simple protocol, which runs every statement in it. Kysely - * hands it a query's parameters as they are, so {@link extendedQuery} adds a - * marker as the last parameter, and the connection hook - * {@link sendMarkedQueriesExtended} (installed on every client of TREK's pool - * by `postgres-driver.ts`) takes the marker off again and asks `pg` for the - * extended protocol. A query that reached a client without the hook still goes - * over the extended protocol, with one parameter too many, and Postgres - * refuses it: the marker fails closed. - * - * Plugin SQL (`plugin-data-store.ts`) runs through here. A read streamed - * through a cursor needs none of this: a cursor always uses the extended - * protocol. - */ - -/** The last parameter of a query {@link extendedQuery} built. */ -const EXTENDED_ONLY = Object.freeze({ trekExtendedProtocol: true }); - -/** `sql` with `args` as a query sent over the extended protocol, so it runs as one statement or not at all. */ -export function extendedQuery(sql: string, args: readonly unknown[]): CompiledQuery { - return CompiledQuery.raw(sql, [...args, EXTENDED_ONLY]); -} - -/** The values `pg` was handed, without the marker, when {@link extendedQuery} built them; otherwise null. */ -export function unmarkedValues(values: unknown): unknown[] | null { - if (!Array.isArray(values) || values.length === 0 || values[values.length - 1] !== EXTENDED_ONLY) return null; - return values.slice(0, -1); -} - -/** The part of a `pg` client this hook replaces. */ -interface QueryClient { - query: (...args: unknown[]) => unknown; -} - -/** - * Makes `client` send a query {@link extendedQuery} built over the extended - * protocol (`queryMode: 'extended'`, which `pg` honours even without values), - * and every other query as before. - */ -export function sendMarkedQueriesExtended(client: QueryClient): void { - const query = client.query.bind(client); - client.query = (...args: unknown[]) => { - const [text, values, ...rest] = args; - const unmarked = unmarkedValues(values); - if (typeof text !== 'string' || unmarked === null) return query(...args); - return query({ text, values: unmarked, queryMode: 'extended' }, ...rest); - }; -} diff --git a/server/src/db/engine/postgres/plugin-data-store.ts b/server/src/db/engine/postgres/plugin-data-store.ts index e392dbb39..3ea7eb25c 100644 --- a/server/src/db/engine/postgres/plugin-data-store.ts +++ b/server/src/db/engine/postgres/plugin-data-store.ts @@ -1,25 +1,33 @@ +import { nulFreeParameter } from '../../nul-free'; import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data'; -import { extendedQuery } from './extended-query'; import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema'; import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard'; -import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; +import { CompiledQuery, type Kysely } from 'kysely'; +import Cursor from 'pg-cursor'; /** * A plugin's own data on Postgres (`db:own`): a schema of its own in TREK's * database (`plugin-schema.ts`), reached through TREK's pool. * - * Every call runs in one transaction that first makes sure the schema and its - * migration table exist, then puts only that schema on the `search_path`, with - * `pg_temp` explicitly last so a temporary table is never found before the - * plugin's own, and sets `statement_timeout` to the call's time budget. The - * plugin's SQL is checked by `plugin-sql-guard.ts` before it runs, so it cannot - * name another schema, and each statement goes to Postgres on its own over the - * extended query protocol (`extended-query.ts`), which runs one statement and - * no more. Reads stream through a cursor and stop at the row cap. A call that - * may write measures the schema's size before and after, and rolls back one - * that leaves it past the quota and larger than it found it, so a plugin at - * its quota can still delete. + * Every call checks a `pg` client out of the pool for itself + * ({@link poolClients}) and runs one transaction on it directly, not through + * Kysely. The transaction first makes sure the schema and its migration table + * exist, then puts only that schema on the `search_path`, with `pg_temp` + * explicitly last so a temporary table is never found before the plugin's + * own, and sets `statement_timeout` to the call's time budget; every setting + * is `local` and ends with the transaction. The plugin's SQL is checked by + * `plugin-sql-guard.ts` before it runs, so it cannot name another schema, and + * each statement goes to Postgres on its own over the extended query + * protocol, which runs one statement and no more: a write as a query with + * `queryMode: 'extended'`, which `pg` honours even without values, and a read + * through a `pg-cursor`, which always uses that protocol. Reads stop at the + * row cap. A call that may write measures the schema's size before and + * after, and rolls back one that leaves it past the quota and larger than it + * found it, so a plugin at its quota can still delete. Before the client goes + * back to the pool, whatever session state a statement could have left on it + * is dropped ({@link END_SESSION}); a client that cannot be brought back to a + * clean state is closed instead of reused. * * The calls of one plugin run one at a time, as they do on its SQLite file, * and all plugins together hold at most half of TREK's pool @@ -57,6 +65,54 @@ const CHUNK_ROWS = 500; /** What better-sqlite3 says about a closed handle, which the host recognizes. */ const CLOSED = 'The database connection is not open'; +/** + * What a client runs before it goes back to the pool, outside any + * transaction: `DISCARD ALL` without the two parts that would undo the pool's + * own session state. `RESET ALL` would drop the `search_path` every pool + * connection sets with `pg_temp` last (`session-search-path.ts`), and + * `DEALLOCATE ALL` the statements `pg` prepared by name for other callers. + * The plugin's settings need neither: they are all `local`. The guard + * refuses every statement that could leave anything this drops, so this is + * the layer behind it. + */ +export const END_SESSION = 'close all; unlisten *; select pg_advisory_unlock_all(); discard temp; discard sequences'; + +/** A query result, as far as the store reads it. */ +interface PgResult { + rows: unknown[]; + rowCount: number | null; +} + +/** The part of a `pg` client the store uses. It is the store's alone from checkout to release. */ +export interface PluginSqlClient { + /** SQL of TREK's own, without values: over the simple protocol. */ + query(text: string): Promise; + /** A query with values, or a plugin statement with `queryMode: 'extended'`. */ + query(config: { text: string; values: unknown[]; queryMode?: 'extended' }): Promise; + /** Submits a cursor and hands it back. */ + query(cursor: Cursor): Cursor; + /** Back to the pool, or closed when `destroy` is true. */ + release(destroy?: boolean): void; +} + +/** Checks a client out of the pool; resolves once one is free. */ +export type ConnectPluginClient = () => Promise; + +/** The part of `pg`'s pool the store checks clients out of. */ +interface PgPool { + connect(): Promise; +} + +/** + * Clients of the `pg` pool behind MikroORM's Postgres connection, which its + * `getNativeClient` hands out. Its options apply to them as to every other + * client of the pool: the TLS settings, and the `int8` and `numeric` parsers + * (`postgres-driver.ts`). Nothing of the pool is changed for them. + */ +export function poolClients(connection: { getNativeClient(): Promise }): ConnectPluginClient { + return async () => ((await connection.getNativeClient()) as PgPool).connect(); +} + /** How many plugin transactions may hold a pool connection at once: half of TREK's pool, at least one. */ export function pluginDataSlots(poolMax: number): number { return Math.max(1, Math.floor(poolMax / 2)); @@ -83,8 +139,6 @@ export class PluginDataSlots { } } -type Trx = Transaction; - /** The rows one call has returned so far, against the cap they share. */ interface RowBudget { count: number; @@ -109,7 +163,7 @@ export class PostgresPluginDataStore implements PluginDataStore { constructor( readonly pluginId: string, - private readonly db: Kysely, + private readonly connect: ConnectPluginClient, private readonly slots: PluginDataSlots, limits: Partial = {}, ) { @@ -122,7 +176,7 @@ export class PostgresPluginDataStore implements PluginDataStore { return this.open; } - /** Stops new calls. One already running finishes: it holds a pool connection of its own, not this store. */ + /** Stops new calls. One already running finishes: it holds a pool client of its own, not this store. */ close(): void { this.open = false; } @@ -131,9 +185,9 @@ export class PostgresPluginDataStore implements PluginDataStore { const statement = this.single(sql, args); const { maxRows } = this.limits; return this.call('query', (budget) => - this.transaction(budget.ms, [statement], (trx) => { + this.transaction(budget.ms, [statement], (client) => { const rows = { count: 0, max: maxRows, message: `query returned more than ${maxRows} rows` }; - return this.stream(trx, statement, args, rows, budget.check); + return this.stream(client, statement, args, rows, budget.check); }), ); } @@ -142,7 +196,9 @@ export class PostgresPluginDataStore implements PluginDataStore { if (args.length > 0) { const statement = this.single(sql, args); return this.call('exec', (budget) => - this.transaction(budget.ms, [statement], async (trx) => ({ changes: await this.run(trx, statement, args) })), + this.transaction(budget.ms, [statement], async (client) => ({ + changes: await this.run(client, statement, args), + })), ); } const statements = this.parse(sql); @@ -151,7 +207,7 @@ export class PostgresPluginDataStore implements PluginDataStore { // the clock is read before every statement but the first. await inOrder(statements, (statement, index) => { if (index > 0) budget.check(); - return this.transaction(budget.remaining(), [statement], (trx) => this.run(trx, statement, [])); + return this.transaction(budget.remaining(), [statement], (client) => this.run(client, statement, [])); }); return { changes: 0 }; }); @@ -166,15 +222,15 @@ export class PostgresPluginDataStore implements PluginDataStore { const statements = ops.map((op) => this.single(op?.sql, op?.args ?? [])); const { maxRows } = this.limits; return this.call('tx', (budget) => - this.transaction(budget.ms, statements, async (trx) => { + this.transaction(budget.ms, statements, async (client) => { const rows = { count: 0, max: maxRows, message: `tx returned more than ${maxRows} rows in total` }; const results = await inOrder(statements, async (statement, index) => { const args = ops[index].args ?? []; // One budget for the whole batch: each statement after the first gets what is left of it. - if (index > 0) await this.setTimeout(trx, budget.remaining()); + if (index > 0) await this.setTimeout(client, budget.remaining()); const result = statement.returnsRows - ? { rows: await this.stream(trx, statement, args, rows, budget.check) } - : { changes: await this.run(trx, statement, args) }; + ? { rows: await this.stream(client, statement, args, rows, budget.check) } + : { changes: await this.run(client, statement, args) }; budget.check(); return result; }); @@ -187,18 +243,17 @@ export class PostgresPluginDataStore implements PluginDataStore { async migrate(id: string, sql: string): Promise<{ applied: boolean }> { const statements = this.parse(sql); return this.enqueue(() => - this.transaction(null, statements, async (trx) => { - const seen = await trx.executeQuery( - CompiledQuery.raw(`select 1 from ${this.quoted}._plugin_migrations where id = $1`, [id]), - ); + this.transaction(null, statements, async (client) => { + const seen = await client.query({ + text: `select 1 from ${this.quoted}._plugin_migrations where id = $1`, + values: [id], + }); if (seen.rows.length > 0) return { applied: false }; - await inOrder(statements, (statement) => this.run(trx, statement, [])); - await trx.executeQuery( - CompiledQuery.raw(`insert into ${this.quoted}._plugin_migrations (id, applied_at) values ($1, $2)`, [ - id, - Date.now(), - ]), - ); + await inOrder(statements, (statement) => this.run(client, statement, [])); + await client.query({ + text: `insert into ${this.quoted}._plugin_migrations (id, applied_at) values ($1, $2)`, + values: [id, Date.now()], + }); return { applied: true }; }), ); @@ -257,81 +312,111 @@ export class PostgresPluginDataStore implements PluginDataStore { } /** - * One transaction on this plugin's schema: created if missing, alone on the - * `search_path`, with `statement_timeout` at `timeoutMs` (none for null). - * The statements' qualified names are checked against the database's - * schemas first, and a call that may write is held to the quota. + * One transaction on this plugin's schema, on a client of its own: created + * if missing, alone on the `search_path`, with `statement_timeout` at + * `timeoutMs` (none for null). The statements' qualified names are checked + * against the database's schemas first, and a call that may write is held + * to the quota. The client goes back to the pool with its session state + * dropped, or is closed when the rollback or that cleanup fails. */ - private transaction( + private async transaction( timeoutMs: number | null, statements: readonly PluginStatement[], - work: (trx: Trx) => Promise, + work: (client: PluginSqlClient) => Promise, ): Promise { - return this.db.transaction().execute(async (trx) => { - const setup = [ - `create schema if not exists ${this.quoted}`, - `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, - `set local search_path to ${this.quoted}, pg_temp`, - 'set local standard_conforming_strings to on', - ]; - if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); - await trx.executeQuery(CompiledQuery.raw(setup.join(';\n'))); - if (statements.some((s) => s.qualifies)) { - const schemas = await this.schemaNames(trx); - for (const statement of statements) checkQualifiers(statement, this.schema, schemas); + const client = await this.connect(); + let reusable = true; + try { + await client.query('begin'); + try { + const result = await this.inTransaction(client, timeoutMs, statements, work); + await client.query('commit'); + return result; + } catch (err) { + await client.query('rollback').catch(() => { + reusable = false; + }); + throw err; } - const writes = statements.some((s) => s.mayWrite); - const before = writes ? await this.sizeBytes(trx) : 0; - const result = await work(trx); - if (writes) { - const after = await this.sizeBytes(trx); - if (after > this.limits.quotaBytes && after > before) { - throw new Error(`the plugin database is past its ${this.limits.quotaBytes / 1024 / 1024} MB quota`); - } + } finally { + if (reusable) { + await client.query(END_SESSION).catch(() => { + reusable = false; + }); } - return result; + client.release(!reusable); + } + } + + /** What {@link transaction} runs between `begin` and `commit`. */ + private async inTransaction( + client: PluginSqlClient, + timeoutMs: number | null, + statements: readonly PluginStatement[], + work: (client: PluginSqlClient) => Promise, + ): Promise { + const setup = [ + `create schema if not exists ${this.quoted}`, + `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, + `set local search_path to ${this.quoted}, pg_temp`, + 'set local standard_conforming_strings to on', + ]; + if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); + await client.query(setup.join(';\n')); + if (statements.some((s) => s.qualifies)) { + const schemas = await this.schemaNames(client); + for (const statement of statements) checkQualifiers(statement, this.schema, schemas); + } + const writes = statements.some((s) => s.mayWrite); + const before = writes ? await this.sizeBytes(client) : 0; + const result = await work(client); + if (writes) { + const after = await this.sizeBytes(client); + if (after > this.limits.quotaBytes && after > before) { + throw new Error(`the plugin database is past its ${this.limits.quotaBytes / 1024 / 1024} MB quota`); + } + } + return result; + } + + private async setTimeout(client: PluginSqlClient, ms: number): Promise { + await client.query({ + text: "select set_config('statement_timeout', $1, true)", + values: [String(Math.max(1, Math.floor(ms)))], }); } - private async setTimeout(trx: Trx, ms: number): Promise { - await trx.executeQuery( - CompiledQuery.raw("select set_config('statement_timeout', $1, true)", [String(Math.max(1, Math.floor(ms)))]), - ); - } - /** Every schema name of the database, which a qualified name in plugin SQL may not use. */ - private async schemaNames(trx: Trx): Promise> { - const result = await trx.executeQuery<{ nspname: string }>(CompiledQuery.raw('select nspname from pg_namespace')); - return new Set(result.rows.map((row) => row.nspname)); + private async schemaNames(client: PluginSqlClient): Promise> { + const result = await client.query('select nspname from pg_namespace'); + return new Set((result.rows as Array<{ nspname: string }>).map((row) => row.nspname)); } /** What this plugin's tables, indexes and materialized views take on disk. */ - private async sizeBytes(trx: Trx): Promise { - const result = await trx.executeQuery<{ bytes: number | string }>( - CompiledQuery.raw( + private async sizeBytes(client: PluginSqlClient): Promise { + const result = await client.query({ + text: 'select coalesce(sum(pg_total_relation_size(c.oid)), 0) as bytes from pg_class c ' + - "join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'm')", - [this.schema], - ), - ); - return Number(result.rows[0]?.bytes ?? 0); + "join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'm')", + values: [this.schema], + }); + const [row] = result.rows as Array<{ bytes: number | string }>; + return Number(row?.bytes ?? 0); } /** * Runs a statement for its effect: the rows it changed, or 0 for a read, - * whose rows are streamed past and dropped. Both ways use the extended - * protocol: a cursor always does, and a write goes through `extendedQuery`. + * whose rows are read past and dropped. Both go over the extended protocol: + * a write as an extended query, a read through a cursor. */ - private async run(trx: Trx, statement: PluginStatement, args: readonly unknown[]): Promise { + private async run(client: PluginSqlClient, statement: PluginStatement, args: readonly unknown[]): Promise { if (!statement.mayWrite) { - const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); - for await (const _chunk of chunks) { - // Read to the end so the statement runs whole; its rows are not wanted. - } + // Read to the end so the statement runs whole; its rows are not wanted. + await readCursor(client.query(new Cursor(statement.text, boundValues(args))), () => undefined); return 0; } - const result = await trx.executeQuery(extendedQuery(statement.text, args)); - return Number(result.numAffectedRows ?? 0); + const result = await client.query({ text: statement.text, values: boundValues(args), queryMode: 'extended' }); + return result.rowCount ?? 0; } /** @@ -339,26 +424,48 @@ export class PostgresPluginDataStore implements PluginDataStore { * row cap the read stops and throws, and the clock is read between chunks. */ private async stream( - trx: Trx, + client: PluginSqlClient, statement: PluginStatement, args: readonly unknown[], budget: RowBudget, checkTime: () => void, ): Promise { const rows: unknown[] = []; - const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); - for await (const chunk of chunks) { + await readCursor(client.query(new Cursor(statement.text, boundValues(args))), (chunk) => { if (rows.length > 0) checkTime(); - for (const row of chunk.rows) { + for (const row of chunk) { rows.push(row); budget.count++; if (budget.count > budget.max) throw new Error(budget.message); } - } + }); return rows; } } +/** A plugin's args as they are bound: without NUL, which Postgres text cannot hold (`nul-free.ts`). */ +function boundValues(args: readonly unknown[]): unknown[] { + return args.map(nulFreeParameter); +} + +/** + * Reads `cursor` to its end a chunk at a time, handing each chunk to `take` + * (which may throw to stop the read), and closes it however the read ends. + */ +async function readCursor(cursor: Cursor, take: (rows: unknown[]) => void): Promise { + const next = async (): Promise => { + const rows = await cursor.read(CHUNK_ROWS); + if (rows.length === 0) return; + take(rows); + return next(); + }; + try { + await next(); + } finally { + await cursor.close(); + } +} + /** `work` for each item, one after the other, each started once the one before it has finished. */ async function inOrder(items: readonly T[], work: (item: T, index: number) => Promise): Promise { const results: R[] = []; diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index 2c8029cd8..0922a48e5 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,14 +1,8 @@ import { nulFreeJson, stripNul, withNulFreeParameters } from '../../nul-free'; -import { sendMarkedQueriesExtended } from './extended-query'; import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; import { type Configuration, type TransformContext } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; -/** The `pg` pool event a new client is handed to before anyone uses it. */ -interface ClientEvents { - on(event: 'connect', listener: (client: { query: (...args: unknown[]) => unknown }) => void): void; -} - /** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */ const INT8_OID = 20; /** `numeric`: what `sum()` of a bigint and `avg()` answer. */ @@ -60,18 +54,11 @@ class NumberTypesConnection extends PostgreSqlConnection { /** * The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL - * position (`null-order.ts`), sending every parameter without NUL - * (`nul-free.ts`), on a pool whose every client sends the queries - * `extendedQuery` builds over the extended protocol (`extended-query.ts`). + * position (`null-order.ts`) and sending every parameter without NUL + * (`nul-free.ts`). */ override createKyselyDialect(overrides: Parameters[0]) { - const given = (overrides ?? {}) as { onPoolCreated?: (pool: unknown) => unknown }; - const onPoolCreated = (pool: unknown) => { - (pool as ClientEvents).on('connect', sendMarkedQueriesExtended); - return given.onPoolCreated?.(pool); - }; - const withHook = { ...given, onPoolCreated } as Parameters[0]; - return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(withHook))) as ReturnType< + return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(overrides))) as ReturnType< PostgreSqlConnection['createKyselyDialect'] >; } diff --git a/server/src/db/nul-free.ts b/server/src/db/nul-free.ts index 32f1a770c..9305cfbb6 100644 --- a/server/src/db/nul-free.ts +++ b/server/src/db/nul-free.ts @@ -20,9 +20,11 @@ import type { AbortableOperationOptions, CompiledQuery, DatabaseConnection, Dial * - the Kysely dialects of both engines (`orm-driver.ts`, * `engine/postgres/postgres-driver.ts`) send every query's parameters * through {@link withNulFreeParameters}, which covers the Kysely - * statements of the repositories and a plugin's own SQL on Postgres; + * statements of the repositories; * - a plugin's own SQLite file (`nest/plugins/host/plugin-data.service.ts`) - * binds its arguments through {@link nulFreeParameter}; + * and its schema on Postgres (`engine/postgres/plugin-data-store.ts`, + * which sends on a `pg` client of its own) bind its arguments through + * {@link nulFreeParameter}; * - the move from SQLite to Postgres (`engine/postgres/sqlite-move/values.ts`) * copies text the same way. * diff --git a/server/src/nest/plugins/host/plugin-data.backend.ts b/server/src/nest/plugins/host/plugin-data.backend.ts index e632fc310..d93b3305b 100644 --- a/server/src/nest/plugins/host/plugin-data.backend.ts +++ b/server/src/nest/plugins/host/plugin-data.backend.ts @@ -6,6 +6,7 @@ import { dropPluginSchema, PluginDataSlots, pluginDataSlots, + poolClients, PostgresPluginDataStore, } from '../../../db/engine/postgres/plugin-data-store'; import { DATABASE_ENGINE, DATABASE_SETTINGS } from '../../database/database-engine.module'; @@ -52,8 +53,9 @@ export class SqlitePluginDataBackend implements PluginDataBackend { /** * Each plugin's own schema in TREK's Postgres database - * (`db/engine/postgres/plugin-data-store.ts`), reached through the ORM's - * pool. The plugin's data directory is still created, for the files a plugin + * (`db/engine/postgres/plugin-data-store.ts`). Each call checks a client + * out of the ORM's pool for itself; erasing a plugin runs on the ORM's Kysely + * client. The plugin's data directory is still created, for the files a plugin * keeps beside its database, and it is what marks a plugin uninstalled with * its data kept (the erasure queue reads it). */ @@ -70,7 +72,7 @@ export class PostgresPluginDataBackend implements PluginDataBackend { open(pluginId: string): PluginDataStore { fs.mkdirSync(pluginDataDir(pluginId), { recursive: true }); - return new PostgresPluginDataStore(pluginId, this.client(), this.sharedSlots()); + return new PostgresPluginDataStore(pluginId, poolClients(this.orm.em.getConnection()), this.sharedSlots()); } async erase(pluginId: string): Promise { diff --git a/server/src/typings/pg-cursor.d.ts b/server/src/typings/pg-cursor.d.ts new file mode 100644 index 000000000..993031deb --- /dev/null +++ b/server/src/typings/pg-cursor.d.ts @@ -0,0 +1,14 @@ +// `pg-cursor` ships no type definitions. These are the parts the Postgres +// plugin data store uses (`db/engine/postgres/plugin-data-store.ts`): a cursor +// is handed to a `pg` client's `query`, which submits it over the extended +// query protocol, and is then read a number of rows at a time. + +declare module 'pg-cursor' { + export default class Cursor { + constructor(text: string, values?: unknown[]); + /** The next `rows` rows, or none once the statement has returned them all. */ + read(rows: number): Promise; + /** Closes the portal; safe after the last read and after an error. */ + close(): Promise; + } +} diff --git a/server/tests/postgres/plugin-data.test.ts b/server/tests/postgres/plugin-data.test.ts index 8d70d35fb..df9b443f5 100644 --- a/server/tests/postgres/plugin-data.test.ts +++ b/server/tests/postgres/plugin-data.test.ts @@ -1,27 +1,32 @@ /** - * PGPLUG-001..011 — a plugin's own data (`db:own`) on Postgres, against the + * PGPLUG-001..012 — a plugin's own data (`db:own`) on Postgres, against the * real app on the lane's database: each plugin gets a schema of its own and no * SQLite file, its SQL keeps the SQLite store's semantics (migrate once per * id, a script's statements committed one by one, an atomic tx, the row cap, * the time budget, the quota), it cannot reach TREK's tables or another - * plugin's schema however it spells the name, and erasing a plugin drops its - * schema and nothing else. + * plugin's schema however it spells the name, every statement runs alone, + * the pool client a call used goes back without anything the call set, and + * erasing a plugin drops its schema and nothing else. */ import { buildApp } from '../../src/bootstrap'; -import { extendedQuery } from '../../src/db/engine/postgres/extended-query'; -import { PluginDataSlots, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; +import { + type ConnectPluginClient, + PluginDataSlots, + type PluginSqlClient, + poolClients, + PostgresPluginDataStore, +} from '../../src/db/engine/postgres/plugin-data-store'; import { withRequestContext } from '../../src/nest/database/request-context'; import { PLUGIN_DATA, type PluginDataBackend } from '../../src/nest/plugins/host/plugin-data.backend'; import { closePluginDataDb } from '../../src/nest/plugins/host/plugin-host-state'; import { PluginRpcHostFactory } from '../../src/nest/plugins/host/plugin-rpc-host.factory'; import { MikroORM } from '@mikro-orm/core'; -import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { INestApplication } from '@nestjs/common'; -import { CompiledQuery } from 'kysely'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import Cursor from 'pg-cursor'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; let app: INestApplication; @@ -57,10 +62,48 @@ async function pluginSchemas(): Promise { return rows.map((row) => row.nspname); } +/** Clients of the app's pool, as the backend hands them to a store. */ +function appClients(): ConnectPluginClient { + return poolClients(orm.em.getConnection()); +} + /** A store on the app's pool with limits small enough to reach in a test. */ function cappedStore(id: string, limits: { maxRows?: number; timeBudgetMs?: number; quotaBytes?: number }) { - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); - return new PostgresPluginDataStore(id, client, new PluginDataSlots(1), limits); + return new PostgresPluginDataStore(id, appClients(), new PluginDataSlots(1), limits); +} + +/** `client` for every call of a store, kept checked out by the test: a release only records whether it would close it. */ +function keep(client: PluginSqlClient, released: boolean[] = []): ConnectPluginClient { + return async () => + ({ + query: (arg: never) => client.query(arg), + release: (destroy?: boolean) => released.push(destroy === true), + }) as unknown as PluginSqlClient; +} + +/** + * Pool clients that hand Postgres `swap[text]` in place of a plugin statement + * whose text is `text`, as a store would if the guard let a second statement + * through: as an extended query or as a cursor, whichever the store sends. + */ +function smuggling(swap: Record): ConnectPluginClient { + const connect = appClients(); + return async () => { + const client = await connect(); + const query = (arg: unknown): unknown => { + if (arg instanceof Cursor) { + const cursor = arg as unknown as { text: string }; + cursor.text = swap[cursor.text] ?? cursor.text; + return client.query(arg); + } + const config = arg as { text?: unknown; values: unknown[] }; + if (typeof config.text === 'string' && swap[config.text]) { + return client.query({ ...config, text: swap[config.text] }); + } + return client.query(arg as never); + }; + return { query, release: (destroy?: boolean) => client.release(destroy) } as unknown as PluginSqlClient; + }; } describe('plugin data on Postgres', () => { @@ -275,24 +318,36 @@ describe('plugin data on Postgres', () => { expect(await store.query('SELECT body FROM notes')).toEqual([{ body: 'x' }]); }); - it('PGPLUG-010: a statement extendedQuery builds runs alone: a second one in its text is refused by Postgres, not run', async () => { + it('PGPLUG-010: every plugin statement runs alone: a second one smuggled into its text is refused by Postgres, not run', async () => { const store = data.open('pg-proto'); await store.migrate('001', 'CREATE TABLE t (x int)'); - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); const two = 'insert into trek_plugin_pg_proto.t values (1); insert into trek_plugin_pg_proto.t values (2)'; - // What a guard that let a second statement through would hand the pool: Postgres parses one statement only. - await expect(client.executeQuery(extendedQuery(two, []))).rejects.toThrow( + // What the store would hand Postgres if the guard let a second statement through, as a write and as a read. + const fooled = new PostgresPluginDataStore( + 'pg-proto', + smuggling({ 'INSERT INTO t VALUES (7)': two, 'SELECT x FROM t': `select x from t; ${two}` }), + new PluginDataSlots(1), + ); + await expect(fooled.exec('INSERT INTO t VALUES (7)')).rejects.toThrow( /cannot insert multiple commands into a prepared statement/, ); + await expect(fooled.exec('INSERT INTO t VALUES (7)', [])).rejects.toThrow(/cannot insert multiple commands/); + await expect(fooled.tx([{ sql: 'INSERT INTO t VALUES (7)' }])).rejects.toThrow(/cannot insert multiple commands/); + await expect(fooled.query('SELECT x FROM t')).rejects.toThrow(/cannot insert multiple commands/); expect(await store.query('SELECT count(*) AS n FROM t')).toEqual([{ n: 0 }]); - // The same text without the marker goes over the simple protocol and runs both, which is what the marker prevents. - await client.executeQuery(CompiledQuery.raw(two, [])); + + // The same text over the simple protocol runs both, which is what sending each statement extended prevents. + const client = await appClients()(); + try { + await client.query(two); + } finally { + client.release(); + } expect(await store.query('SELECT count(*) AS n FROM t')).toEqual([{ n: 2 }]); // A single statement runs and reports its rows, with or without args. - const one = await client.executeQuery(extendedQuery('delete from trek_plugin_pg_proto.t where x = $1', [1])); - expect(one.numAffectedRows).toBe(1n); + expect(await store.exec('DELETE FROM t WHERE x = $1', [1])).toEqual({ changes: 1 }); expect(await store.exec('DELETE FROM t')).toEqual({ changes: 0 }); expect(await store.exec('INSERT INTO t VALUES (5)', [])).toEqual({ changes: 0 }); expect(await store.tx([{ sql: 'INSERT INTO t VALUES (6)' }, { sql: 'DELETE FROM t' }])).toEqual({ @@ -316,23 +371,49 @@ describe('plugin data on Postgres', () => { const store = data.open('pg-shadow'); await store.migrate('001', 'CREATE TABLE notes (body text)'); await store.exec('INSERT INTO notes VALUES ($1)', ['real']); - const client = (orm.em.getConnection() as AbstractSqlConnection).getClient(); - // One pool connection holding temporary tables named like TREK's and the plugin's, as one left behind would. - await client.connection().execute(async (conn) => { - try { - await conn.executeQuery(CompiledQuery.raw("create temp table users as select 'forged'::text as email")); - await conn.executeQuery(CompiledQuery.raw("create temp table notes as select 'forged'::text as body")); - const resolved = await conn.executeQuery<{ ns: string }>( - CompiledQuery.raw( - "select relnamespace::regnamespace::text as ns from pg_class where oid = 'users'::regclass", - ), - ); - expect(resolved.rows).toEqual([{ ns: 'public' }]); - const onConnection = new PostgresPluginDataStore('pg-shadow', conn, new PluginDataSlots(1)); - expect(await onConnection.query('SELECT body FROM notes')).toEqual([{ body: 'real' }]); - } finally { - await conn.executeQuery(CompiledQuery.raw('drop table if exists pg_temp.users, pg_temp.notes')); - } - }); + // A pool client holding temporary tables named like TREK's and the plugin's, as one left behind would. + const client = await appClients()(); + try { + await client.query("create temp table users as select 'forged'::text as email"); + await client.query("create temp table notes as select 'forged'::text as body"); + const released: boolean[] = []; + const onClient = new PostgresPluginDataStore('pg-shadow', keep(client, released), new PluginDataSlots(1)); + expect(await onClient.query('SELECT body FROM notes')).toEqual([{ body: 'real' }]); + // The call dropped them with the rest of the session's state before it gave the client back. + const temps = await client.query( + "select count(*)::int as n from pg_class where relpersistence = 't' and relnamespace = pg_my_temp_schema()", + ); + expect(temps.rows).toEqual([{ n: 0 }]); + expect(released).toEqual([false]); + } finally { + await client.query('drop table if exists pg_temp.users, pg_temp.notes'); + client.release(); + } + }); + + it('PGPLUG-012: a call leaves nothing it set on the pool client it used, a cancelled one neither', async () => { + const store = data.open('pg-session'); + await store.migrate('001', 'CREATE TABLE notes (body text)'); + const settings = + "select current_setting('search_path') as path, current_setting('statement_timeout') as timeout, " + + "current_setting('standard_conforming_strings') as strings, now() = statement_timestamp() as outside"; + const client = await appClients()(); + try { + const before = await client.query(settings); + const released: boolean[] = []; + const onClient = new PostgresPluginDataStore('pg-session', keep(client, released), new PluginDataSlots(1), { + timeBudgetMs: 300, + }); + await onClient.tx([{ sql: "INSERT INTO notes VALUES ('a')" }, { sql: 'SELECT body FROM notes' }]); + await expect(onClient.query('SELECT count(*) FROM generate_series(1, 2000000000)')).rejects.toThrow( + 'query exceeded its 300 ms time budget', + ); + await onClient.migrate('002', 'CREATE INDEX notes_body ON notes (body)'); + // Same search_path and settings as before, and no transaction left open (now() moves with each statement). + expect((await client.query(settings)).rows).toEqual(before.rows); + expect(released).toEqual([false, false, false]); + } finally { + client.release(); + } }); }); diff --git a/server/tests/postgres/sqlite-move.test.ts b/server/tests/postgres/sqlite-move.test.ts index 2b6c05cac..40d51ef56 100644 --- a/server/tests/postgres/sqlite-move.test.ts +++ b/server/tests/postgres/sqlite-move.test.ts @@ -14,7 +14,7 @@ import { buildApp } from '../../src/bootstrap'; import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; -import { PluginDataSlots, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; +import { PluginDataSlots, poolClients, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; import { AppSettings } from '../../src/db/entities/AppSettings.entity'; @@ -45,7 +45,6 @@ import { makeTrip, readTripDays } from '../helpers/factories/trips'; import { makeUser } from '../helpers/factories/users'; import { databaseUrl, withAdmin } from './databases'; import { MikroORM, type EntityClass, type Options } from '@mikro-orm/core'; -import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { INestApplication } from '@nestjs/common'; import Database from 'better-sqlite3'; @@ -270,8 +269,7 @@ describe('moving a SQLite install to Postgres', () => { const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id)); expect(user.id).toBeGreaterThan(highest); - const client = (pgOrm.em.getConnection() as AbstractSqlConnection).getClient(); - const store = new PostgresPluginDataStore(PLUGIN_ID, client, new PluginDataSlots(1)); + const store = new PostgresPluginDataStore(PLUGIN_ID, poolClients(pgOrm.em.getConnection()), new PluginDataSlots(1)); expect(await store.query('select id, body, score from notes order by id')).toEqual([ { id: 1, body: 'first', score: 1.5 }, { id: 2, body: 'second', score: null }, diff --git a/server/tests/unit/db/engine/postgres/extended-query.test.ts b/server/tests/unit/db/engine/postgres/extended-query.test.ts deleted file mode 100644 index cad2e542e..000000000 --- a/server/tests/unit/db/engine/postgres/extended-query.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * The extended-protocol marker of plugin SQL - * (src/db/engine/postgres/extended-query.ts), against a stand-in `pg` client - * that records what it is handed. That Postgres then runs one statement and - * no more is PGPLUG-010 in the Postgres lane. - */ -import { - extendedQuery, - sendMarkedQueriesExtended, - unmarkedValues, -} from '../../../../../src/db/engine/postgres/extended-query'; - -import { describe, expect, it } from 'vitest'; - -/** A `pg` client that records the arguments of every `query` call. */ -function recordingClient() { - const calls: unknown[][] = []; - const client = { - query: (...args: unknown[]): unknown => { - calls.push(args); - return Promise.resolve({ rows: [] }); - }, - }; - return { client, calls }; -} - -describe('extended query', () => { - it('EXTQ-001: a marked query reaches pg as one extended-protocol query with its own values, even with none', async () => { - const { client, calls } = recordingClient(); - sendMarkedQueriesExtended(client); - const none = extendedQuery('insert into t values (1)', []); - const some = extendedQuery('insert into t values ($1, $2)', ['a', 2]); - await client.query(none.sql, none.parameters); - await client.query(some.sql, some.parameters); - expect(calls).toEqual([ - [{ text: 'insert into t values (1)', values: [], queryMode: 'extended' }], - [{ text: 'insert into t values ($1, $2)', values: ['a', 2], queryMode: 'extended' }], - ]); - }); - - it('EXTQ-002: any other query, a cursor included, reaches pg unchanged', async () => { - const { client, calls } = recordingClient(); - sendMarkedQueriesExtended(client); - const cursor = { submit: () => undefined }; - await client.query('select 1; select 2', []); - await client.query('select $1', [1]); - await client.query(cursor); - expect(calls).toEqual([['select 1; select 2', []], ['select $1', [1]], [cursor]]); - }); - - it('EXTQ-003: without the hook the marker stays a parameter, so pg still binds and Postgres refuses the count', () => { - const query = extendedQuery('select 1', []); - expect(query.parameters).toHaveLength(1); - expect(unmarkedValues(query.parameters)).toEqual([]); - expect(unmarkedValues([])).toBeNull(); - expect(unmarkedValues(['a'])).toBeNull(); - expect(unmarkedValues(undefined)).toBeNull(); - }); -}); diff --git a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts index ff6dede07..4693d9e91 100644 --- a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts @@ -1,20 +1,26 @@ /** * The Postgres store of a plugin's own data - * (src/db/engine/postgres/plugin-data-store.ts) against a stand-in Kysely - * client that records what it is handed: the transaction every call opens and - * what it sets first, the schema check before plugin SQL runs, the row cap, - * the time budget, the quota, a script committed statement by statement, the - * tx and migrate shapes, and one call at a time per plugin. The same store on - * a real database is the Postgres lane's tests/postgres/plugin-data.test.ts. + * (src/db/engine/postgres/plugin-data-store.ts) against a stand-in `pg` + * client that records what it is handed: the client every call checks out + * and releases, the transaction it opens and what it sets first, the schema + * check before plugin SQL runs, the row cap, the time budget, the quota, a + * script committed statement by statement, the tx and migrate shapes, one + * call at a time per plugin, the protocol every plugin statement goes over, + * and the session cleanup before a client goes back to the pool. The same + * store on a real database is the Postgres lane's + * tests/postgres/plugin-data.test.ts. */ -import { unmarkedValues } from '../../../../../src/db/engine/postgres/extended-query'; import { dropPluginSchema, + END_SESSION, PluginDataSlots, + poolClients, PostgresPluginDataStore, + type PluginSqlClient, } from '../../../../../src/db/engine/postgres/plugin-data-store'; import type { Kysely } from 'kysely'; +import Cursor from 'pg-cursor'; import { describe, expect, it } from 'vitest'; interface FakeOptions { @@ -22,7 +28,7 @@ interface FakeOptions { schemas?: string[]; /** The sizes the quota reads answer, in order. */ sizes?: number[]; - /** Rows a streamed statement yields, by its text; streamed two at a time. */ + /** Rows a cursor yields, by its statement's text; read two at a time. */ rows?: Record; /** What a write reports as changed. */ changes?: number; @@ -30,67 +36,87 @@ interface FakeOptions { migrated?: boolean; /** An error for a statement, or undefined to run it. */ fail?: (sql: string) => Error | undefined; - /** Called before every streamed chunk, so a test can move its clock. */ + /** Called before every chunk a cursor reads, so a test can move its clock. */ onChunk?: () => void; - /** Awaited before a stream starts. */ + /** Awaited before a cursor's first read. */ beforeStream?: (sql: string) => Promise; } -/** A Kysely client that records each statement, and `begin`/`commit`/`rollback` around each transaction. */ -function fakeDb(options: FakeOptions = {}) { +/** What a plugin statement or a query with values was handed to the client as. */ +interface QueryConfig { + text: string; + values: unknown[]; + queryMode?: 'extended'; +} + +/** + * A pool of stand-in `pg` clients. `log` holds every statement's text in the + * order the clients ran them (`begin`, `commit`, `rollback` and the session + * cleanup among them), `configs` every query handed over as a config object, + * `cursors` every cursor submitted, `releases` whether each client was + * released to be closed. + */ +function fakePool(options: FakeOptions = {}) { const log: string[] = []; - /** Every statement handed to executeQuery, with its parameters as Kysely passes them to pg. */ - const executed: Array<{ sql: string; parameters: readonly unknown[] }> = []; + const configs: QueryConfig[] = []; + const cursors: Array<{ text: string; values: unknown[] | null; closed: boolean }> = []; + const releases: boolean[] = []; let size = 0; const run = (sql: string) => { log.push(sql); const err = options.fail?.(sql); if (err) throw err; }; - const trx = { - executeQuery: async (query: { sql: string; parameters: readonly unknown[] }) => { - executed.push(query); - run(query.sql); - if (query.sql.startsWith('select nspname')) - return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; - if (query.sql.startsWith('select coalesce(sum(')) return { rows: [{ bytes: options.sizes?.[size++] ?? 0 }] }; - if (query.sql.startsWith('select 1 from')) return { rows: options.migrated ? [{ '?column?': 1 }] : [] }; - return { rows: [], numAffectedRows: BigInt(options.changes ?? 0) }; + const answer = (sql: string) => { + if (sql.startsWith('select nspname')) return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; + if (sql.startsWith('select coalesce(sum(')) return { rows: [{ bytes: options.sizes?.[size++] ?? 0 }], rowCount: 1 }; + if (sql.startsWith('select 1 from')) return { rows: options.migrated ? [{ '?column?': 1 }] : [], rowCount: 0 }; + return { rows: [], rowCount: options.changes ?? 0 }; + }; + const cursorOf = (cursor: Cursor) => { + const { text, values } = cursor as unknown as { text: string; values: unknown[] | null }; + const seen = { text, values, closed: false }; + cursors.push(seen); + let next: number | null = null; + return { + async read(): Promise { + if (next === null) { + await options.beforeStream?.(text); + run(text); + next = 0; + } + const rows = options.rows?.[text] ?? []; + if (next >= rows.length) return []; + options.onChunk?.(); + next += 2; + return rows.slice(next - 2, next); + }, + async close(): Promise { + seen.closed = true; + }, + }; + }; + const client = { + query(arg: string | QueryConfig | Cursor): unknown { + if (arg instanceof Cursor) return cursorOf(arg); + const sql = typeof arg === 'string' ? arg : arg.text; + if (typeof arg !== 'string') configs.push(arg); + return (async () => { + run(sql); + return answer(sql); + })(); + }, + release(destroy?: boolean) { + releases.push(destroy === true); }, - getExecutor: () => ({ - async *stream(query: { sql: string }) { - await options.beforeStream?.(query.sql); - run(query.sql); - const rows = options.rows?.[query.sql] ?? []; - for (let i = 0; i < rows.length; i += 2) { - options.onChunk?.(); - yield { rows: rows.slice(i, i + 2) }; - } - }, - }), }; - const db = { - transaction: () => ({ - execute: async (work: (t: typeof trx) => Promise): Promise => { - log.push('begin'); - try { - const result = await work(trx); - log.push('commit'); - return result; - } catch (err) { - log.push('rollback'); - throw err; - } - }, - }), - executeQuery: trx.executeQuery, - }; - return { db: db as unknown as Kysely, log, executed }; + const connect = async () => client as unknown as PluginSqlClient; + return { connect, log, configs, cursors, releases }; } function store(options: FakeOptions = {}, limits = {}) { - const fake = fakeDb(options); - return { ...fake, store: new PostgresPluginDataStore('notes', fake.db, new PluginDataSlots(4), limits) }; + const pool = fakePool(options); + return { ...pool, store: new PostgresPluginDataStore('notes', pool.connect, new PluginDataSlots(4), limits) }; } /** A pg error with its SQLSTATE. */ @@ -116,6 +142,7 @@ describe('Postgres plugin data store', () => { ].join(';\n'), 'SELECT body FROM notes', 'commit', + END_SESSION, ]); }); @@ -123,7 +150,7 @@ describe('Postgres plugin data store', () => { const { store: s, log } = store({ schemas: ['public', 'auth', 'trek_plugin_notes'] }); await expect(s.query('SELECT x FROM auth.secret')).rejects.toThrow(/"auth" is another schema/); expect(log).not.toContain('SELECT x FROM auth.secret'); - expect(log.at(-1)).toBe('rollback'); + expect(log.slice(-2)).toEqual(['rollback', END_SESSION]); await expect(s.query('SELECT n.body FROM trek_plugin_notes.notes n')).resolves.toEqual([]); await expect(s.query('SELECT * FROM public.users')).rejects.toThrow(/another schema/); }); @@ -165,7 +192,7 @@ describe('Postgres plugin data store', () => { await expect(grows.store.exec('INSERT INTO t VALUES ($1)', ['x'])).rejects.toThrow( 'the plugin database is past its 0.0009765625 MB quota', ); - expect(grows.log.at(-1)).toBe('rollback'); + expect(grows.log.slice(-2)).toEqual(['rollback', END_SESSION]); const shrinks = store({ sizes: [4096, 4096], changes: 3 }, { quotaBytes: 1024 }); expect(await shrinks.store.exec('DELETE FROM t WHERE x = $1', ['x'])).toEqual({ changes: 3 }); @@ -230,13 +257,14 @@ describe('Postgres plugin data store', () => { }); it('PGSTORE-009: the SQL is checked before anything runs, and a closed store takes no new call', async () => { - const { store: s, log } = store(); + const { store: s, log, releases } = store(); await expect(s.query(42 as never)).rejects.toThrow('sql must be a string'); await expect(s.query('x'.repeat(100_001))).rejects.toThrow('sql too long'); await expect(s.query('-- only a comment')).rejects.toThrow('contains no statements'); await expect(s.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); await expect(s.exec('DO $$ BEGIN END $$')).rejects.toThrow(/dollar-quoted/); expect(log).toEqual([]); + expect(releases).toEqual([]); s.close(); expect(s.isOpen()).toBe(false); @@ -256,25 +284,87 @@ describe('Postgres plugin data store', () => { expect(log.filter((sql) => sql === 'begin' || sql === 'commit')).toEqual(['begin', 'commit', 'begin', 'commit']); }); - it('PGSTORE-012: every plugin statement it executes goes over the extended protocol, alone and with its own args', async () => { - const { store: s, executed } = store({ changes: 1 }); + it('PGSTORE-012: every plugin write goes to pg as one extended-protocol query with its own args, even with none', async () => { + // EXTQ-001 before the store checked out its own clients. + const { store: s, configs } = store({ changes: 1 }); await s.exec('INSERT INTO a VALUES (1); UPDATE a SET x = 2 -- note\rWHERE x = 1'); await s.exec('INSERT INTO a VALUES ($1)', ['x']); await s.tx([{ sql: 'DELETE FROM a WHERE x = $1', args: [3] }]); await s.migrate('002', 'CREATE TABLE b (y int); CREATE INDEX b_y ON b (y)'); - const plugin = executed.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.sql)); - expect(plugin.map((q) => [q.sql, unmarkedValues(q.parameters)])).toEqual([ - ['INSERT INTO a VALUES (1)', []], - ['UPDATE a SET x = 2 -- note\rWHERE x = 1', []], - ['INSERT INTO a VALUES ($1)', ['x']], - ['DELETE FROM a WHERE x = $1', [3]], - ['CREATE TABLE b (y int)', []], - ['CREATE INDEX b_y ON b (y)', []], + const plugin = configs.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.text)); + expect(plugin).toEqual([ + { text: 'INSERT INTO a VALUES (1)', values: [], queryMode: 'extended' }, + { text: 'UPDATE a SET x = 2 -- note\rWHERE x = 1', values: [], queryMode: 'extended' }, + { text: 'INSERT INTO a VALUES ($1)', values: ['x'], queryMode: 'extended' }, + { text: 'DELETE FROM a WHERE x = $1', values: [3], queryMode: 'extended' }, + { text: 'CREATE TABLE b (y int)', values: [], queryMode: 'extended' }, + { text: 'CREATE INDEX b_y ON b (y)', values: [], queryMode: 'extended' }, ]); + // TREK's own statements with values are never sent as a plugin's are. + expect(configs.filter((q) => q.queryMode === 'extended')).toHaveLength(plugin.length); + }); + + it('PGSTORE-013: every read goes through a cursor on the same client, closed however the read ends, args without NUL', async () => { + // EXTQ-002 before the store checked out its own clients: what is not a plugin write reaches pg as it was. + const rows = Array.from({ length: 7 }, (_, i) => ({ i })); + const { + store: s, + cursors, + configs, + } = store({ rows: { 'SELECT i FROM t WHERE x = $1': rows, 'SELECT 1 FROM t': [{}] } }, { maxRows: 5 }); + await expect(s.query('SELECT i FROM t WHERE x = $1', ['a\u0000b'])).rejects.toThrow('more than 5 rows'); + await s.exec('SELECT 1 FROM t', ['x']); + await s.exec('INSERT INTO t VALUES ($1)', ['c\u0000d']); + expect(cursors).toEqual([ + { text: 'SELECT i FROM t WHERE x = $1', values: ['ab'], closed: true }, + { text: 'SELECT 1 FROM t', values: ['x'], closed: true }, + ]); + expect(configs.find((q) => q.text.startsWith('INSERT'))?.values).toEqual(['cd']); + }); + + it('PGSTORE-014: each transaction checks out a client of its own and gives it back clean, or closes it', async () => { + const ok = store({ changes: 1 }); + await ok.store.exec('INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)'); + await expect(ok.store.query('SELECT 1; SELECT 2')).rejects.toThrow(/exactly one statement/); + // One client per statement of the script, each after its session state was dropped; the refused read took none. + expect(ok.releases).toEqual([false, false]); + expect(ok.log.filter((sql) => sql === END_SESSION)).toHaveLength(2); + + const failed = store({ fail: (sql) => (sql.startsWith('INSERT') ? new Error('boom') : undefined) }); + await expect(failed.store.exec('INSERT INTO a VALUES ($1)', [1])).rejects.toThrow('boom'); + expect(failed.log.slice(-2)).toEqual(['rollback', END_SESSION]); + expect(failed.releases).toEqual([false]); + + // A client whose rollback fails is in a state nobody knows: it is closed, and the call reports its own error. + const stuck = store({ + fail: (sql) => + sql.startsWith('INSERT') ? new Error('boom') : sql === 'rollback' ? new Error('gone') : undefined, + }); + await expect(stuck.store.exec('INSERT INTO a VALUES ($1)', [1])).rejects.toThrow('boom'); + expect(stuck.log).not.toContain(END_SESSION); + expect(stuck.releases).toEqual([true]); + + // So is one whose session state cannot be dropped, after a call that succeeded. + const dirty = store({ fail: (sql) => (sql === END_SESSION ? new Error('gone') : undefined) }); + expect(await dirty.store.query('SELECT 1')).toEqual([]); + expect(dirty.releases).toEqual([true]); + }); + + it("PGSTORE-015: poolClients checks a client out of the pool behind the ORM's connection", async () => { + const client = { query: async () => ({ rows: [], rowCount: 0 }), release: () => undefined }; + const pool = { connect: async () => client }; + const connect = poolClients({ getNativeClient: async () => pool }); + expect(await connect()).toBe(client); }); it('PGSTORE-011: dropping a plugin schema names it quoted, and an id no schema comes from drops nothing', async () => { - const { db, log } = fakeDb(); + const log: string[] = []; + const db = { + executeQuery: async (query: { sql: string }) => { + log.push(query.sql); + return { rows: [] }; + }, + } as unknown as Kysely; await dropPluginSchema(db, 'trip-notes'); await dropPluginSchema(db, '../etc'); expect(log).toEqual(['drop schema if exists "trek_plugin_trip_notes" cascade']); diff --git a/server/tests/unit/db/nul-free.test.ts b/server/tests/unit/db/nul-free.test.ts index 311a99293..f6b74faae 100644 --- a/server/tests/unit/db/nul-free.test.ts +++ b/server/tests/unit/db/nul-free.test.ts @@ -4,7 +4,6 @@ * tests/integration/places-nul.test.ts runs a write and a search through the * whole app on SQLite and on Postgres. */ -import { extendedQuery, unmarkedValues } from '../../../src/db/engine/postgres/extended-query'; import { TrekPostgreSqlPlatform } from '../../../src/db/engine/postgres/postgres-driver'; import { nulFreeJson, nulFreeParameter, stripNul, withNulFreeParameters } from '../../../src/db/nul-free'; @@ -86,8 +85,6 @@ describe('NUL-free values', () => { expect(nulFreeParameter(numbers)).toBe(numbers); const bytes = Buffer.from([0, 1]); expect(nulFreeParameter(bytes)).toBe(bytes); - const marker = { trekExtendedProtocol: true }; - expect(nulFreeParameter(marker)).toBe(marker); expect(nulFreeParameter(null)).toBeNull(); }); @@ -118,13 +115,9 @@ describe('NUL-free values', () => { } expect(connection.sent[2].parameters).toEqual(['s']); - // The extended-protocol marker survives by identity. - await acquired.executeQuery(extendedQuery('select $1', ['m\u0000'])); - expect(unmarkedValues(connection.sent[3].parameters)).toEqual(['m']); - // The driver's own methods get the wrapped connection and still reach its private state. await driver.beginTransaction(acquired, {}); - expect(connection.sent[4].sql).toBe('begin'); + expect(connection.sent[3].sql).toBe('begin'); await driver.releaseConnection(acquired); expect(connection.released).toBe(true); });