From 861f6ba789d660a12297428dde88c8e48d374eb8 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 20:45:39 +0100 Subject: [PATCH] fix(db): run plugin SQL one statement at a time and search pg_temp last The plugin SQL guard was the only thing between a plugin and TREK's tables on Postgres, and two gaps in it were enough to leave the plugin schema. Two layers now hold even when the guard is fooled. A plugin write with no bound args went over the simple query protocol, which runs every statement in its text. Plugin statements now go through extendedQuery, which pg sends over the extended protocol whatever the args, so a second statement is a syntax error instead of a statement that runs. A marker parameter carries the request to the pool's clients, and a client without the hook binds one value too many, so it fails closed. pg_temp is searched before the search_path unless the path names it, and a temporary table lives as long as its pooled connection, so one left behind would stand in for users, sessions and every other table for the next caller. Every connection of TREK's pool now puts pg_temp last on the search_path the role configured, and a plugin transaction sets its schema with pg_temp last. --- .../src/db/engine/postgres/extended-query.ts | 55 +++++++++++++++++ .../db/engine/postgres/plugin-data-store.ts | 22 ++++--- .../src/db/engine/postgres/postgres-driver.ts | 20 ++++++- .../src/db/engine/postgres/postgres-engine.ts | 5 ++ .../db/engine/postgres/session-search-path.ts | 43 ++++++++++++++ server/tests/unit/db/engine/engine.test.ts | 2 + .../db/engine/postgres/extended-query.test.ts | 59 +++++++++++++++++++ .../engine/postgres/plugin-data-store.test.ts | 25 +++++++- .../postgres/session-search-path.test.ts | 39 ++++++++++++ 9 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 server/src/db/engine/postgres/extended-query.ts create mode 100644 server/src/db/engine/postgres/session-search-path.ts create mode 100644 server/tests/unit/db/engine/postgres/extended-query.test.ts create mode 100644 server/tests/unit/db/engine/postgres/session-search-path.test.ts diff --git a/server/src/db/engine/postgres/extended-query.ts b/server/src/db/engine/postgres/extended-query.ts new file mode 100644 index 000000000..8af8702c5 --- /dev/null +++ b/server/src/db/engine/postgres/extended-query.ts @@ -0,0 +1,55 @@ +import { CompiledQuery } from 'kysely'; + +/** + * Statements that must run alone: sent over Postgres's extended query + * protocol, whose Parse message takes exactly one statement, so a second one + * in the text is a syntax error instead of a statement that runs. + * + * `pg` picks the protocol by the values it is handed: with none it sends the + * text over the simple protocol, which runs every statement in it. Kysely + * hands it a query's parameters as they are, so {@link extendedQuery} adds a + * marker as the last parameter, and the connection hook + * {@link sendMarkedQueriesExtended} (installed on every client of TREK's pool + * by `postgres-driver.ts`) takes the marker off again and asks `pg` for the + * extended protocol. A query that reached a client without the hook still goes + * over the extended protocol, with one parameter too many, and Postgres + * refuses it: the marker fails closed. + * + * Plugin SQL (`plugin-data-store.ts`) runs through here. A read streamed + * through a cursor needs none of this: a cursor always uses the extended + * protocol. + */ + +/** The last parameter of a query {@link extendedQuery} built. */ +const EXTENDED_ONLY = Object.freeze({ trekExtendedProtocol: true }); + +/** `sql` with `args` as a query sent over the extended protocol, so it runs as one statement or not at all. */ +export function extendedQuery(sql: string, args: readonly unknown[]): CompiledQuery { + return CompiledQuery.raw(sql, [...args, EXTENDED_ONLY]); +} + +/** The values `pg` was handed, without the marker, when {@link extendedQuery} built them; otherwise null. */ +export function unmarkedValues(values: unknown): unknown[] | null { + if (!Array.isArray(values) || values.length === 0 || values[values.length - 1] !== EXTENDED_ONLY) return null; + return values.slice(0, -1); +} + +/** The part of a `pg` client this hook replaces. */ +interface QueryClient { + query: (...args: unknown[]) => unknown; +} + +/** + * Makes `client` send a query {@link extendedQuery} built over the extended + * protocol (`queryMode: 'extended'`, which `pg` honours even without values), + * and every other query as before. + */ +export function sendMarkedQueriesExtended(client: QueryClient): void { + const query = client.query.bind(client); + client.query = (...args: unknown[]) => { + const [text, values, ...rest] = args; + const unmarked = unmarkedValues(values); + if (typeof text !== 'string' || unmarked === null) return query(...args); + return query({ text, values: unmarked, queryMode: 'extended' }, ...rest); + }; +} diff --git a/server/src/db/engine/postgres/plugin-data-store.ts b/server/src/db/engine/postgres/plugin-data-store.ts index 307ca9e64..e392dbb39 100644 --- a/server/src/db/engine/postgres/plugin-data-store.ts +++ b/server/src/db/engine/postgres/plugin-data-store.ts @@ -1,4 +1,5 @@ import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data'; +import { extendedQuery } from './extended-query'; import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema'; import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard'; @@ -9,10 +10,13 @@ import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; * database (`plugin-schema.ts`), reached through TREK's pool. * * Every call runs in one transaction that first makes sure the schema and its - * migration table exist, then puts only that schema on the `search_path` and - * sets `statement_timeout` to the call's time budget. The plugin's SQL is - * checked by `plugin-sql-guard.ts` before it runs, so it cannot name another - * schema. Reads stream through a cursor and stop at the row cap. A call that + * migration table exist, then puts only that schema on the `search_path`, with + * `pg_temp` explicitly last so a temporary table is never found before the + * plugin's own, and sets `statement_timeout` to the call's time budget. The + * plugin's SQL is checked by `plugin-sql-guard.ts` before it runs, so it cannot + * name another schema, and each statement goes to Postgres on its own over the + * extended query protocol (`extended-query.ts`), which runs one statement and + * no more. Reads stream through a cursor and stop at the row cap. A call that * may write measures the schema's size before and after, and rolls back one * that leaves it past the quota and larger than it found it, so a plugin at * its quota can still delete. @@ -267,7 +271,7 @@ export class PostgresPluginDataStore implements PluginDataStore { const setup = [ `create schema if not exists ${this.quoted}`, `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, - `set local search_path to ${this.quoted}`, + `set local search_path to ${this.quoted}, pg_temp`, 'set local standard_conforming_strings to on', ]; if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); @@ -313,7 +317,11 @@ export class PostgresPluginDataStore implements PluginDataStore { return Number(result.rows[0]?.bytes ?? 0); } - /** Runs a statement for its effect: the rows it changed, or 0 for a read, whose rows are streamed past and dropped. */ + /** + * Runs a statement for its effect: the rows it changed, or 0 for a read, + * whose rows are streamed past and dropped. Both ways use the extended + * protocol: a cursor always does, and a write goes through `extendedQuery`. + */ private async run(trx: Trx, statement: PluginStatement, args: readonly unknown[]): Promise { if (!statement.mayWrite) { const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); @@ -322,7 +330,7 @@ export class PostgresPluginDataStore implements PluginDataStore { } return 0; } - const result = await trx.executeQuery(CompiledQuery.raw(statement.text, [...args])); + const result = await trx.executeQuery(extendedQuery(statement.text, args)); return Number(result.numAffectedRows ?? 0); } diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index 272c6ae96..a646daa94 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,7 +1,13 @@ +import { sendMarkedQueriesExtended } from './extended-query'; import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; import { type Configuration } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; +/** The `pg` pool event a new client is handed to before anyone uses it. */ +interface ClientEvents { + on(event: 'connect', listener: (client: { query: (...args: unknown[]) => unknown }) => void): void; +} + /** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */ const INT8_OID = 20; /** `numeric`: what `sum()` of a bigint and `avg()` answer. */ @@ -51,9 +57,19 @@ class NumberTypesConnection extends PostgreSqlConnection { return options; } - /** The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL position (`null-order.ts`). */ + /** + * The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL + * position (`null-order.ts`), on a pool whose every client sends the queries + * `extendedQuery` builds over the extended protocol (`extended-query.ts`). + */ override createKyselyDialect(overrides: Parameters[0]) { - return withSqliteNullOrder(super.createKyselyDialect(overrides)) as ReturnType< + const given = (overrides ?? {}) as { onPoolCreated?: (pool: unknown) => unknown }; + const onPoolCreated = (pool: unknown) => { + (pool as ClientEvents).on('connect', sendMarkedQueriesExtended); + return given.onPoolCreated?.(pool); + }; + const withHook = { ...given, onPoolCreated } as Parameters[0]; + return withSqliteNullOrder(super.createKyselyDialect(withHook)) as ReturnType< PostgreSqlConnection['createKyselyDialect'] >; } diff --git a/server/src/db/engine/postgres/postgres-engine.ts b/server/src/db/engine/postgres/postgres-engine.ts index e7b48b137..60cdaa15c 100644 --- a/server/src/db/engine/postgres/postgres-engine.ts +++ b/server/src/db/engine/postgres/postgres-engine.ts @@ -5,9 +5,12 @@ import type { DatabaseEngine } from '../engine'; import { redactUrl } from '../redact-url'; import { TrekPostgreSqlDriver } from './postgres-driver'; import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap'; +import { putTempSchemaLast } from './session-search-path'; import { sslOptionFor } from './ssl'; import type { EntityManager } from '@mikro-orm/core'; +import type { DatabaseConnection } from 'kysely'; + /** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */ type PostgresSettings = Extract; @@ -78,6 +81,7 @@ export class PostgresEngine implements DatabaseEngine { return `Postgres ${redactUrl(this.settings.url)}`; } + /** Every new pool connection first puts `pg_temp` last on its `search_path` (`session-search-path.ts`). */ ormOptions() { const ssl = sslOptionFor(this.settings.url); return { @@ -85,6 +89,7 @@ export class PostgresEngine implements DatabaseEngine { clientUrl: this.settings.url, pool: { min: 0, max: this.settings.poolMax }, driverOptions: ssl === undefined ? {} : { ssl }, + onCreateConnection: (connection: unknown) => putTempSchemaLast(connection as DatabaseConnection), }; } } diff --git a/server/src/db/engine/postgres/session-search-path.ts b/server/src/db/engine/postgres/session-search-path.ts new file mode 100644 index 000000000..ef479af97 --- /dev/null +++ b/server/src/db/engine/postgres/session-search-path.ts @@ -0,0 +1,43 @@ +import { CompiledQuery, type DatabaseConnection } from 'kysely'; + +/** + * Puts `pg_temp` last on the `search_path` of every connection of TREK's pool. + * + * Postgres searches the session's temporary schema before everything on the + * `search_path` unless the path names `pg_temp` itself, and a temporary table + * lives as long as its connection. On a pool, whatever one caller left there + * would stand in for TREK's tables (`users`, `sessions`, ...) for every later + * caller of that connection. Named last, the temporary schema is searched + * after every schema TREK uses, so its tables are never found first. The rest + * of the path stays as the role or the database configured it: TREK may run + * in a schema other than `public`, and finds it through that path. + * + * Plugin transactions set their own path the same way + * (`plugin-data-store.ts`); `plugin-sql-guard.ts` explains the layers. + */ + +/** `path` (as `SHOW search_path` prints it) with every `pg_temp` entry taken out and one put last. */ +export function searchPathWithTempLast(path: string): string { + const entries: string[] = []; + let current = ''; + let quoted = false; + for (const ch of path) { + if (ch === '"') quoted = !quoted; + if (ch === ',' && !quoted) { + entries.push(current.trim()); + current = ''; + } else { + current += ch; + } + } + entries.push(current.trim()); + const kept = entries.filter((entry) => entry !== '' && entry.toLowerCase() !== 'pg_temp' && entry !== '"pg_temp"'); + return [...kept, 'pg_temp'].join(', '); +} + +/** The connection hook (`onCreateConnection`) that sets {@link searchPathWithTempLast} for the session. */ +export async function putTempSchemaLast(connection: DatabaseConnection): Promise { + const shown = await connection.executeQuery<{ search_path: string }>(CompiledQuery.raw('show search_path')); + const path = searchPathWithTempLast(shown.rows[0]?.search_path ?? ''); + await connection.executeQuery(CompiledQuery.raw("select set_config('search_path', $1, false)", [path])); +} diff --git a/server/tests/unit/db/engine/engine.test.ts b/server/tests/unit/db/engine/engine.test.ts index 0b002b733..d60b68026 100644 --- a/server/tests/unit/db/engine/engine.test.ts +++ b/server/tests/unit/db/engine/engine.test.ts @@ -30,6 +30,8 @@ describe('resolveEngine', () => { clientUrl: PG_URL, pool: { min: 0, max: 7 }, driverOptions: {}, + // Puts pg_temp last on every new connection's search_path (session-search-path.ts). + onCreateConnection: expect.any(Function), }); }); diff --git a/server/tests/unit/db/engine/postgres/extended-query.test.ts b/server/tests/unit/db/engine/postgres/extended-query.test.ts new file mode 100644 index 000000000..cad2e542e --- /dev/null +++ b/server/tests/unit/db/engine/postgres/extended-query.test.ts @@ -0,0 +1,59 @@ +/** + * The extended-protocol marker of plugin SQL + * (src/db/engine/postgres/extended-query.ts), against a stand-in `pg` client + * that records what it is handed. That Postgres then runs one statement and + * no more is PGPLUG-010 in the Postgres lane. + */ +import { + extendedQuery, + sendMarkedQueriesExtended, + unmarkedValues, +} from '../../../../../src/db/engine/postgres/extended-query'; + +import { describe, expect, it } from 'vitest'; + +/** A `pg` client that records the arguments of every `query` call. */ +function recordingClient() { + const calls: unknown[][] = []; + const client = { + query: (...args: unknown[]): unknown => { + calls.push(args); + return Promise.resolve({ rows: [] }); + }, + }; + return { client, calls }; +} + +describe('extended query', () => { + it('EXTQ-001: a marked query reaches pg as one extended-protocol query with its own values, even with none', async () => { + const { client, calls } = recordingClient(); + sendMarkedQueriesExtended(client); + const none = extendedQuery('insert into t values (1)', []); + const some = extendedQuery('insert into t values ($1, $2)', ['a', 2]); + await client.query(none.sql, none.parameters); + await client.query(some.sql, some.parameters); + expect(calls).toEqual([ + [{ text: 'insert into t values (1)', values: [], queryMode: 'extended' }], + [{ text: 'insert into t values ($1, $2)', values: ['a', 2], queryMode: 'extended' }], + ]); + }); + + it('EXTQ-002: any other query, a cursor included, reaches pg unchanged', async () => { + const { client, calls } = recordingClient(); + sendMarkedQueriesExtended(client); + const cursor = { submit: () => undefined }; + await client.query('select 1; select 2', []); + await client.query('select $1', [1]); + await client.query(cursor); + expect(calls).toEqual([['select 1; select 2', []], ['select $1', [1]], [cursor]]); + }); + + it('EXTQ-003: without the hook the marker stays a parameter, so pg still binds and Postgres refuses the count', () => { + const query = extendedQuery('select 1', []); + expect(query.parameters).toHaveLength(1); + expect(unmarkedValues(query.parameters)).toEqual([]); + expect(unmarkedValues([])).toBeNull(); + expect(unmarkedValues(['a'])).toBeNull(); + expect(unmarkedValues(undefined)).toBeNull(); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts index e98a7fe11..ff6dede07 100644 --- a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts @@ -7,6 +7,7 @@ * tx and migrate shapes, and one call at a time per plugin. The same store on * a real database is the Postgres lane's tests/postgres/plugin-data.test.ts. */ +import { unmarkedValues } from '../../../../../src/db/engine/postgres/extended-query'; import { dropPluginSchema, PluginDataSlots, @@ -38,6 +39,8 @@ interface FakeOptions { /** A Kysely client that records each statement, and `begin`/`commit`/`rollback` around each transaction. */ function fakeDb(options: FakeOptions = {}) { const log: string[] = []; + /** Every statement handed to executeQuery, with its parameters as Kysely passes them to pg. */ + const executed: Array<{ sql: string; parameters: readonly unknown[] }> = []; let size = 0; const run = (sql: string) => { log.push(sql); @@ -46,6 +49,7 @@ function fakeDb(options: FakeOptions = {}) { }; const trx = { executeQuery: async (query: { sql: string; parameters: readonly unknown[] }) => { + executed.push(query); run(query.sql); if (query.sql.startsWith('select nspname')) return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; @@ -81,7 +85,7 @@ function fakeDb(options: FakeOptions = {}) { }), executeQuery: trx.executeQuery, }; - return { db: db as unknown as Kysely, log }; + return { db: db as unknown as Kysely, log, executed }; } function store(options: FakeOptions = {}, limits = {}) { @@ -106,7 +110,7 @@ describe('Postgres plugin data store', () => { [ 'create schema if not exists "trek_plugin_notes"', 'create table if not exists "trek_plugin_notes"._plugin_migrations (id text primary key, applied_at bigint)', - 'set local search_path to "trek_plugin_notes"', + 'set local search_path to "trek_plugin_notes", pg_temp', 'set local standard_conforming_strings to on', 'set local statement_timeout to 2000', ].join(';\n'), @@ -252,6 +256,23 @@ describe('Postgres plugin data store', () => { expect(log.filter((sql) => sql === 'begin' || sql === 'commit')).toEqual(['begin', 'commit', 'begin', 'commit']); }); + it('PGSTORE-012: every plugin statement it executes goes over the extended protocol, alone and with its own args', async () => { + const { store: s, executed } = store({ changes: 1 }); + await s.exec('INSERT INTO a VALUES (1); UPDATE a SET x = 2 -- note\rWHERE x = 1'); + await s.exec('INSERT INTO a VALUES ($1)', ['x']); + await s.tx([{ sql: 'DELETE FROM a WHERE x = $1', args: [3] }]); + await s.migrate('002', 'CREATE TABLE b (y int); CREATE INDEX b_y ON b (y)'); + const plugin = executed.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.sql)); + expect(plugin.map((q) => [q.sql, unmarkedValues(q.parameters)])).toEqual([ + ['INSERT INTO a VALUES (1)', []], + ['UPDATE a SET x = 2 -- note\rWHERE x = 1', []], + ['INSERT INTO a VALUES ($1)', ['x']], + ['DELETE FROM a WHERE x = $1', [3]], + ['CREATE TABLE b (y int)', []], + ['CREATE INDEX b_y ON b (y)', []], + ]); + }); + it('PGSTORE-011: dropping a plugin schema names it quoted, and an id no schema comes from drops nothing', async () => { const { db, log } = fakeDb(); await dropPluginSchema(db, 'trip-notes'); diff --git a/server/tests/unit/db/engine/postgres/session-search-path.test.ts b/server/tests/unit/db/engine/postgres/session-search-path.test.ts new file mode 100644 index 000000000..cd55538a9 --- /dev/null +++ b/server/tests/unit/db/engine/postgres/session-search-path.test.ts @@ -0,0 +1,39 @@ +/** + * `pg_temp` last on the search_path of TREK's pool connections + * (src/db/engine/postgres/session-search-path.ts), without a database. That a + * temporary table then never stands in for TREK's own is PGPLUG-011 in the + * Postgres lane. + */ +import { putTempSchemaLast, searchPathWithTempLast } from '../../../../../src/db/engine/postgres/session-search-path'; + +import type { CompiledQuery, DatabaseConnection, QueryResult } from 'kysely'; +import { describe, expect, it } from 'vitest'; + +describe('session search_path', () => { + it.each([ + ['"$user", public', '"$user", public, pg_temp'], + ['trek', 'trek, pg_temp'], + ['pg_temp, public', 'public, pg_temp'], + ['PG_TEMP,public', 'public, pg_temp'], + ['"pg_temp", "$user", public', '"$user", public, pg_temp'], + ['"a, b", "c""d", public', '"a, b", "c""d", public, pg_temp'], + ['', 'pg_temp'], + ])('SPATH-001: %j keeps its schemas in order and gets pg_temp last: %j', (path, expected) => { + expect(searchPathWithTempLast(path)).toBe(expected); + }); + + it('SPATH-002: the connection hook reads the path and sets it for the session, the value bound', async () => { + const sent: Array<{ sql: string; parameters: readonly unknown[] }> = []; + const connection = { + executeQuery: async (query: CompiledQuery): Promise> => { + sent.push({ sql: query.sql, parameters: query.parameters }); + return { rows: query.sql === 'show search_path' ? [{ search_path: 'trek, public' }] : [] }; + }, + } as unknown as DatabaseConnection; + await putTempSchemaLast(connection); + expect(sent).toEqual([ + { sql: 'show search_path', parameters: [] }, + { sql: "select set_config('search_path', $1, false)", parameters: ['trek, public, pg_temp'] }, + ]); + }); +});