diff --git a/server/src/db/engine/postgres/extended-query.ts b/server/src/db/engine/postgres/extended-query.ts new file mode 100644 index 000000000..8af8702c5 --- /dev/null +++ b/server/src/db/engine/postgres/extended-query.ts @@ -0,0 +1,55 @@ +import { CompiledQuery } from 'kysely'; + +/** + * Statements that must run alone: sent over Postgres's extended query + * protocol, whose Parse message takes exactly one statement, so a second one + * in the text is a syntax error instead of a statement that runs. + * + * `pg` picks the protocol by the values it is handed: with none it sends the + * text over the simple protocol, which runs every statement in it. Kysely + * hands it a query's parameters as they are, so {@link extendedQuery} adds a + * marker as the last parameter, and the connection hook + * {@link sendMarkedQueriesExtended} (installed on every client of TREK's pool + * by `postgres-driver.ts`) takes the marker off again and asks `pg` for the + * extended protocol. A query that reached a client without the hook still goes + * over the extended protocol, with one parameter too many, and Postgres + * refuses it: the marker fails closed. + * + * Plugin SQL (`plugin-data-store.ts`) runs through here. A read streamed + * through a cursor needs none of this: a cursor always uses the extended + * protocol. + */ + +/** The last parameter of a query {@link extendedQuery} built. */ +const EXTENDED_ONLY = Object.freeze({ trekExtendedProtocol: true }); + +/** `sql` with `args` as a query sent over the extended protocol, so it runs as one statement or not at all. */ +export function extendedQuery(sql: string, args: readonly unknown[]): CompiledQuery { + return CompiledQuery.raw(sql, [...args, EXTENDED_ONLY]); +} + +/** The values `pg` was handed, without the marker, when {@link extendedQuery} built them; otherwise null. */ +export function unmarkedValues(values: unknown): unknown[] | null { + if (!Array.isArray(values) || values.length === 0 || values[values.length - 1] !== EXTENDED_ONLY) return null; + return values.slice(0, -1); +} + +/** The part of a `pg` client this hook replaces. */ +interface QueryClient { + query: (...args: unknown[]) => unknown; +} + +/** + * Makes `client` send a query {@link extendedQuery} built over the extended + * protocol (`queryMode: 'extended'`, which `pg` honours even without values), + * and every other query as before. + */ +export function sendMarkedQueriesExtended(client: QueryClient): void { + const query = client.query.bind(client); + client.query = (...args: unknown[]) => { + const [text, values, ...rest] = args; + const unmarked = unmarkedValues(values); + if (typeof text !== 'string' || unmarked === null) return query(...args); + return query({ text, values: unmarked, queryMode: 'extended' }, ...rest); + }; +} diff --git a/server/src/db/engine/postgres/plugin-data-store.ts b/server/src/db/engine/postgres/plugin-data-store.ts index 307ca9e64..e392dbb39 100644 --- a/server/src/db/engine/postgres/plugin-data-store.ts +++ b/server/src/db/engine/postgres/plugin-data-store.ts @@ -1,4 +1,5 @@ import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data'; +import { extendedQuery } from './extended-query'; import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema'; import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard'; @@ -9,10 +10,13 @@ import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; * database (`plugin-schema.ts`), reached through TREK's pool. * * Every call runs in one transaction that first makes sure the schema and its - * migration table exist, then puts only that schema on the `search_path` and - * sets `statement_timeout` to the call's time budget. The plugin's SQL is - * checked by `plugin-sql-guard.ts` before it runs, so it cannot name another - * schema. Reads stream through a cursor and stop at the row cap. A call that + * migration table exist, then puts only that schema on the `search_path`, with + * `pg_temp` explicitly last so a temporary table is never found before the + * plugin's own, and sets `statement_timeout` to the call's time budget. The + * plugin's SQL is checked by `plugin-sql-guard.ts` before it runs, so it cannot + * name another schema, and each statement goes to Postgres on its own over the + * extended query protocol (`extended-query.ts`), which runs one statement and + * no more. Reads stream through a cursor and stop at the row cap. A call that * may write measures the schema's size before and after, and rolls back one * that leaves it past the quota and larger than it found it, so a plugin at * its quota can still delete. @@ -267,7 +271,7 @@ export class PostgresPluginDataStore implements PluginDataStore { const setup = [ `create schema if not exists ${this.quoted}`, `create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`, - `set local search_path to ${this.quoted}`, + `set local search_path to ${this.quoted}, pg_temp`, 'set local standard_conforming_strings to on', ]; if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`); @@ -313,7 +317,11 @@ export class PostgresPluginDataStore implements PluginDataStore { return Number(result.rows[0]?.bytes ?? 0); } - /** Runs a statement for its effect: the rows it changed, or 0 for a read, whose rows are streamed past and dropped. */ + /** + * Runs a statement for its effect: the rows it changed, or 0 for a read, + * whose rows are streamed past and dropped. Both ways use the extended + * protocol: a cursor always does, and a write goes through `extendedQuery`. + */ private async run(trx: Trx, statement: PluginStatement, args: readonly unknown[]): Promise { if (!statement.mayWrite) { const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS); @@ -322,7 +330,7 @@ export class PostgresPluginDataStore implements PluginDataStore { } return 0; } - const result = await trx.executeQuery(CompiledQuery.raw(statement.text, [...args])); + const result = await trx.executeQuery(extendedQuery(statement.text, args)); return Number(result.numAffectedRows ?? 0); } diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index 272c6ae96..a646daa94 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,7 +1,13 @@ +import { sendMarkedQueriesExtended } from './extended-query'; import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; import { type Configuration } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; +/** The `pg` pool event a new client is handed to before anyone uses it. */ +interface ClientEvents { + on(event: 'connect', listener: (client: { query: (...args: unknown[]) => unknown }) => void): void; +} + /** `int8` (bigint), a `pg_type.oid`: every integer column and every `count(*)` on Postgres. */ const INT8_OID = 20; /** `numeric`: what `sum()` of a bigint and `avg()` answer. */ @@ -51,9 +57,19 @@ class NumberTypesConnection extends PostgreSqlConnection { return options; } - /** The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL position (`null-order.ts`). */ + /** + * The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL + * position (`null-order.ts`), on a pool whose every client sends the queries + * `extendedQuery` builds over the extended protocol (`extended-query.ts`). + */ override createKyselyDialect(overrides: Parameters[0]) { - return withSqliteNullOrder(super.createKyselyDialect(overrides)) as ReturnType< + const given = (overrides ?? {}) as { onPoolCreated?: (pool: unknown) => unknown }; + const onPoolCreated = (pool: unknown) => { + (pool as ClientEvents).on('connect', sendMarkedQueriesExtended); + return given.onPoolCreated?.(pool); + }; + const withHook = { ...given, onPoolCreated } as Parameters[0]; + return withSqliteNullOrder(super.createKyselyDialect(withHook)) as ReturnType< PostgreSqlConnection['createKyselyDialect'] >; } diff --git a/server/src/db/engine/postgres/postgres-engine.ts b/server/src/db/engine/postgres/postgres-engine.ts index e7b48b137..60cdaa15c 100644 --- a/server/src/db/engine/postgres/postgres-engine.ts +++ b/server/src/db/engine/postgres/postgres-engine.ts @@ -5,9 +5,12 @@ import type { DatabaseEngine } from '../engine'; import { redactUrl } from '../redact-url'; import { TrekPostgreSqlDriver } from './postgres-driver'; import { bootstrapPostgresSchema, defaultPostgresBootstrapDeps, type PostgresBootstrapDeps } from './schema-bootstrap'; +import { putTempSchemaLast } from './session-search-path'; import { sslOptionFor } from './ssl'; import type { EntityManager } from '@mikro-orm/core'; +import type { DatabaseConnection } from 'kysely'; + /** What `TREK_DB_URL` and `TREK_DB_POOL_MAX` resolve to, when they name a Postgres database. */ type PostgresSettings = Extract; @@ -78,6 +81,7 @@ export class PostgresEngine implements DatabaseEngine { return `Postgres ${redactUrl(this.settings.url)}`; } + /** Every new pool connection first puts `pg_temp` last on its `search_path` (`session-search-path.ts`). */ ormOptions() { const ssl = sslOptionFor(this.settings.url); return { @@ -85,6 +89,7 @@ export class PostgresEngine implements DatabaseEngine { clientUrl: this.settings.url, pool: { min: 0, max: this.settings.poolMax }, driverOptions: ssl === undefined ? {} : { ssl }, + onCreateConnection: (connection: unknown) => putTempSchemaLast(connection as DatabaseConnection), }; } } diff --git a/server/src/db/engine/postgres/session-search-path.ts b/server/src/db/engine/postgres/session-search-path.ts new file mode 100644 index 000000000..ef479af97 --- /dev/null +++ b/server/src/db/engine/postgres/session-search-path.ts @@ -0,0 +1,43 @@ +import { CompiledQuery, type DatabaseConnection } from 'kysely'; + +/** + * Puts `pg_temp` last on the `search_path` of every connection of TREK's pool. + * + * Postgres searches the session's temporary schema before everything on the + * `search_path` unless the path names `pg_temp` itself, and a temporary table + * lives as long as its connection. On a pool, whatever one caller left there + * would stand in for TREK's tables (`users`, `sessions`, ...) for every later + * caller of that connection. Named last, the temporary schema is searched + * after every schema TREK uses, so its tables are never found first. The rest + * of the path stays as the role or the database configured it: TREK may run + * in a schema other than `public`, and finds it through that path. + * + * Plugin transactions set their own path the same way + * (`plugin-data-store.ts`); `plugin-sql-guard.ts` explains the layers. + */ + +/** `path` (as `SHOW search_path` prints it) with every `pg_temp` entry taken out and one put last. */ +export function searchPathWithTempLast(path: string): string { + const entries: string[] = []; + let current = ''; + let quoted = false; + for (const ch of path) { + if (ch === '"') quoted = !quoted; + if (ch === ',' && !quoted) { + entries.push(current.trim()); + current = ''; + } else { + current += ch; + } + } + entries.push(current.trim()); + const kept = entries.filter((entry) => entry !== '' && entry.toLowerCase() !== 'pg_temp' && entry !== '"pg_temp"'); + return [...kept, 'pg_temp'].join(', '); +} + +/** The connection hook (`onCreateConnection`) that sets {@link searchPathWithTempLast} for the session. */ +export async function putTempSchemaLast(connection: DatabaseConnection): Promise { + const shown = await connection.executeQuery<{ search_path: string }>(CompiledQuery.raw('show search_path')); + const path = searchPathWithTempLast(shown.rows[0]?.search_path ?? ''); + await connection.executeQuery(CompiledQuery.raw("select set_config('search_path', $1, false)", [path])); +} diff --git a/server/tests/unit/db/engine/engine.test.ts b/server/tests/unit/db/engine/engine.test.ts index 0b002b733..d60b68026 100644 --- a/server/tests/unit/db/engine/engine.test.ts +++ b/server/tests/unit/db/engine/engine.test.ts @@ -30,6 +30,8 @@ describe('resolveEngine', () => { clientUrl: PG_URL, pool: { min: 0, max: 7 }, driverOptions: {}, + // Puts pg_temp last on every new connection's search_path (session-search-path.ts). + onCreateConnection: expect.any(Function), }); }); diff --git a/server/tests/unit/db/engine/postgres/extended-query.test.ts b/server/tests/unit/db/engine/postgres/extended-query.test.ts new file mode 100644 index 000000000..cad2e542e --- /dev/null +++ b/server/tests/unit/db/engine/postgres/extended-query.test.ts @@ -0,0 +1,59 @@ +/** + * The extended-protocol marker of plugin SQL + * (src/db/engine/postgres/extended-query.ts), against a stand-in `pg` client + * that records what it is handed. That Postgres then runs one statement and + * no more is PGPLUG-010 in the Postgres lane. + */ +import { + extendedQuery, + sendMarkedQueriesExtended, + unmarkedValues, +} from '../../../../../src/db/engine/postgres/extended-query'; + +import { describe, expect, it } from 'vitest'; + +/** A `pg` client that records the arguments of every `query` call. */ +function recordingClient() { + const calls: unknown[][] = []; + const client = { + query: (...args: unknown[]): unknown => { + calls.push(args); + return Promise.resolve({ rows: [] }); + }, + }; + return { client, calls }; +} + +describe('extended query', () => { + it('EXTQ-001: a marked query reaches pg as one extended-protocol query with its own values, even with none', async () => { + const { client, calls } = recordingClient(); + sendMarkedQueriesExtended(client); + const none = extendedQuery('insert into t values (1)', []); + const some = extendedQuery('insert into t values ($1, $2)', ['a', 2]); + await client.query(none.sql, none.parameters); + await client.query(some.sql, some.parameters); + expect(calls).toEqual([ + [{ text: 'insert into t values (1)', values: [], queryMode: 'extended' }], + [{ text: 'insert into t values ($1, $2)', values: ['a', 2], queryMode: 'extended' }], + ]); + }); + + it('EXTQ-002: any other query, a cursor included, reaches pg unchanged', async () => { + const { client, calls } = recordingClient(); + sendMarkedQueriesExtended(client); + const cursor = { submit: () => undefined }; + await client.query('select 1; select 2', []); + await client.query('select $1', [1]); + await client.query(cursor); + expect(calls).toEqual([['select 1; select 2', []], ['select $1', [1]], [cursor]]); + }); + + it('EXTQ-003: without the hook the marker stays a parameter, so pg still binds and Postgres refuses the count', () => { + const query = extendedQuery('select 1', []); + expect(query.parameters).toHaveLength(1); + expect(unmarkedValues(query.parameters)).toEqual([]); + expect(unmarkedValues([])).toBeNull(); + expect(unmarkedValues(['a'])).toBeNull(); + expect(unmarkedValues(undefined)).toBeNull(); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts index e98a7fe11..ff6dede07 100644 --- a/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts +++ b/server/tests/unit/db/engine/postgres/plugin-data-store.test.ts @@ -7,6 +7,7 @@ * tx and migrate shapes, and one call at a time per plugin. The same store on * a real database is the Postgres lane's tests/postgres/plugin-data.test.ts. */ +import { unmarkedValues } from '../../../../../src/db/engine/postgres/extended-query'; import { dropPluginSchema, PluginDataSlots, @@ -38,6 +39,8 @@ interface FakeOptions { /** A Kysely client that records each statement, and `begin`/`commit`/`rollback` around each transaction. */ function fakeDb(options: FakeOptions = {}) { const log: string[] = []; + /** Every statement handed to executeQuery, with its parameters as Kysely passes them to pg. */ + const executed: Array<{ sql: string; parameters: readonly unknown[] }> = []; let size = 0; const run = (sql: string) => { log.push(sql); @@ -46,6 +49,7 @@ function fakeDb(options: FakeOptions = {}) { }; const trx = { executeQuery: async (query: { sql: string; parameters: readonly unknown[] }) => { + executed.push(query); run(query.sql); if (query.sql.startsWith('select nspname')) return { rows: (options.schemas ?? []).map((nspname) => ({ nspname })) }; @@ -81,7 +85,7 @@ function fakeDb(options: FakeOptions = {}) { }), executeQuery: trx.executeQuery, }; - return { db: db as unknown as Kysely, log }; + return { db: db as unknown as Kysely, log, executed }; } function store(options: FakeOptions = {}, limits = {}) { @@ -106,7 +110,7 @@ describe('Postgres plugin data store', () => { [ 'create schema if not exists "trek_plugin_notes"', 'create table if not exists "trek_plugin_notes"._plugin_migrations (id text primary key, applied_at bigint)', - 'set local search_path to "trek_plugin_notes"', + 'set local search_path to "trek_plugin_notes", pg_temp', 'set local standard_conforming_strings to on', 'set local statement_timeout to 2000', ].join(';\n'), @@ -252,6 +256,23 @@ describe('Postgres plugin data store', () => { expect(log.filter((sql) => sql === 'begin' || sql === 'commit')).toEqual(['begin', 'commit', 'begin', 'commit']); }); + it('PGSTORE-012: every plugin statement it executes goes over the extended protocol, alone and with its own args', async () => { + const { store: s, executed } = store({ changes: 1 }); + await s.exec('INSERT INTO a VALUES (1); UPDATE a SET x = 2 -- note\rWHERE x = 1'); + await s.exec('INSERT INTO a VALUES ($1)', ['x']); + await s.tx([{ sql: 'DELETE FROM a WHERE x = $1', args: [3] }]); + await s.migrate('002', 'CREATE TABLE b (y int); CREATE INDEX b_y ON b (y)'); + const plugin = executed.filter((q) => /^(INSERT|UPDATE|DELETE|CREATE)/.test(q.sql)); + expect(plugin.map((q) => [q.sql, unmarkedValues(q.parameters)])).toEqual([ + ['INSERT INTO a VALUES (1)', []], + ['UPDATE a SET x = 2 -- note\rWHERE x = 1', []], + ['INSERT INTO a VALUES ($1)', ['x']], + ['DELETE FROM a WHERE x = $1', [3]], + ['CREATE TABLE b (y int)', []], + ['CREATE INDEX b_y ON b (y)', []], + ]); + }); + it('PGSTORE-011: dropping a plugin schema names it quoted, and an id no schema comes from drops nothing', async () => { const { db, log } = fakeDb(); await dropPluginSchema(db, 'trip-notes'); diff --git a/server/tests/unit/db/engine/postgres/session-search-path.test.ts b/server/tests/unit/db/engine/postgres/session-search-path.test.ts new file mode 100644 index 000000000..cd55538a9 --- /dev/null +++ b/server/tests/unit/db/engine/postgres/session-search-path.test.ts @@ -0,0 +1,39 @@ +/** + * `pg_temp` last on the search_path of TREK's pool connections + * (src/db/engine/postgres/session-search-path.ts), without a database. That a + * temporary table then never stands in for TREK's own is PGPLUG-011 in the + * Postgres lane. + */ +import { putTempSchemaLast, searchPathWithTempLast } from '../../../../../src/db/engine/postgres/session-search-path'; + +import type { CompiledQuery, DatabaseConnection, QueryResult } from 'kysely'; +import { describe, expect, it } from 'vitest'; + +describe('session search_path', () => { + it.each([ + ['"$user", public', '"$user", public, pg_temp'], + ['trek', 'trek, pg_temp'], + ['pg_temp, public', 'public, pg_temp'], + ['PG_TEMP,public', 'public, pg_temp'], + ['"pg_temp", "$user", public', '"$user", public, pg_temp'], + ['"a, b", "c""d", public', '"a, b", "c""d", public, pg_temp'], + ['', 'pg_temp'], + ])('SPATH-001: %j keeps its schemas in order and gets pg_temp last: %j', (path, expected) => { + expect(searchPathWithTempLast(path)).toBe(expected); + }); + + it('SPATH-002: the connection hook reads the path and sets it for the session, the value bound', async () => { + const sent: Array<{ sql: string; parameters: readonly unknown[] }> = []; + const connection = { + executeQuery: async (query: CompiledQuery): Promise> => { + sent.push({ sql: query.sql, parameters: query.parameters }); + return { rows: query.sql === 'show search_path' ? [{ search_path: 'trek, public' }] : [] }; + }, + } as unknown as DatabaseConnection; + await putTempSchemaLast(connection); + expect(sent).toEqual([ + { sql: 'show search_path', parameters: [] }, + { sql: "select set_config('search_path', $1, false)", parameters: ['trek, public, pg_temp'] }, + ]); + }); +});