mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
feat(db): keep a plugin's own data in a Postgres schema of its own
A plugin with db:own got a SQLite WAL file in the data volume whatever the engine, so a Postgres install still needed a local disk (discussion #386). This adds the plugin data port (db/engine/plugin-data.ts) and its Postgres side: one schema per plugin, trek_plugin_<id>, reached through TREK's pool. Every call runs in a transaction that creates the schema if missing, puts only it on the search_path and sets statement_timeout to the call's budget. Reads stream through a cursor and stop at the row cap, a write that grows the schema past the quota rolls back, and the calls of one plugin run one at a time on at most half of the pool. TREK's role may read every table, and a role per plugin would need CREATEROLE, so plugin-sql-guard.ts checks each statement first: it lexes the SQL as Postgres does and refuses another schema as a qualifier, the pg_ catalogs and functions, functions that run SQL from a string or reach an object by a name in one, temporary tables, and every statement but DML and DDL on tables, indexes, views, sequences and types.
This commit is contained in:
@@ -0,0 +1,378 @@
|
||||
import { PLUGIN_DATA_CAPS, type PluginDataStore, type PluginTxOp, type PluginTxResult } from '../plugin-data';
|
||||
import { isPluginId, pluginSchemaName, quotePluginSchema } from './plugin-schema';
|
||||
import { checkPlaceholders, checkQualifiers, parsePluginSql, type PluginStatement } from './plugin-sql-guard';
|
||||
|
||||
import { CompiledQuery, type Kysely, type Transaction } from 'kysely';
|
||||
|
||||
/**
|
||||
* A plugin's own data on Postgres (`db:own`): a schema of its own in TREK's
|
||||
* database (`plugin-schema.ts`), reached through TREK's pool.
|
||||
*
|
||||
* Every call runs in one transaction that first makes sure the schema and its
|
||||
* migration table exist, then puts only that schema on the `search_path` and
|
||||
* sets `statement_timeout` to the call's time budget. The plugin's SQL is
|
||||
* checked by `plugin-sql-guard.ts` before it runs, so it cannot name another
|
||||
* schema. Reads stream through a cursor and stop at the row cap. A call that
|
||||
* may write measures the schema's size before and after, and rolls back one
|
||||
* that leaves it past the quota and larger than it found it, so a plugin at
|
||||
* its quota can still delete.
|
||||
*
|
||||
* The calls of one plugin run one at a time, as they do on its SQLite file,
|
||||
* and all plugins together hold at most half of TREK's pool
|
||||
* (`pluginDataSlots`), so a busy plugin never takes the connections TREK's
|
||||
* own requests need.
|
||||
*
|
||||
* The SQLite store's semantics carry over: `query` and `tx` are held to the
|
||||
* row cap, `query`, a `tx` batch and an `exec` script to the time budget, a
|
||||
* script's statements each commit on their own, and `migrate` runs once per
|
||||
* id with no time budget. What Postgres adds is that the budget also stops a
|
||||
* single statement (`statement_timeout`), which SQLite cannot, and that an
|
||||
* `exec` with bound args is held to it as well.
|
||||
*/
|
||||
|
||||
/** The limits one store enforces. Overridable for tests; production uses `PLUGIN_DATA_CAPS`. */
|
||||
interface PostgresPluginDataLimits {
|
||||
maxRows: number;
|
||||
timeBudgetMs: number;
|
||||
quotaBytes: number;
|
||||
/** The clock the time budget reads. */
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
const DEFAULT_LIMITS: PostgresPluginDataLimits = {
|
||||
maxRows: PLUGIN_DATA_CAPS.maxRows,
|
||||
timeBudgetMs: PLUGIN_DATA_CAPS.timeBudgetMs,
|
||||
quotaBytes: PLUGIN_DATA_CAPS.quotaBytes,
|
||||
now: () => Date.now(),
|
||||
};
|
||||
|
||||
/** Postgres's SQLSTATE for a statement cancelled by `statement_timeout`. */
|
||||
const QUERY_CANCELED = '57014';
|
||||
/** Rows a cursor fetches at once. */
|
||||
const CHUNK_ROWS = 500;
|
||||
/** What better-sqlite3 says about a closed handle, which the host recognizes. */
|
||||
const CLOSED = 'The database connection is not open';
|
||||
|
||||
/** How many plugin transactions may hold a pool connection at once: half of TREK's pool, at least one. */
|
||||
export function pluginDataSlots(poolMax: number): number {
|
||||
return Math.max(1, Math.floor(poolMax / 2));
|
||||
}
|
||||
|
||||
/** A counting semaphore: at most `size` pieces of work at once, the rest wait in order. */
|
||||
export class PluginDataSlots {
|
||||
private active = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
|
||||
constructor(readonly size: number) {}
|
||||
|
||||
async run<T>(work: () => Promise<T>): Promise<T> {
|
||||
if (this.active < this.size) this.active++;
|
||||
else await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
// A waiting call takes the slot over, so the count stays where it is.
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.active--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Trx = Transaction<object>;
|
||||
|
||||
/** The rows one call has returned so far, against the cap they share. */
|
||||
interface RowBudget {
|
||||
count: number;
|
||||
readonly max: number;
|
||||
readonly message: string;
|
||||
}
|
||||
|
||||
/** Drops the plugin `id`'s schema and everything in it. An id no schema can be made from has none. */
|
||||
export async function dropPluginSchema(db: Kysely<object>, id: string): Promise<void> {
|
||||
if (!isPluginId(id)) return;
|
||||
const schema = quotePluginSchema(pluginSchemaName(id));
|
||||
await db.executeQuery(CompiledQuery.raw(`drop schema if exists ${schema} cascade`));
|
||||
}
|
||||
|
||||
export class PostgresPluginDataStore implements PluginDataStore {
|
||||
readonly engine = 'postgres' as const;
|
||||
private readonly schema: string;
|
||||
private readonly quoted: string;
|
||||
private readonly limits: PostgresPluginDataLimits;
|
||||
private open = true;
|
||||
private queue: Promise<unknown> = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
readonly pluginId: string,
|
||||
private readonly db: Kysely<object>,
|
||||
private readonly slots: PluginDataSlots,
|
||||
limits: Partial<PostgresPluginDataLimits> = {},
|
||||
) {
|
||||
this.schema = pluginSchemaName(pluginId);
|
||||
this.quoted = quotePluginSchema(this.schema);
|
||||
this.limits = { ...DEFAULT_LIMITS, ...limits };
|
||||
}
|
||||
|
||||
isOpen(): boolean {
|
||||
return this.open;
|
||||
}
|
||||
|
||||
/** Stops new calls. One already running finishes: it holds a pool connection of its own, not this store. */
|
||||
close(): void {
|
||||
this.open = false;
|
||||
}
|
||||
|
||||
async query(sql: string, args: unknown[] = []): Promise<unknown[]> {
|
||||
const statement = this.single(sql, args);
|
||||
const { maxRows } = this.limits;
|
||||
return this.call('query', (budget) =>
|
||||
this.transaction(budget.ms, [statement], (trx) => {
|
||||
const rows = { count: 0, max: maxRows, message: `query returned more than ${maxRows} rows` };
|
||||
return this.stream(trx, statement, args, rows, budget.check);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async exec(sql: string, args: unknown[] = []): Promise<{ changes: number }> {
|
||||
if (args.length > 0) {
|
||||
const statement = this.single(sql, args);
|
||||
return this.call('exec', (budget) =>
|
||||
this.transaction(budget.ms, [statement], async (trx) => ({ changes: await this.run(trx, statement, args) })),
|
||||
);
|
||||
}
|
||||
const statements = this.parse(sql);
|
||||
return this.call('exec', async (budget) => {
|
||||
// A script runs as SQLite runs one: each statement commits on its own, and
|
||||
// the clock is read before every statement but the first.
|
||||
await inOrder(statements, (statement, index) => {
|
||||
if (index > 0) budget.check();
|
||||
return this.transaction(budget.remaining(), [statement], (trx) => this.run(trx, statement, []));
|
||||
});
|
||||
return { changes: 0 };
|
||||
});
|
||||
}
|
||||
|
||||
async tx(ops: PluginTxOp[]): Promise<PluginTxResult> {
|
||||
if (!Array.isArray(ops)) throw new Error('tx requires an array of { sql, args }');
|
||||
if (ops.length === 0) return { results: [] };
|
||||
if (ops.length > PLUGIN_DATA_CAPS.maxTxOps) {
|
||||
throw new Error(`tx allows at most ${PLUGIN_DATA_CAPS.maxTxOps} statements`);
|
||||
}
|
||||
const statements = ops.map((op) => this.single(op?.sql, op?.args ?? []));
|
||||
const { maxRows } = this.limits;
|
||||
return this.call('tx', (budget) =>
|
||||
this.transaction(budget.ms, statements, async (trx) => {
|
||||
const rows = { count: 0, max: maxRows, message: `tx returned more than ${maxRows} rows in total` };
|
||||
const results = await inOrder(statements, async (statement, index) => {
|
||||
const args = ops[index].args ?? [];
|
||||
// One budget for the whole batch: each statement after the first gets what is left of it.
|
||||
if (index > 0) await this.setTimeout(trx, budget.remaining());
|
||||
const result = statement.returnsRows
|
||||
? { rows: await this.stream(trx, statement, args, rows, budget.check) }
|
||||
: { changes: await this.run(trx, statement, args) };
|
||||
budget.check();
|
||||
return result;
|
||||
});
|
||||
return { results };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** No time budget, as on SQLite: a migration stopped part of the way would fail the same way on every start. */
|
||||
async migrate(id: string, sql: string): Promise<{ applied: boolean }> {
|
||||
const statements = this.parse(sql);
|
||||
return this.enqueue(() =>
|
||||
this.transaction(null, statements, async (trx) => {
|
||||
const seen = await trx.executeQuery(
|
||||
CompiledQuery.raw(`select 1 from ${this.quoted}._plugin_migrations where id = $1`, [id]),
|
||||
);
|
||||
if (seen.rows.length > 0) return { applied: false };
|
||||
await inOrder(statements, (statement) => this.run(trx, statement, []));
|
||||
await trx.executeQuery(
|
||||
CompiledQuery.raw(`insert into ${this.quoted}._plugin_migrations (id, applied_at) values ($1, $2)`, [
|
||||
id,
|
||||
Date.now(),
|
||||
]),
|
||||
);
|
||||
return { applied: true };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** `sql` as exactly one statement, checked against `args`. */
|
||||
private single(sql: unknown, args: readonly unknown[]): PluginStatement {
|
||||
const statements = this.parse(sql);
|
||||
if (statements.length !== 1) throw new Error('sql must hold exactly one statement here; use exec() for a script');
|
||||
checkPlaceholders(statements[0], args);
|
||||
return statements[0];
|
||||
}
|
||||
|
||||
/** The statements of `sql`, refused when the guard does not allow one or none is there. */
|
||||
private parse(sql: unknown): PluginStatement[] {
|
||||
if (!this.open) throw new Error(CLOSED);
|
||||
if (typeof sql !== 'string') throw new Error('sql must be a string');
|
||||
if (sql.length > PLUGIN_DATA_CAPS.maxSqlLength) throw new Error('sql too long');
|
||||
const statements = parsePluginSql(sql);
|
||||
if (statements.length === 0) throw new Error('The supplied SQL string contains no statements');
|
||||
return statements;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `work` in this plugin's turn with a time budget that starts when its
|
||||
* turn comes, and reports a statement Postgres cancelled at
|
||||
* `statement_timeout` as the budget running out.
|
||||
*/
|
||||
private call<T>(what: string, work: (budget: Budget) => Promise<T>): Promise<T> {
|
||||
const { now, timeBudgetMs } = this.limits;
|
||||
return this.enqueue(async () => {
|
||||
const startedAt = now();
|
||||
const budget: Budget = {
|
||||
ms: timeBudgetMs,
|
||||
remaining: () => Math.max(1, timeBudgetMs - (now() - startedAt)),
|
||||
check: () => {
|
||||
if (now() - startedAt > timeBudgetMs) throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`);
|
||||
},
|
||||
};
|
||||
try {
|
||||
return await work(budget);
|
||||
} catch (err) {
|
||||
if (sqlState(err) === QUERY_CANCELED) {
|
||||
throw new Error(`${what} exceeded its ${timeBudgetMs} ms time budget`, { cause: err });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Queues `work` behind this plugin's earlier calls, then waits for a pool slot. */
|
||||
private enqueue<T>(work: () => Promise<T>): Promise<T> {
|
||||
const turn = this.queue.then(() => this.slots.run(work));
|
||||
this.queue = turn.catch(() => undefined);
|
||||
return turn;
|
||||
}
|
||||
|
||||
/**
|
||||
* One transaction on this plugin's schema: created if missing, alone on the
|
||||
* `search_path`, with `statement_timeout` at `timeoutMs` (none for null).
|
||||
* The statements' qualified names are checked against the database's
|
||||
* schemas first, and a call that may write is held to the quota.
|
||||
*/
|
||||
private transaction<T>(
|
||||
timeoutMs: number | null,
|
||||
statements: readonly PluginStatement[],
|
||||
work: (trx: Trx) => Promise<T>,
|
||||
): Promise<T> {
|
||||
return this.db.transaction().execute(async (trx) => {
|
||||
const setup = [
|
||||
`create schema if not exists ${this.quoted}`,
|
||||
`create table if not exists ${this.quoted}._plugin_migrations (id text primary key, applied_at bigint)`,
|
||||
`set local search_path to ${this.quoted}`,
|
||||
'set local standard_conforming_strings to on',
|
||||
];
|
||||
if (timeoutMs !== null) setup.push(`set local statement_timeout to ${Math.max(1, Math.floor(timeoutMs))}`);
|
||||
await trx.executeQuery(CompiledQuery.raw(setup.join(';\n')));
|
||||
if (statements.some((s) => s.qualifies)) {
|
||||
const schemas = await this.schemaNames(trx);
|
||||
for (const statement of statements) checkQualifiers(statement, this.schema, schemas);
|
||||
}
|
||||
const writes = statements.some((s) => s.mayWrite);
|
||||
const before = writes ? await this.sizeBytes(trx) : 0;
|
||||
const result = await work(trx);
|
||||
if (writes) {
|
||||
const after = await this.sizeBytes(trx);
|
||||
if (after > this.limits.quotaBytes && after > before) {
|
||||
throw new Error(`the plugin database is past its ${this.limits.quotaBytes / 1024 / 1024} MB quota`);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
private async setTimeout(trx: Trx, ms: number): Promise<void> {
|
||||
await trx.executeQuery(
|
||||
CompiledQuery.raw("select set_config('statement_timeout', $1, true)", [String(Math.max(1, Math.floor(ms)))]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Every schema name of the database, which a qualified name in plugin SQL may not use. */
|
||||
private async schemaNames(trx: Trx): Promise<Set<string>> {
|
||||
const result = await trx.executeQuery<{ nspname: string }>(CompiledQuery.raw('select nspname from pg_namespace'));
|
||||
return new Set(result.rows.map((row) => row.nspname));
|
||||
}
|
||||
|
||||
/** What this plugin's tables, indexes and materialized views take on disk. */
|
||||
private async sizeBytes(trx: Trx): Promise<number> {
|
||||
const result = await trx.executeQuery<{ bytes: number | string }>(
|
||||
CompiledQuery.raw(
|
||||
'select coalesce(sum(pg_total_relation_size(c.oid)), 0) as bytes from pg_class c ' +
|
||||
"join pg_namespace n on n.oid = c.relnamespace where n.nspname = $1 and c.relkind in ('r', 'm')",
|
||||
[this.schema],
|
||||
),
|
||||
);
|
||||
return Number(result.rows[0]?.bytes ?? 0);
|
||||
}
|
||||
|
||||
/** Runs a statement for its effect: the rows it changed, or 0 for a read, whose rows are streamed past and dropped. */
|
||||
private async run(trx: Trx, statement: PluginStatement, args: readonly unknown[]): Promise<number> {
|
||||
if (!statement.mayWrite) {
|
||||
const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS);
|
||||
for await (const _chunk of chunks) {
|
||||
// Read to the end so the statement runs whole; its rows are not wanted.
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
const result = await trx.executeQuery(CompiledQuery.raw(statement.text, [...args]));
|
||||
return Number(result.numAffectedRows ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows of a statement, read through a cursor a chunk at a time: past the
|
||||
* row cap the read stops and throws, and the clock is read between chunks.
|
||||
*/
|
||||
private async stream(
|
||||
trx: Trx,
|
||||
statement: PluginStatement,
|
||||
args: readonly unknown[],
|
||||
budget: RowBudget,
|
||||
checkTime: () => void,
|
||||
): Promise<unknown[]> {
|
||||
const rows: unknown[] = [];
|
||||
const chunks = trx.getExecutor().stream(CompiledQuery.raw(statement.text, [...args]), CHUNK_ROWS);
|
||||
for await (const chunk of chunks) {
|
||||
if (rows.length > 0) checkTime();
|
||||
for (const row of chunk.rows) {
|
||||
rows.push(row);
|
||||
budget.count++;
|
||||
if (budget.count > budget.max) throw new Error(budget.message);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
/** `work` for each item, one after the other, each started once the one before it has finished. */
|
||||
async function inOrder<T, R>(items: readonly T[], work: (item: T, index: number) => Promise<R>): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
await items.reduce<Promise<void>>(
|
||||
(previous, item, index) =>
|
||||
previous.then(async () => {
|
||||
results.push(await work(item, index));
|
||||
}),
|
||||
Promise.resolve(),
|
||||
);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** A call's time budget: its whole length, what is left of it, and a check that throws once it is spent. */
|
||||
interface Budget {
|
||||
readonly ms: number;
|
||||
remaining(): number;
|
||||
check(): void;
|
||||
}
|
||||
|
||||
/** The SQLSTATE of a `pg` error, or null. */
|
||||
function sqlState(err: unknown): string | null {
|
||||
if (typeof err !== 'object' || err === null || !('code' in err)) return null;
|
||||
return typeof err.code === 'string' ? err.code : null;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* The schema a plugin's own data lives in on Postgres (`db:own`): one per
|
||||
* plugin, named from its id, beside TREK's schema in the same database.
|
||||
*
|
||||
* A plugin id is what the manifest allows (`PLUGIN_ID_RE` in
|
||||
* `nest/plugins/protocol/manifest-rules.ts`): a lowercase letter, then 2 to 39
|
||||
* lowercase letters, digits or dashes. The schema name is the prefix plus the
|
||||
* id with each dash spelled as an underscore. An id never holds an underscore,
|
||||
* so two ids never share a name, and the longest name (52 characters) stays
|
||||
* under Postgres's 63. The pattern is repeated here rather than imported,
|
||||
* because `src/db` does not import `src/nest`; a unit test holds the two equal.
|
||||
*
|
||||
* Every name TREK gives a plugin schema starts with the prefix, so a backup,
|
||||
* a restore and the plugin SQL guard can tell them from every other schema of
|
||||
* the database. Two TREK instances on one database would share these names,
|
||||
* so each instance needs a database of its own (the wiki says so).
|
||||
*/
|
||||
|
||||
/** What every plugin schema's name starts with. */
|
||||
export const PLUGIN_SCHEMA_PREFIX = 'trek_plugin_';
|
||||
|
||||
/** The plugin ids a schema name can be made from: the manifest's rule. */
|
||||
const PLUGIN_ID = /^[a-z][a-z0-9-]{2,39}$/;
|
||||
|
||||
/** A plugin schema's name, as `pluginSchemaName` makes it. */
|
||||
const PLUGIN_SCHEMA = /^trek_plugin_[a-z][a-z0-9_]{2,39}$/;
|
||||
|
||||
/** Whether `id` is a plugin id a schema name can be made from. */
|
||||
export function isPluginId(id: string): boolean {
|
||||
return PLUGIN_ID.test(id);
|
||||
}
|
||||
|
||||
/** The schema of the plugin `id`. Throws for anything the manifest would not accept as an id. */
|
||||
export function pluginSchemaName(id: string): string {
|
||||
if (!isPluginId(id)) throw new Error(`"${id}" is not a plugin id, so it has no data schema.`);
|
||||
return PLUGIN_SCHEMA_PREFIX + id.replaceAll('-', '_');
|
||||
}
|
||||
|
||||
/** Whether `name` is a schema name `pluginSchemaName` makes. */
|
||||
export function isPluginSchemaName(name: string): boolean {
|
||||
return PLUGIN_SCHEMA.test(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* `name` as a quoted SQL identifier. Only plugin schema names reach it, which
|
||||
* hold no quote, so the quoting cannot be escaped; it is checked all the same.
|
||||
*/
|
||||
export function quotePluginSchema(name: string): string {
|
||||
if (!isPluginSchemaName(name)) throw new Error(`"${name}" is not a plugin schema name.`);
|
||||
return `"${name}"`;
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
import { PLUGIN_SCHEMA_PREFIX } from './plugin-schema';
|
||||
|
||||
/**
|
||||
* What keeps a plugin's own SQL on Postgres inside its own schema.
|
||||
*
|
||||
* On SQLite a plugin's database is a file of its own, so it cannot name
|
||||
* TREK's tables at all. On Postgres its schema sits in TREK's database and
|
||||
* its statements run as TREK's role, which may read every table TREK has: a
|
||||
* role per plugin would need CREATEROLE, which TREK's role is not given. So
|
||||
* the statement itself is held to what cannot leave the schema, before it
|
||||
* runs, and the transaction it runs in names only that schema on its
|
||||
* `search_path`, so an unqualified name finds nothing else.
|
||||
*
|
||||
* The SQL is split into tokens the way Postgres's own lexer splits it (string
|
||||
* and identifier quoting, comments, dollar signs), and refused when it holds:
|
||||
*
|
||||
* - a statement other than SELECT, INSERT, UPDATE, DELETE, MERGE, WITH,
|
||||
* VALUES, TABLE, TRUNCATE, REFRESH, or a CREATE, ALTER or DROP of a table,
|
||||
* index, view, materialized view, sequence or type. No function, schema,
|
||||
* role, setting, transaction control, COPY, LOCK, prepared statement or
|
||||
* notification. Several statements are split at their semicolons and each
|
||||
* is checked;
|
||||
* - a qualified name whose qualifier is another schema of the database
|
||||
* (`public.users`, another plugin's schema), or any name starting with
|
||||
* `pg_` or naming `information_schema`: the catalogs, the server functions
|
||||
* and the temporary schema;
|
||||
* - a function that runs SQL handed to it as text (`query_to_xml`, `ts_stat`,
|
||||
* `ts_rewrite`), reads or changes settings (`set_config`,
|
||||
* `current_setting`), reaches objects by a name in a string (`nextval`,
|
||||
* `setval`, the `reg*` casts), large objects (`lo_*`) or `dblink`;
|
||||
* - a temporary table (`CREATE TEMP`, `SELECT ... INTO`), which would outlive
|
||||
* the call on a pooled connection another plugin uses next, and
|
||||
* `WITH RECURSIVE`, refused on SQLite too;
|
||||
* - in an ALTER, a change of a table's schema, owner or tablespace;
|
||||
* - escape strings (`E'...'`), Unicode escapes (`U&'...'`), dollar-quoted
|
||||
* strings and a number running straight into a name, whose reading
|
||||
* differs between Postgres versions and settings.
|
||||
*
|
||||
* An alias that happens to share a schema's name (`auth.id` on a database
|
||||
* that has an `auth` schema) is refused too; the plugin renames the alias.
|
||||
*/
|
||||
|
||||
/** Why a plugin's SQL is refused. The message is the plugin's to read, so it says what to change. */
|
||||
export class PluginSqlRefused extends Error {
|
||||
constructor(reason: string) {
|
||||
super(`statement type not allowed for plugin databases: ${reason}`);
|
||||
this.name = 'PluginSqlRefused';
|
||||
}
|
||||
}
|
||||
|
||||
/** One token of plugin SQL. A word is a name or a keyword: `value` is folded to lower case unless it was quoted. */
|
||||
type Token = (
|
||||
| { kind: 'word'; value: string; quoted: boolean }
|
||||
| { kind: 'string' }
|
||||
| { kind: 'number' }
|
||||
| { kind: 'param' }
|
||||
| { kind: 'punct'; value: string }
|
||||
) & {
|
||||
/** Where the token starts in the SQL text. */
|
||||
at: number;
|
||||
};
|
||||
|
||||
/** One statement of plugin SQL: its tokens, without the semicolon that ended it. */
|
||||
export interface PluginStatement {
|
||||
/** The statement's own text, as Postgres is handed it. */
|
||||
readonly text: string;
|
||||
readonly tokens: readonly Token[];
|
||||
/** Whether it returns rows: a SELECT, VALUES or TABLE, or anything with a RETURNING clause. */
|
||||
readonly returnsRows: boolean;
|
||||
/** Whether it may change data or schema, which is what the disk quota is checked after. */
|
||||
readonly mayWrite: boolean;
|
||||
/** Whether it names a qualified object (`x.y`), which needs the database's schema names to check. */
|
||||
readonly qualifies: boolean;
|
||||
/** How many `$n` placeholders it holds. */
|
||||
readonly params: number;
|
||||
}
|
||||
|
||||
/** Postgres keeps the first 63 bytes of a name (NAMEDATALEN - 1) and drops the rest. */
|
||||
const MAX_NAME_BYTES = 63;
|
||||
|
||||
/** The first word of each statement a plugin may run. */
|
||||
const STATEMENTS = new Set([
|
||||
'select',
|
||||
'insert',
|
||||
'update',
|
||||
'delete',
|
||||
'merge',
|
||||
'with',
|
||||
'values',
|
||||
'table',
|
||||
'truncate',
|
||||
'refresh',
|
||||
'create',
|
||||
'alter',
|
||||
'drop',
|
||||
]);
|
||||
|
||||
/** Transaction control: every call already runs in a transaction of the host's. */
|
||||
const TRANSACTION_CONTROL = new Set(['begin', 'start', 'commit', 'end', 'rollback', 'abort', 'savepoint', 'release']);
|
||||
|
||||
/** What a CREATE, ALTER or DROP may act on. */
|
||||
const OBJECTS = new Set(['table', 'index', 'view', 'materialized', 'sequence', 'type']);
|
||||
|
||||
/** The words that make up a statement's main verb, after a WITH clause. */
|
||||
const VERBS = new Set(['select', 'insert', 'update', 'delete', 'merge', 'values', 'table']);
|
||||
|
||||
/** The verbs of a statement that only reads. */
|
||||
const READ_VERBS = new Set(['select', 'values', 'table']);
|
||||
|
||||
/** Functions and types refused under any spelling, quoted or not. */
|
||||
const REFUSED_NAMES = new Set([
|
||||
'set_config',
|
||||
'current_setting',
|
||||
'ts_stat',
|
||||
'ts_rewrite',
|
||||
'nextval',
|
||||
'setval',
|
||||
'currval',
|
||||
'lastval',
|
||||
'loread',
|
||||
'lowrite',
|
||||
'regclass',
|
||||
'regproc',
|
||||
'regprocedure',
|
||||
'regoper',
|
||||
'regoperator',
|
||||
'regnamespace',
|
||||
'regrole',
|
||||
'regcollation',
|
||||
'information_schema',
|
||||
]);
|
||||
|
||||
/** Name prefixes refused under any spelling. */
|
||||
const REFUSED_PREFIXES = ['pg_', 'lo_', 'dblink', 'to_reg', 'binary_upgrade_'];
|
||||
|
||||
/** A name refused wherever it appears: the XML exports of a query, a table, a schema or the database. */
|
||||
const XML_EXPORT = /_to_xml/;
|
||||
|
||||
/** Starts a name: a letter, an underscore or any character past ASCII, as Postgres reads it. */
|
||||
const NAME_START = /[A-Za-z_\u0080-]/;
|
||||
/** Continues a name; a dollar sign is part of one. */
|
||||
const NAME_PART = /[A-Za-z0-9_$\u0080-]/;
|
||||
/** The single-character punctuation and operator characters. */
|
||||
const PUNCT = new Set([...'.,()[];+-*/<>=~!@#%^&|?']);
|
||||
|
||||
/** `name` lower-cased the way Postgres folds an unquoted name: ASCII letters only. */
|
||||
function fold(name: string): string {
|
||||
return name.replace(/[A-Z]/g, (c) => c.toLowerCase());
|
||||
}
|
||||
|
||||
/** `name` cut to the bytes Postgres keeps of it, never in the middle of a character. */
|
||||
function truncateName(name: string): string {
|
||||
if (Buffer.byteLength(name, 'utf8') <= MAX_NAME_BYTES) return name;
|
||||
let out = '';
|
||||
for (const ch of name) {
|
||||
if (Buffer.byteLength(out + ch, 'utf8') > MAX_NAME_BYTES) break;
|
||||
out += ch;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Splits `sql` into tokens, or refuses what this guard does not read the way Postgres would. */
|
||||
export function tokenize(sql: string): Token[] {
|
||||
const tokens: Token[] = [];
|
||||
let i = 0;
|
||||
const n = sql.length;
|
||||
while (i < n) {
|
||||
const at = i;
|
||||
const c = sql[i];
|
||||
const next = sql[i + 1];
|
||||
if (/\s/.test(c)) {
|
||||
i++;
|
||||
} else if (c === '-' && next === '-') {
|
||||
while (i < n && sql[i] !== '\n') i++;
|
||||
} else if (c === '/' && next === '*') {
|
||||
i = skipBlockComment(sql, i);
|
||||
} else if (c === "'") {
|
||||
i = skipQuoted(sql, i, "'", 'a string');
|
||||
tokens.push({ kind: 'string', at });
|
||||
} else if (c === '"') {
|
||||
const end = skipQuoted(sql, i, '"', 'a quoted name');
|
||||
const value = sql.slice(i + 1, end - 1).replaceAll('""', '"');
|
||||
if (value === '') throw new PluginSqlRefused('a quoted name may not be empty');
|
||||
tokens.push({ kind: 'word', value: truncateName(value), quoted: true, at });
|
||||
i = end;
|
||||
} else if (c === '$') {
|
||||
if (!/[0-9]/.test(next ?? '')) {
|
||||
throw new PluginSqlRefused('dollar-quoted strings are not accepted; write the string in single quotes');
|
||||
}
|
||||
i++;
|
||||
while (i < n && /[0-9]/.test(sql[i])) i++;
|
||||
tokens.push({ kind: 'param', at });
|
||||
} else if (/[0-9]/.test(c) || (c === '.' && /[0-9]/.test(next ?? ''))) {
|
||||
i = skipNumber(sql, i);
|
||||
if (i < n && NAME_PART.test(sql[i])) {
|
||||
throw new PluginSqlRefused('a number may not run straight into a name; put a space between them');
|
||||
}
|
||||
tokens.push({ kind: 'number', at });
|
||||
} else if (NAME_START.test(c)) {
|
||||
if ((c === 'e' || c === 'E') && next === "'") {
|
||||
throw new PluginSqlRefused("escape strings (E'...') are not accepted; write the string in single quotes");
|
||||
}
|
||||
if ((c === 'u' || c === 'U') && next === '&' && (sql[i + 2] === "'" || sql[i + 2] === '"')) {
|
||||
throw new PluginSqlRefused('Unicode escapes (U&) are not accepted');
|
||||
}
|
||||
while (i < n && NAME_PART.test(sql[i])) i++;
|
||||
tokens.push({ kind: 'word', value: truncateName(fold(sql.slice(at, i))), quoted: false, at });
|
||||
} else if (c === ':' && next === ':') {
|
||||
tokens.push({ kind: 'punct', value: '::', at });
|
||||
i += 2;
|
||||
} else if (c === ':' || PUNCT.has(c)) {
|
||||
tokens.push({ kind: 'punct', value: c, at });
|
||||
i++;
|
||||
} else {
|
||||
throw new PluginSqlRefused(`unexpected character ${JSON.stringify(c)}`);
|
||||
}
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/** The index past a block comment starting at `start`; Postgres nests them. */
|
||||
function skipBlockComment(sql: string, start: number): number {
|
||||
let depth = 0;
|
||||
let i = start;
|
||||
while (i < sql.length) {
|
||||
if (sql[i] === '/' && sql[i + 1] === '*') {
|
||||
depth++;
|
||||
i += 2;
|
||||
} else if (sql[i] === '*' && sql[i + 1] === '/') {
|
||||
depth--;
|
||||
i += 2;
|
||||
if (depth === 0) return i;
|
||||
} else {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
throw new PluginSqlRefused('a comment is not closed');
|
||||
}
|
||||
|
||||
/** The index past a quoted string or name starting at `start`; a doubled quote stands for one. */
|
||||
function skipQuoted(sql: string, start: number, quote: string, what: string): number {
|
||||
let i = start + 1;
|
||||
while (i < sql.length) {
|
||||
if (sql[i] === quote) {
|
||||
if (sql[i + 1] === quote) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
return i + 1;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
throw new PluginSqlRefused(`${what} is not closed`);
|
||||
}
|
||||
|
||||
/** The index past a numeric constant starting at `start`: decimal, with a fraction and an exponent, or 0x/0o/0b. */
|
||||
function skipNumber(sql: string, start: number): number {
|
||||
const rest = sql.slice(start);
|
||||
const match =
|
||||
/^0[xX][0-9a-fA-F_]+|^0[oO][0-7_]+|^0[bB][01_]+|^(?:[0-9][0-9_]*(?:\.[0-9_]*)?|\.[0-9][0-9_]*)(?:[eE][+-]?[0-9]+)?/.exec(
|
||||
rest,
|
||||
);
|
||||
return start + (match ? match[0].length : 1);
|
||||
}
|
||||
|
||||
/** Splits `sql` into its statements, each checked on its own; empty ones (`;;`, a comment) are dropped. */
|
||||
export function parsePluginSql(sql: string): PluginStatement[] {
|
||||
const statements: PluginStatement[] = [];
|
||||
let current: Token[] = [];
|
||||
const flush = (end: number) => {
|
||||
if (current.length > 0) statements.push(describe(sql.slice(current[0].at, end).trimEnd(), current));
|
||||
current = [];
|
||||
};
|
||||
for (const token of tokenize(sql)) {
|
||||
if (token.kind === 'punct' && token.value === ';') flush(token.at);
|
||||
else current.push(token);
|
||||
}
|
||||
flush(sql.length);
|
||||
return statements;
|
||||
}
|
||||
|
||||
/** The unquoted keyword at `token`, or null for anything else. */
|
||||
function keyword(token: Token | undefined): string | null {
|
||||
return token?.kind === 'word' && !token.quoted ? token.value : null;
|
||||
}
|
||||
|
||||
/** Reads what a statement's caller needs to know about it, and refuses one this guard does not allow. */
|
||||
function describe(text: string, tokens: Token[]): PluginStatement {
|
||||
const first = tokens[0];
|
||||
const opensWithParen = first.kind === 'punct' && first.value === '(';
|
||||
const verb = opensWithParen ? 'select' : keyword(first);
|
||||
if (verb !== null && TRANSACTION_CONTROL.has(verb)) {
|
||||
throw new PluginSqlRefused(
|
||||
'transaction-control statements are not allowed: each call runs in its own transaction, and tx() makes a batch atomic',
|
||||
);
|
||||
}
|
||||
if (verb === null || !STATEMENTS.has(verb)) {
|
||||
throw new PluginSqlRefused(`a statement may not start with ${describeToken(first)}`);
|
||||
}
|
||||
if (verb === 'create' || verb === 'alter' || verb === 'drop' || verb === 'refresh') checkObject(verb, tokens);
|
||||
|
||||
let depth = 0;
|
||||
let main: string | null = verb === 'with' ? null : verb;
|
||||
let returning = false;
|
||||
let selectInto = false;
|
||||
let params = 0;
|
||||
let qualifies = false;
|
||||
for (const [index, token] of tokens.entries()) {
|
||||
if (token.kind === 'param') params++;
|
||||
if (token.kind === 'punct' && token.value === '(') depth++;
|
||||
if (token.kind === 'punct' && token.value === ')') depth--;
|
||||
if (token.kind !== 'word') continue;
|
||||
checkName(token.value);
|
||||
const after = tokens[index + 1];
|
||||
if (after?.kind === 'punct' && after.value === '.') qualifies = true;
|
||||
if (token.quoted) continue;
|
||||
if (token.value === 'recursive') throw new PluginSqlRefused('WITH RECURSIVE is not allowed');
|
||||
if (token.value === 'tablespace') throw new PluginSqlRefused('a tablespace may not be named');
|
||||
if (verb === 'alter' && (token.value === 'schema' || token.value === 'owner')) {
|
||||
throw new PluginSqlRefused(`an ALTER may not change the ${token.value}`);
|
||||
}
|
||||
if (depth !== 0) continue;
|
||||
if (main === null && VERBS.has(token.value)) main = token.value;
|
||||
if (token.value === 'returning') returning = true;
|
||||
if (token.value === 'into' && main === 'select') selectInto = true;
|
||||
}
|
||||
if (selectInto) throw new PluginSqlRefused('SELECT ... INTO is not allowed; use CREATE TABLE ... AS');
|
||||
|
||||
const reads = main !== null && READ_VERBS.has(main);
|
||||
const writesInside = tokens.some((t) => {
|
||||
const word = keyword(t);
|
||||
return word !== null && ['insert', 'update', 'delete', 'merge'].includes(word);
|
||||
});
|
||||
return {
|
||||
text,
|
||||
tokens,
|
||||
returnsRows: reads || returning,
|
||||
mayWrite: !reads || writesInside,
|
||||
qualifies,
|
||||
params,
|
||||
};
|
||||
}
|
||||
|
||||
/** Refuses a CREATE, ALTER, DROP or REFRESH of anything but the objects a plugin keeps in its schema. */
|
||||
function checkObject(verb: string, tokens: readonly Token[]): void {
|
||||
let at = 1;
|
||||
if (verb === 'create') {
|
||||
if (keyword(tokens[at]) === 'or' && keyword(tokens[at + 1]) === 'replace') at += 2;
|
||||
while (keyword(tokens[at]) === 'unique' || keyword(tokens[at]) === 'unlogged') at++;
|
||||
}
|
||||
const object = keyword(tokens[at]);
|
||||
const allowed = verb === 'refresh' ? object === 'materialized' : object !== null && OBJECTS.has(object);
|
||||
if (!allowed) {
|
||||
throw new PluginSqlRefused(`${verb.toUpperCase()} ${describeToken(tokens[at])} is not allowed`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Refuses a name that reaches past the plugin's schema whatever qualifies it. */
|
||||
function checkName(name: string): void {
|
||||
if (REFUSED_NAMES.has(name) || REFUSED_PREFIXES.some((p) => name.startsWith(p)) || XML_EXPORT.test(name)) {
|
||||
throw new PluginSqlRefused(`"${name}" may not be named in a plugin's SQL`);
|
||||
}
|
||||
}
|
||||
|
||||
/** A token as an error message shows it. */
|
||||
function describeToken(token: Token | undefined): string {
|
||||
if (token === undefined) return 'nothing';
|
||||
if (token.kind === 'word') return token.quoted ? `"${token.value}"` : token.value.toUpperCase();
|
||||
if (token.kind === 'punct') return `"${token.value}"`;
|
||||
return `a ${token.kind}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses a qualified name in `statement` whose qualifier is a schema other
|
||||
* than `ownSchema`: one of `schemas` (every schema of the database), or one
|
||||
* named like a plugin schema even before it exists.
|
||||
*/
|
||||
export function checkQualifiers(statement: PluginStatement, ownSchema: string, schemas: ReadonlySet<string>): void {
|
||||
const { tokens } = statement;
|
||||
for (const [index, token] of tokens.entries()) {
|
||||
const after = tokens[index + 1];
|
||||
if (token.kind !== 'word' || after?.kind !== 'punct' || after.value !== '.') continue;
|
||||
if (token.value === ownSchema) continue;
|
||||
if (schemas.has(token.value) || token.value.startsWith(PLUGIN_SCHEMA_PREFIX) || token.value === 'public') {
|
||||
throw new PluginSqlRefused(`"${token.value}" is another schema; a plugin reaches its own tables only`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuses `?` placeholders where bound args were given: Postgres numbers its
|
||||
* placeholders `$1`, `$2`, ..., and reads a `?` as an operator.
|
||||
*/
|
||||
export function checkPlaceholders(statement: PluginStatement, args: readonly unknown[]): void {
|
||||
if (args.length === 0 || statement.params > 0) return;
|
||||
const questionMark = statement.tokens.some((t) => t.kind === 'punct' && t.value === '?');
|
||||
if (questionMark) {
|
||||
throw new PluginSqlRefused('this TREK runs on Postgres, which numbers its placeholders $1, $2, ... instead of ?');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user