mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 03:26:07 +00:00
feat(db): keep a plugin's own data in a Postgres schema of its own
A plugin with db:own got a SQLite WAL file in the data volume whatever the engine, so a Postgres install still needed a local disk (discussion #386). This adds the plugin data port (db/engine/plugin-data.ts) and its Postgres side: one schema per plugin, trek_plugin_<id>, reached through TREK's pool. Every call runs in a transaction that creates the schema if missing, puts only it on the search_path and sets statement_timeout to the call's budget. Reads stream through a cursor and stop at the row cap, a write that grows the schema past the quota rolls back, and the calls of one plugin run one at a time on at most half of the pool. TREK's role may read every table, and a role per plugin would need CREATEROLE, so plugin-sql-guard.ts checks each statement first: it lexes the SQL as Postgres does and refuses another schema as a qualifier, the pg_ catalogs and functions, functions that run SQL from a string or reach an object by a name in one, temporary tables, and every statement but DML and DDL on tables, indexes, views, sequences and types.
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* The schema a plugin's own data lives in on Postgres
|
||||
* (src/db/engine/postgres/plugin-schema.ts), the restore's statements for the
|
||||
* plugin schemas (clear-schema.ts) and the pool slots the plugin stores share
|
||||
* (plugin-data-store.ts), without a database.
|
||||
*/
|
||||
import { pluginSchemasSql } from '../../../../../src/db/engine/postgres/clear-schema';
|
||||
import { PluginDataSlots, pluginDataSlots } from '../../../../../src/db/engine/postgres/plugin-data-store';
|
||||
import {
|
||||
isPluginId,
|
||||
isPluginSchemaName,
|
||||
PLUGIN_SCHEMA_PREFIX,
|
||||
pluginSchemaName,
|
||||
quotePluginSchema,
|
||||
} from '../../../../../src/db/engine/postgres/plugin-schema';
|
||||
import { PLUGIN_ID_RE } from '../../../../../src/nest/plugins/protocol/manifest-rules';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
describe('plugin schema names', () => {
|
||||
it('PGSCHEMA-001: a name is the prefix plus the id, each dash an underscore', () => {
|
||||
expect(pluginSchemaName('notes')).toBe('trek_plugin_notes');
|
||||
expect(pluginSchemaName('trip-weather-2')).toBe('trek_plugin_trip_weather_2');
|
||||
expect(quotePluginSchema(pluginSchemaName('trip-weather-2'))).toBe('"trek_plugin_trip_weather_2"');
|
||||
});
|
||||
|
||||
it('PGSCHEMA-002: accepts exactly the ids the manifest accepts', () => {
|
||||
const samples = ['abc', 'a-b', 'a--b', 'z9', 'notes', 'Notes', '9abc', '-abc', 'a_b', 'a.b', '', 'a'.repeat(40)];
|
||||
samples.push('a'.repeat(41), '../x', 'abc ', 'ab"c', 'abc;drop');
|
||||
for (const id of samples) expect(isPluginId(id), id).toBe(PLUGIN_ID_RE.test(id));
|
||||
expect(String(PLUGIN_ID_RE)).toBe('/^[a-z][a-z0-9-]{2,39}$/');
|
||||
});
|
||||
|
||||
it('PGSCHEMA-003: refuses anything that is not an id, before any SQL is written', () => {
|
||||
for (const id of ['Notes', '../x', 'a"b', 'ab', 'a b', 'a'.repeat(41)]) {
|
||||
expect(() => pluginSchemaName(id), id).toThrow(/is not a plugin id/);
|
||||
}
|
||||
expect(() => quotePluginSchema('public')).toThrow(/is not a plugin schema name/);
|
||||
expect(() => quotePluginSchema('trek_plugin_x"; drop')).toThrow(/is not a plugin schema name/);
|
||||
});
|
||||
|
||||
it('PGSCHEMA-004: two ids never share a name, and every name fits in a Postgres identifier', () => {
|
||||
expect(pluginSchemaName('a-b-c')).not.toBe(pluginSchemaName('a--bc'));
|
||||
const longest = pluginSchemaName(`a${'-'.repeat(39)}`);
|
||||
expect(longest.length).toBeLessThanOrEqual(63);
|
||||
expect(isPluginSchemaName(longest)).toBe(true);
|
||||
expect(isPluginSchemaName('trek_plugin_ab')).toBe(false);
|
||||
expect(isPluginSchemaName('public')).toBe(false);
|
||||
expect(isPluginSchemaName(`${PLUGIN_SCHEMA_PREFIX}Notes`)).toBe(false);
|
||||
});
|
||||
|
||||
it("PGSCHEMA-005: a restore drops only the role's own plugin schemas, then creates the dump's", () => {
|
||||
const sql = pluginSchemasSql(['trek_plugin_a', 'trek_plugin_b']);
|
||||
expect(sql).toContain("starts_with(nspname, 'trek_plugin_')");
|
||||
expect(sql).toContain('pg_get_userbyid(nspowner) = current_user');
|
||||
expect(sql).toContain("EXECUTE format('DROP SCHEMA %I CASCADE', r.nspname);");
|
||||
expect(sql.endsWith('CREATE SCHEMA "trek_plugin_a";\nCREATE SCHEMA "trek_plugin_b";\n')).toBe(true);
|
||||
expect(pluginSchemasSql([])).not.toContain('CREATE SCHEMA');
|
||||
expect(() => pluginSchemasSql(['auth'])).toThrow(/is not a plugin schema name/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('plugin data pool slots', () => {
|
||||
it('PGSCHEMA-006: plugins hold at most half of the pool, at least one connection', () => {
|
||||
expect(pluginDataSlots(10)).toBe(5);
|
||||
expect(pluginDataSlots(3)).toBe(1);
|
||||
expect(pluginDataSlots(1)).toBe(1);
|
||||
});
|
||||
|
||||
it('PGSCHEMA-007: no more than `size` pieces of work run at once, and the rest run in order', async () => {
|
||||
const slots = new PluginDataSlots(2);
|
||||
let running = 0;
|
||||
let most = 0;
|
||||
const order: number[] = [];
|
||||
const work = (n: number) =>
|
||||
slots.run(async () => {
|
||||
running++;
|
||||
most = Math.max(most, running);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
order.push(n);
|
||||
running--;
|
||||
if (n === 2) throw new Error('fails');
|
||||
return n;
|
||||
});
|
||||
const results = await Promise.allSettled([1, 2, 3, 4, 5].map(work));
|
||||
expect(most).toBe(2);
|
||||
expect(order.slice(2)).toEqual([3, 4, 5]);
|
||||
expect(results.map((r) => r.status)).toEqual(['fulfilled', 'rejected', 'fulfilled', 'fulfilled', 'fulfilled']);
|
||||
// A failure gives its slot back like a success does.
|
||||
await expect(slots.run(async () => 'next')).resolves.toBe('next');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* The guard that keeps a plugin's own SQL inside its schema on Postgres
|
||||
* (src/db/engine/postgres/plugin-sql-guard.ts), without a database: what it
|
||||
* lets through, what it refuses and why, how it reads quoting and comments
|
||||
* the way Postgres does, and what it tells the store about a statement. The
|
||||
* same rules against a real database are PGPLUG-003/004 in the Postgres lane.
|
||||
*/
|
||||
import {
|
||||
checkPlaceholders,
|
||||
checkQualifiers,
|
||||
parsePluginSql,
|
||||
PluginSqlRefused,
|
||||
tokenize,
|
||||
} from '../../../../../src/db/engine/postgres/plugin-sql-guard';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const OWN = 'trek_plugin_notes';
|
||||
const SCHEMAS = new Set(['public', 'auth', 'pg_catalog', 'information_schema', OWN, 'trek_plugin_other']);
|
||||
|
||||
/** Parses `sql` and checks its qualified names as the store does. */
|
||||
function check(sql: string): void {
|
||||
for (const statement of parsePluginSql(sql)) checkQualifiers(statement, OWN, SCHEMAS);
|
||||
}
|
||||
|
||||
describe('plugin SQL guard', () => {
|
||||
it.each([
|
||||
'SELECT * FROM notes WHERE id = $1',
|
||||
'select n.body, t.name from notes n join tags t on t.note_id = n.id',
|
||||
'INSERT INTO notes (body) VALUES ($1) RETURNING id',
|
||||
"UPDATE notes SET body = 'it''s' WHERE id = 1",
|
||||
'DELETE FROM notes WHERE id IN (SELECT note_id FROM trash)',
|
||||
'WITH recent AS (SELECT * FROM notes ORDER BY id DESC LIMIT 5) SELECT count(*) FROM recent',
|
||||
'VALUES (1), (2)',
|
||||
'TABLE notes',
|
||||
'(SELECT 1) UNION (SELECT 2)',
|
||||
'CREATE TABLE notes (id bigserial PRIMARY KEY, body text, temp real, schema text)',
|
||||
'CREATE UNIQUE INDEX notes_body ON notes (body)',
|
||||
'CREATE UNLOGGED TABLE cache (k text)',
|
||||
'CREATE OR REPLACE VIEW v AS SELECT body FROM notes',
|
||||
'CREATE MATERIALIZED VIEW mv AS SELECT 1',
|
||||
'CREATE SEQUENCE s',
|
||||
"CREATE TYPE mood AS ENUM ('ok', 'sad')",
|
||||
'ALTER TABLE notes ADD COLUMN tag text',
|
||||
'DROP TABLE IF EXISTS notes CASCADE',
|
||||
'TRUNCATE notes',
|
||||
'REFRESH MATERIALIZED VIEW mv',
|
||||
'SELECT body FROM trek_plugin_notes.notes',
|
||||
'SELECT "notes"."body" FROM "notes"',
|
||||
"SELECT 'public.users -- not code' AS text",
|
||||
'SELECT 1 -- public.users\n',
|
||||
'SELECT /* public.users */ 1',
|
||||
'SELECT 1.5e3, .5, 0x1F',
|
||||
'INSERT INTO kv VALUES ($1, $2) ON CONFLICT (k) DO NOTHING',
|
||||
])('lets through %s', (sql) => {
|
||||
expect(() => check(sql)).not.toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['SELECT * FROM public.users', /another schema/],
|
||||
['SELECT * FROM "public"."users"', /another schema/],
|
||||
['SELECT * FROM PUBLIC . users', /another schema/],
|
||||
['SELECT * FROM public/**/.users', /another schema/],
|
||||
['SELECT * FROM auth.secret', /another schema/],
|
||||
['SELECT * FROM trek_plugin_other.notes', /another schema/],
|
||||
['SELECT * FROM trek_plugin_not_yet.notes', /another schema/],
|
||||
['SELECT * FROM trek.public.users', /another schema/],
|
||||
['SELECT * FROM pg_stats', /"pg_stats" may not be named/],
|
||||
['SELECT * FROM "pg_catalog".pg_authid', /"pg_catalog" may not be named/],
|
||||
['SELECT * FROM information_schema.tables', /"information_schema" may not be named/],
|
||||
["SELECT query_to_xml('select 1', true, true, '')", /"query_to_xml" may not be named/],
|
||||
["SELECT database_to_xml(true, true, '')", /"database_to_xml" may not be named/],
|
||||
["SELECT ts_stat('select 1')", /"ts_stat" may not be named/],
|
||||
["SELECT \"set_config\"('search_path', 'public', true)", /"set_config" may not be named/],
|
||||
["SELECT current_setting('search_path')", /"current_setting" may not be named/],
|
||||
["SELECT nextval('users_id_seq')", /"nextval" may not be named/],
|
||||
["SELECT 'users'::regclass", /"regclass" may not be named/],
|
||||
["SELECT to_regclass('users')", /"to_regclass" may not be named/],
|
||||
['SELECT lo_get(1)', /"lo_get" may not be named/],
|
||||
["SELECT dblink('x', 'y')", /"dblink" may not be named/],
|
||||
['WITH RECURSIVE r(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM r) SELECT * FROM r', /RECURSIVE/],
|
||||
['SELECT 1 AS x INTO copy_of', /SELECT \.\.\. INTO/],
|
||||
['CREATE TEMP TABLE t (x int)', /CREATE TEMP is not allowed/],
|
||||
['CREATE TEMPORARY TABLE t (x int)', /CREATE TEMPORARY is not allowed/],
|
||||
["CREATE FUNCTION f() RETURNS int LANGUAGE sql AS 'select 1'", /CREATE FUNCTION is not allowed/],
|
||||
['CREATE OR REPLACE FUNCTION f() RETURNS int', /CREATE FUNCTION is not allowed/],
|
||||
['CREATE SCHEMA mine', /CREATE SCHEMA is not allowed/],
|
||||
['CREATE EXTENSION dblink', /CREATE EXTENSION is not allowed/],
|
||||
['DROP SCHEMA trek_plugin_notes', /DROP SCHEMA is not allowed/],
|
||||
['DROP OWNED BY current_user', /DROP OWNED is not allowed/],
|
||||
['ALTER TABLE notes SET SCHEMA x', /may not change the schema/],
|
||||
['ALTER TABLE notes OWNER TO someone', /may not change the owner/],
|
||||
['ALTER ROLE trek SUPERUSER', /ALTER ROLE is not allowed/],
|
||||
['CREATE TABLE t (x int) TABLESPACE pg_default', /tablespace/],
|
||||
['SET search_path TO public', /may not start with SET/],
|
||||
['RESET ROLE', /may not start with RESET/],
|
||||
['COPY notes TO STDOUT', /may not start with COPY/],
|
||||
['GRANT SELECT ON notes TO PUBLIC', /may not start with GRANT/],
|
||||
['DO $$ BEGIN END $$', /dollar-quoted/],
|
||||
['LISTEN x', /may not start with LISTEN/],
|
||||
['PREPARE p AS SELECT 1', /may not start with PREPARE/],
|
||||
['EXPLAIN ANALYZE SELECT 1', /may not start with EXPLAIN/],
|
||||
['BEGIN', /transaction-control/],
|
||||
['COMMIT', /transaction-control/],
|
||||
['SAVEPOINT a', /transaction-control/],
|
||||
["SELECT E'\\x'", /escape strings/],
|
||||
["SELECT U&'\\0041'", /Unicode escapes/],
|
||||
['SELECT * FROM U&"\\0070ublic".users', /Unicode escapes/],
|
||||
['SELECT $tag$x$tag$', /dollar-quoted/],
|
||||
['SELECT 1.public', /a number may not run straight into a name/],
|
||||
["SELECT 'open", /a string is not closed/],
|
||||
['SELECT "open', /a quoted name is not closed/],
|
||||
['SELECT /* open', /a comment is not closed/],
|
||||
['SELECT ""', /a quoted name may not be empty/],
|
||||
['SELECT `x`', /unexpected character/],
|
||||
['SELECT 1; SELECT * FROM public.users', /another schema/],
|
||||
])('refuses %s', (sql, reason) => {
|
||||
expect(() => check(sql)).toThrow(PluginSqlRefused);
|
||||
expect(() => check(sql)).toThrow(reason);
|
||||
});
|
||||
|
||||
it('reads nested block comments as Postgres does, so code after the inner close is still a comment', () => {
|
||||
expect(() => check('SELECT 1 /* outer /* inner */ public.users */')).not.toThrow();
|
||||
expect(() => check('SELECT 1 /* one */ FROM public.users')).toThrow(/another schema/);
|
||||
});
|
||||
|
||||
it('compares a long name the way Postgres keeps it: cut to 63 bytes', () => {
|
||||
const own = `trek_plugin_${'a'.repeat(51)}`;
|
||||
expect(own).toHaveLength(63);
|
||||
const statement = parsePluginSql(`SELECT * FROM ${own}zzzz.notes`)[0];
|
||||
expect(() => checkQualifiers(statement, own, new Set([own]))).not.toThrow();
|
||||
const other = `public${'x'.repeat(70)}`;
|
||||
const cut = other.slice(0, 63);
|
||||
expect(() => checkQualifiers(parsePluginSql(`SELECT * FROM ${other}.t`)[0], own, new Set([cut]))).toThrow(
|
||||
/another schema/,
|
||||
);
|
||||
});
|
||||
|
||||
it('splits a script at its semicolons, keeps each statement text and drops empty ones', () => {
|
||||
const statements = parsePluginSql("CREATE TABLE a (x text);;\n-- note\nINSERT INTO a VALUES ('a;b');");
|
||||
expect(statements.map((s) => s.text)).toEqual(['CREATE TABLE a (x text)', "INSERT INTO a VALUES ('a;b')"]);
|
||||
expect(parsePluginSql('-- nothing\n;')).toEqual([]);
|
||||
});
|
||||
|
||||
it('says whether a statement returns rows, may write, qualifies a name and how many placeholders it has', () => {
|
||||
const [select] = parsePluginSql('SELECT * FROM notes WHERE id = $1 AND tag = $2');
|
||||
expect(select).toMatchObject({ returnsRows: true, mayWrite: false, qualifies: false, params: 2 });
|
||||
const [returning] = parsePluginSql('UPDATE notes SET body = $1 RETURNING id');
|
||||
expect(returning).toMatchObject({ returnsRows: true, mayWrite: true });
|
||||
const [insert] = parsePluginSql('INSERT INTO notes (body) VALUES ($1)');
|
||||
expect(insert).toMatchObject({ returnsRows: false, mayWrite: true });
|
||||
const [cte] = parsePluginSql('WITH gone AS (DELETE FROM notes RETURNING id) SELECT count(*) FROM gone');
|
||||
expect(cte).toMatchObject({ returnsRows: true, mayWrite: true });
|
||||
const [ddl] = parsePluginSql('CREATE TABLE t (x int)');
|
||||
expect(ddl).toMatchObject({ returnsRows: false, mayWrite: true });
|
||||
const [qualified] = parsePluginSql('SELECT n.body FROM notes n');
|
||||
expect(qualified.qualifies).toBe(true);
|
||||
});
|
||||
|
||||
it('turns a SQLite ? placeholder into a message about $1 when args are bound', () => {
|
||||
const [sqliteStyle] = parsePluginSql('SELECT * FROM notes WHERE id = ?');
|
||||
expect(() => checkPlaceholders(sqliteStyle, [1])).toThrow(/\$1, \$2/);
|
||||
expect(() => checkPlaceholders(sqliteStyle, [])).not.toThrow();
|
||||
const [pgStyle] = parsePluginSql('SELECT * FROM notes WHERE id = $1');
|
||||
expect(() => checkPlaceholders(pgStyle, [1])).not.toThrow();
|
||||
});
|
||||
|
||||
it('folds unquoted names to lower case and keeps quoted ones as written', () => {
|
||||
expect(tokenize('Select "MiXed", Plain')).toEqual([
|
||||
{ kind: 'word', value: 'select', quoted: false, at: 0 },
|
||||
{ kind: 'word', value: 'MiXed', quoted: true, at: 7 },
|
||||
{ kind: 'punct', value: ',', at: 14 },
|
||||
{ kind: 'word', value: 'plain', quoted: false, at: 16 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user