fix(db): store no NUL character on either engine

Postgres text cannot hold U+0000, so any write or filter carrying one (a
request body, a query string, an import, a feed, a forwarded mail, a
plugin's own SQL) answered a 500 on Postgres while SQLite stored it. Both
engines now drop the character where every value leaves for the database
(src/db/nul-free.ts): the MikroORM platforms when they inline a string or
serialize a JSON value, the Kysely dialects for every parameter, a
plugin's own SQLite file for its arguments, and the move from SQLite,
which used to refuse such text. The same request now answers the same
status and leaves the same row on both engines, so the places NUL suite
runs on Postgres as well and leaves SQLITE_ONLY.
This commit is contained in:
Maurice
2026-11-17 21:24:00 +01:00
parent 685506d067
commit 5ff5e76603
12 changed files with 374 additions and 157 deletions
@@ -1,6 +1,7 @@
import { nulFreeJson, stripNul, withNulFreeParameters } from '../../nul-free';
import { sendMarkedQueriesExtended } from './extended-query';
import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order';
import { type Configuration } from '@mikro-orm/core';
import { type Configuration, type TransformContext } from '@mikro-orm/core';
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
/** The `pg` pool event a new client is handed to before anyone uses it. */
@@ -59,7 +60,8 @@ class NumberTypesConnection extends PostgreSqlConnection {
/**
* The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL
* position (`null-order.ts`), on a pool whose every client sends the queries
* position (`null-order.ts`), sending every parameter without NUL
* (`nul-free.ts`), on a pool whose every client sends the queries
* `extendedQuery` builds over the extended protocol (`extended-query.ts`).
*/
override createKyselyDialect(overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0]) {
@@ -69,16 +71,31 @@ class NumberTypesConnection extends PostgreSqlConnection {
return given.onPoolCreated?.(pool);
};
const withHook = { ...given, onPoolCreated } as Parameters<PostgreSqlConnection['createKyselyDialect']>[0];
return withSqliteNullOrder(super.createKyselyDialect(withHook)) as ReturnType<
return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(withHook))) as ReturnType<
PostgreSqlConnection['createKyselyDialect']
>;
}
}
/**
* The Postgres platform TREK runs on: ORDER BY with SQLite's NULL position
* (`null-order.ts`), and no NUL in a string it inlines into a statement or a
* JSON value it serializes, which Postgres text cannot hold (`nul-free.ts`).
*/
export class TrekPostgreSqlPlatform extends SqliteNullOrderPlatform {
override escape(value: unknown): string {
return super.escape(typeof value === 'string' ? stripNul(value) : value);
}
override convertJsonToDatabaseValue(value: unknown, context?: TransformContext): unknown {
return super.convertJsonToDatabaseValue(nulFreeJson(value), context);
}
}
/**
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
* as numbers, and on a platform and a query compiler that sort NULLs where
* SQLite does (`null-order.ts`). The parsers belong to this pool only:
* as numbers, and on the platform above with a query compiler that sorts NULLs
* where SQLite does (`null-order.ts`). The parsers belong to this pool only:
* nothing global in `pg` changes.
*/
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
@@ -89,6 +106,6 @@ export class TrekPostgreSqlDriver extends PostgreSqlDriver {
// fields, which are not public, so they are replaced here, before anything
// connects and before the configuration reads the platform.
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
(this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform();
(this as unknown as { platform: TrekPostgreSqlPlatform }).platform = new TrekPostgreSqlPlatform();
}
}
@@ -1,3 +1,5 @@
import { nulFreeJson, stripNul } from '../../../nul-free';
import { createHash } from 'node:crypto';
/**
@@ -7,7 +9,9 @@ import { createHash } from 'node:crypto';
* SQLite stores a value by what it is, not by the column's declared type, so
* a column may hold text where the schema says integer. The move never
* guesses: a value the Postgres column cannot hold exactly is refused with
* the reason, and the move stops before it commits anything.
* the reason, and the move stops before it commits anything. The one
* exception is a NUL character in text or JSON, which TREK stores on neither
* engine (`db/nul-free.ts`): the move drops it, as every write does.
*
* The SQLite side is read with better-sqlite3's safe integers, so an integer
* arrives as a `bigint` and keeps all 64 bits.
@@ -29,7 +33,6 @@ const INT64_MAX = 2n ** 63n - 1n;
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
const DECIMAL_INTEGER = /^[+-]?\d+$/;
const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const NUL = '\u0000';
/**
* The kind of a Postgres column, from `information_schema.columns.data_type`.
@@ -100,11 +103,7 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
}
function toText(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'string') {
return value.includes(NUL)
? refuse('the text holds a NUL character, which Postgres text cannot store')
: { ok: true, value };
}
if (typeof value === 'string') return { ok: true, value: stripNul(value) };
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
return refuse(`${describe(value)} is not text`);
}
@@ -112,15 +111,14 @@ function toText(value: Exclude<SqliteValue, null>): Converted {
function toJson(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
let parsed: unknown;
try {
JSON.parse(value);
parsed = JSON.parse(value);
} catch {
return refuse(`${describe(value)} is not valid JSON`);
}
// jsonb refuses the escaped NUL character as well as the raw one.
if (value.includes(NUL) || value.includes('\\u0000'))
return refuse('the JSON holds a NUL character, which jsonb cannot store');
return { ok: true, value };
// Valid JSON holds a NUL only escaped, which jsonb refuses as well: the value is written again without it.
return { ok: true, value: value.includes('\\u0000') ? JSON.stringify(nulFreeJson(parsed)) : value };
}
function toBytes(value: Exclude<SqliteValue, null>): Converted {
+109
View File
@@ -0,0 +1,109 @@
import type { CompiledQuery, DatabaseConnection, Dialect, Driver } from 'kysely';
/**
* TREK stores no U+0000 (NUL) in text, on either engine.
*
* Postgres cannot hold the character in `text` at all: a statement that binds
* or spells one fails, so a NUL arriving from anywhere (a request body, a
* query string, an import, a provider's feed, a forwarded mail, a plugin)
* answered a 500 on Postgres where SQLite stored it. NUL is a control
* character, never part of a name, a note or an address, so instead of
* refusing the write both engines drop it from every string a statement
* carries, and an install holds the same text on either engine.
*
* It happens once, where every value leaves for the database, so no caller
* can forget it:
* - the MikroORM platforms (`nul-safe-sqlite-platform.ts`, and the Postgres
* platform in `engine/postgres/postgres-driver.ts`) render every string
* they inline into a statement through {@link stripNul}, and every JSON
* value they serialize through {@link nulFreeJson};
* - the Kysely dialects of both engines (`orm-driver.ts`,
* `engine/postgres/postgres-driver.ts`) send every query's parameters
* through {@link withNulFreeParameters}, which covers the Kysely
* statements of the repositories and a plugin's own SQL on Postgres;
* - a plugin's own SQLite file (`nest/plugins/host/plugin-data.service.ts`)
* binds its arguments through {@link nulFreeParameter};
* - the move from SQLite to Postgres (`engine/postgres/sqlite-move/values.ts`)
* copies text the same way.
*
* A filter is stripped as well as a write, so a lookup finds what the write
* stored. Text that spells the character as the six characters `\u0000`
* (JSON already encoded to a string) is left alone: it holds no NUL.
*/
const NUL = '\u0000';
/** `text` without its NUL characters. */
export function stripNul(text: string): string {
return text.includes(NUL) ? text.replaceAll(NUL, '') : text;
}
/** A bound parameter without NUL: a string, or each string of an array; any other value as it is. */
export function nulFreeParameter(value: unknown): unknown {
if (typeof value === 'string') return stripNul(value);
if (Array.isArray(value)) return value.some((item) => typeof item === 'string') ? value.map(nulFreeParameter) : value;
return value;
}
/** A value about to be serialized as JSON, with every string in it, object keys included, without NUL. */
export function nulFreeJson(value: unknown): unknown {
if (typeof value === 'string') return stripNul(value);
if (Array.isArray(value)) return value.map(nulFreeJson);
if (value !== null && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>).map(([key, item]) => [stripNul(key), nulFreeJson(item)]),
);
}
return value;
}
/** `query` with its parameters passed through {@link nulFreeParameter}; the same object when none holds a NUL. */
function nulFreeQuery(query: CompiledQuery): CompiledQuery {
const parameters = query.parameters.map(nulFreeParameter);
if (parameters.every((value, i) => value === query.parameters[i])) return query;
return { ...query, parameters };
}
/**
* `target` with `replaced` in place of some of its members. Every other member
* is the target's own, a method bound to the target, so a driver that reaches
* a connection's private state (Kysely's Postgres driver releases a
* connection through a private method) still finds it.
*/
function overriding<T extends object>(target: T, replaced: Partial<T>): T {
return new Proxy(target, {
get(object, key) {
if (Object.prototype.hasOwnProperty.call(replaced, key)) return replaced[key as keyof T];
const member: unknown = Reflect.get(object, key, object);
return typeof member === 'function' ? member.bind(object) : member;
},
});
}
/** `driver`, whose connections send every query's parameters without NUL. */
function nulFreeDriver(driver: Driver): Driver {
const wrapped = new WeakMap<DatabaseConnection, DatabaseConnection>();
const wrap = (connection: DatabaseConnection): DatabaseConnection => {
let proxy = wrapped.get(connection);
if (!proxy) {
proxy = overriding(connection, {
executeQuery: <R>(query: CompiledQuery) => connection.executeQuery<R>(nulFreeQuery(query)),
streamQuery: <R>(query: CompiledQuery, chunkSize?: number) =>
connection.streamQuery<R>(nulFreeQuery(query), chunkSize),
});
wrapped.set(connection, proxy);
}
return proxy;
};
return overriding(driver, { acquireConnection: async () => wrap(await driver.acquireConnection()) });
}
/** `dialect`, whose driver sends every query's parameters without NUL; the rest is its own. */
export function withNulFreeParameters(dialect: Dialect): Dialect {
return {
createAdapter: () => dialect.createAdapter(),
createDriver: () => nulFreeDriver(dialect.createDriver()),
createIntrospector: (db) => dialect.createIntrospector(db),
createQueryCompiler: () => dialect.createQueryCompiler(),
};
}
+18 -52
View File
@@ -1,3 +1,5 @@
import { nulFreeJson, stripNul } from './nul-free';
import type { TransformContext } from '@mikro-orm/core';
import { SqlitePlatform } from '@mikro-orm/sql';
/**
@@ -5,55 +7,18 @@ import { SqlitePlatform } from '@mikro-orm/sql';
* surfaced by Plan 3c Task 4 review M1: MikroORM 7 never sends the driver a
* parameterised statement. `AbstractSqlConnection.execute` calls
* `platform.formatQuery(sql, params)` (`@mikro-orm/core`'s `Platform.formatQuery`)
* and hands better-sqlite3 the fully INLINED SQL text — `formatQuery` walks the
* `?` placeholders and splices in `this.quoteValue(param)` for each one, so
* every bound value this ORM ever writes is a literal in the statement string,
* not a bound parameter the driver escapes separately. `SqlitePlatform`'s own
* `quoteValue` (`@mikro-orm/sql`'s `AbstractSqlPlatform.quoteValue`, for
* anything that isn't a `Date`) delegates to `escape`, which wraps a string in
* single quotes and doubles embedded `'` — the standard SQL literal shape.
* and hands better-sqlite3 the fully INLINED SQL text: `formatQuery` walks the
* `?` placeholders and splices in `this.quoteValue(param)` for each one, and
* `quoteValue` delegates to `escape` for anything that isn't a `Date`.
*
* A `\u0000` inside that string survives the JS-level quoting untouched (it
* is not `'`, so nothing escapes it), but the C string this literal eventually
* becomes IS NUL-terminated: SQLite's own tokenizer reads the statement text
* as a NUL-terminated buffer, so it stops scanning at the embedded `\0` —
* inside the still-open `'…'` literal, which is exactly why the failure mode
* is always "unterminated string", never anything an attacker could turn into
* an injection (verified in the Task 4 review: every NUL statement in this
* class fails closed with a syntax error, 500, never a different query).
*
* The legacy raw-SQL layer never had this problem: `better-sqlite3` binds `?`
* parameters as real, separately-transmitted values (no inlining, no NUL
* truncation), so a NUL byte in a user string round-tripped byte-for-byte —
* `GET /api/trips/:id/places?search=%00` was 200 and a place named
* `"nul\u0000name"` created (201) with its name intact. Once a route's
* statement moves off `DatabaseService`'s raw bind onto anything this ORM
* inlines (a QB read, `em.insert`, `nativeUpdate` — every one of them,
* regardless of domain), that same input 500s instead, byte-for-byte across
* every domain: this is corollary to rule 22, not specific to any one plan.
*
* The fix restores the round-trip AT THE PLATFORM, once, program-wide: a
* string containing `\u0000` renders not as one `'…'` literal but as a
* parenthesised `||` (SQL string concatenation) chain, splicing in SQLite's
* builtin `char(0)` — which returns a one-character string holding the NUL
* codepoint — everywhere the original string held one. `char()` is a core
* SQLite function (no extension, no `PRAGMA`), and `||` binds tighter than
* every arithmetic/comparison/logical operator that could otherwise appear
* around a bound value, so `(...)`'s parens are redundant for a scalar
* comparison — kept anyway so this expression composes safely as an
* IN-list element or the operand of anything else `escape()`'s caller
* splices it into (`AbstractSqlPlatform.escape` recurses through arrays
* with `value.map(v => this.escape(v)).join(', ')`, so a NUL inside one
* array element must stay a self-contained expression). Splitting on
* `\u0000` and re-quoting each piece with the platform's own `escape` (via
* `super.escape`) reuses the exact same `'`-doubling every non-NUL string
* already goes through, so a NUL next to a quote or a backslash (neither
* of which SQLite string literals treat specially) round-trips exactly as
* it did before this override existed.
*
* Deliberately scoped to `typeof value === 'string'` for the NUL half above:
* that IS the M1 fix, not a general re-implementation of `escape`/
* `quoteValue`.
* **NUL.** A `\u0000` inside a string survives the JS-level quoting, but SQLite
* reads the statement text as a NUL-terminated buffer and stops at it, inside
* the still-open `'…'` literal, so every such statement failed with
* "unterminated string" (a 500, never a different query). This platform once
* spliced SQLite's `char(0)` into the literal to keep the character; TREK now
* stores no NUL on either engine (`nul-free.ts`: Postgres text cannot hold
* one), so `escape` drops it from every string, and `convertJsonToDatabaseValue`
* from every string of a JSON value, the way the Postgres platform does.
*
* **Rule 22 extension (Plan 3c Task 9, close-out review A H1):** non-finite
* numbers hit the SAME inlining mechanism, and rule 21's `toRowId` guards do
@@ -96,14 +61,15 @@ import { SqlitePlatform } from '@mikro-orm/sql';
*/
export class NulSafeSqlitePlatform extends SqlitePlatform {
override escape(value: unknown): string {
if (typeof value === 'string' && value.includes('\u0000')) {
const literals = value.split('\u0000').map((part) => super.escape(part));
return `(${literals.join(' || char(0) || ')})`;
}
if (typeof value === 'string') return super.escape(stripNul(value));
if (typeof value === 'number' && !Number.isFinite(value)) {
if (Number.isNaN(value)) return 'NULL';
return value > 0 ? '9e999' : '-9e999';
}
return super.escape(value);
}
override convertJsonToDatabaseValue(value: unknown, context?: TransformContext): unknown {
return super.convertJsonToDatabaseValue(nulFreeJson(value), context);
}
}
+4 -2
View File
@@ -1,4 +1,5 @@
import { getRawConnection } from './database';
import { withNulFreeParameters } from './nul-free';
import { NulSafeSqlitePlatform } from './nul-safe-sqlite-platform';
import type { Configuration } from '@mikro-orm/core';
import type { SqlitePlatform } from '@mikro-orm/sql';
@@ -57,13 +58,14 @@ export function createBoundSqliteDriver(getHandle: () => Database.Database): typ
(this as unknown as { database: Database.Database }).database = handle;
// Delegate the compiler/adapter/introspector to a throwaway stock dialect
// (stateless factories); only `createDriver()` needs the non-closing swap.
// Every parameter goes out without NUL, as on Postgres (`nul-free.ts`).
const stock = new SqliteDialect(config);
return {
return withNulFreeParameters({
createDriver: () => new NonClosingSqliteDriver(config),
createQueryCompiler: () => stock.createQueryCompiler(),
createAdapter: () => stock.createAdapter(),
createIntrospector: (db) => stock.createIntrospector(db),
};
});
}
}
@@ -1,5 +1,6 @@
import { openDatabase } from '../../../db/connection';
import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data';
import { nulFreeParameter } from '../../../db/nul-free';
import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths';
import { splitAfterSemicolons } from './sql-script';
@@ -10,6 +11,19 @@ import path from 'node:path';
// The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs).
export type { PluginDataStore };
/**
* A plugin's bound arguments without NUL, as TREK binds its own and as a
* plugin's schema on Postgres receives them (`db/nul-free.ts`): positional
* values, or the values of an object of named ones.
*/
function nulFreeArgs(args: readonly unknown[]): never[] {
return args.map((arg) =>
arg !== null && typeof arg === 'object' && Object.getPrototypeOf(arg) === Object.prototype
? Object.fromEntries(Object.entries(arg).map(([name, value]) => [name, nulFreeParameter(value)]))
: nulFreeParameter(arg),
) as never[];
}
/**
* A plugin's own sqlite database (#plugins, db:own): the SQLite side of the
* `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a
@@ -142,7 +156,7 @@ export class PluginDataDb implements PluginDataStore {
// clock is read between rows. It is read only once another row has come back, so
// a result that has been read to its end is returned, however long that took.
const rows: unknown[] = [];
for (const row of this.db.prepare(sql).iterate(...(args as never[]))) {
for (const row of this.db.prepare(sql).iterate(...nulFreeArgs(args))) {
if (rows.length > 0) checkTime();
rows.push(row);
if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`);
@@ -158,7 +172,7 @@ export class PluginDataDb implements PluginDataStore {
exec(sql: string, args: unknown[] = []): { changes: number } {
this.guard(sql);
if (args.length > 0) {
const info = this.db.prepare(sql).run(...(args as never[]));
const info = this.db.prepare(sql).run(...nulFreeArgs(args));
return { changes: info.changes };
}
this.runScript(sql, this.startBudget('exec'));
@@ -242,7 +256,7 @@ export class PluginDataDb implements PluginDataStore {
const results: Array<{ changes?: number; rows?: unknown[] }> = [];
for (const op of batch) {
const stmt = this.db.prepare(op.sql);
const args = (op.args ?? []) as never[];
const args = nulFreeArgs(op.args ?? []);
if (stmt.reader) {
const rows: unknown[] = [];
for (const row of stmt.iterate(...args)) {