diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index a646daa94..2c8029cd8 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,6 +1,7 @@ +import { nulFreeJson, stripNul, withNulFreeParameters } from '../../nul-free'; import { sendMarkedQueriesExtended } from './extended-query'; import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; -import { type Configuration } from '@mikro-orm/core'; +import { type Configuration, type TransformContext } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; /** The `pg` pool event a new client is handed to before anyone uses it. */ @@ -59,7 +60,8 @@ class NumberTypesConnection extends PostgreSqlConnection { /** * The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL - * position (`null-order.ts`), on a pool whose every client sends the queries + * position (`null-order.ts`), sending every parameter without NUL + * (`nul-free.ts`), on a pool whose every client sends the queries * `extendedQuery` builds over the extended protocol (`extended-query.ts`). */ override createKyselyDialect(overrides: Parameters[0]) { @@ -69,16 +71,31 @@ class NumberTypesConnection extends PostgreSqlConnection { return given.onPoolCreated?.(pool); }; const withHook = { ...given, onPoolCreated } as Parameters[0]; - return withSqliteNullOrder(super.createKyselyDialect(withHook)) as ReturnType< + return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(withHook))) as ReturnType< PostgreSqlConnection['createKyselyDialect'] >; } } +/** + * The Postgres platform TREK runs on: ORDER BY with SQLite's NULL position + * (`null-order.ts`), and no NUL in a string it inlines into a statement or a + * JSON value it serializes, which Postgres text cannot hold (`nul-free.ts`). + */ +export class TrekPostgreSqlPlatform extends SqliteNullOrderPlatform { + override escape(value: unknown): string { + return super.escape(typeof value === 'string' ? stripNul(value) : value); + } + + override convertJsonToDatabaseValue(value: unknown, context?: TransformContext): unknown { + return super.convertJsonToDatabaseValue(nulFreeJson(value), context); + } +} + /** * MikroORM's Postgres driver on a connection that reads `bigint` and `numeric` - * as numbers, and on a platform and a query compiler that sort NULLs where - * SQLite does (`null-order.ts`). The parsers belong to this pool only: + * as numbers, and on the platform above with a query compiler that sorts NULLs + * where SQLite does (`null-order.ts`). The parsers belong to this pool only: * nothing global in `pg` changes. */ export class TrekPostgreSqlDriver extends PostgreSqlDriver { @@ -89,6 +106,6 @@ export class TrekPostgreSqlDriver extends PostgreSqlDriver { // fields, which are not public, so they are replaced here, before anything // connects and before the configuration reads the platform. (this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config); - (this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform(); + (this as unknown as { platform: TrekPostgreSqlPlatform }).platform = new TrekPostgreSqlPlatform(); } } diff --git a/server/src/db/engine/postgres/sqlite-move/values.ts b/server/src/db/engine/postgres/sqlite-move/values.ts index 779ed0fe0..b7cd41152 100644 --- a/server/src/db/engine/postgres/sqlite-move/values.ts +++ b/server/src/db/engine/postgres/sqlite-move/values.ts @@ -1,3 +1,5 @@ +import { nulFreeJson, stripNul } from '../../../nul-free'; + import { createHash } from 'node:crypto'; /** @@ -7,7 +9,9 @@ import { createHash } from 'node:crypto'; * SQLite stores a value by what it is, not by the column's declared type, so * a column may hold text where the schema says integer. The move never * guesses: a value the Postgres column cannot hold exactly is refused with - * the reason, and the move stops before it commits anything. + * the reason, and the move stops before it commits anything. The one + * exception is a NUL character in text or JSON, which TREK stores on neither + * engine (`db/nul-free.ts`): the move drops it, as every write does. * * The SQLite side is read with better-sqlite3's safe integers, so an integer * arrives as a `bigint` and keeps all 64 bits. @@ -29,7 +33,6 @@ const INT64_MAX = 2n ** 63n - 1n; const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER); const DECIMAL_INTEGER = /^[+-]?\d+$/; const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i; -const NUL = '\u0000'; /** * The kind of a Postgres column, from `information_schema.columns.data_type`. @@ -100,11 +103,7 @@ function toFloat(value: Exclude): Converted { } function toText(value: Exclude): Converted { - if (typeof value === 'string') { - return value.includes(NUL) - ? refuse('the text holds a NUL character, which Postgres text cannot store') - : { ok: true, value }; - } + if (typeof value === 'string') return { ok: true, value: stripNul(value) }; if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; return refuse(`${describe(value)} is not text`); } @@ -112,15 +111,14 @@ function toText(value: Exclude): Converted { function toJson(value: Exclude): Converted { if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`); + let parsed: unknown; try { - JSON.parse(value); + parsed = JSON.parse(value); } catch { return refuse(`${describe(value)} is not valid JSON`); } - // jsonb refuses the escaped NUL character as well as the raw one. - if (value.includes(NUL) || value.includes('\\u0000')) - return refuse('the JSON holds a NUL character, which jsonb cannot store'); - return { ok: true, value }; + // Valid JSON holds a NUL only escaped, which jsonb refuses as well: the value is written again without it. + return { ok: true, value: value.includes('\\u0000') ? JSON.stringify(nulFreeJson(parsed)) : value }; } function toBytes(value: Exclude): Converted { diff --git a/server/src/db/nul-free.ts b/server/src/db/nul-free.ts new file mode 100644 index 000000000..8f7384866 --- /dev/null +++ b/server/src/db/nul-free.ts @@ -0,0 +1,109 @@ +import type { CompiledQuery, DatabaseConnection, Dialect, Driver } from 'kysely'; + +/** + * TREK stores no U+0000 (NUL) in text, on either engine. + * + * Postgres cannot hold the character in `text` at all: a statement that binds + * or spells one fails, so a NUL arriving from anywhere (a request body, a + * query string, an import, a provider's feed, a forwarded mail, a plugin) + * answered a 500 on Postgres where SQLite stored it. NUL is a control + * character, never part of a name, a note or an address, so instead of + * refusing the write both engines drop it from every string a statement + * carries, and an install holds the same text on either engine. + * + * It happens once, where every value leaves for the database, so no caller + * can forget it: + * - the MikroORM platforms (`nul-safe-sqlite-platform.ts`, and the Postgres + * platform in `engine/postgres/postgres-driver.ts`) render every string + * they inline into a statement through {@link stripNul}, and every JSON + * value they serialize through {@link nulFreeJson}; + * - the Kysely dialects of both engines (`orm-driver.ts`, + * `engine/postgres/postgres-driver.ts`) send every query's parameters + * through {@link withNulFreeParameters}, which covers the Kysely + * statements of the repositories and a plugin's own SQL on Postgres; + * - a plugin's own SQLite file (`nest/plugins/host/plugin-data.service.ts`) + * binds its arguments through {@link nulFreeParameter}; + * - the move from SQLite to Postgres (`engine/postgres/sqlite-move/values.ts`) + * copies text the same way. + * + * A filter is stripped as well as a write, so a lookup finds what the write + * stored. Text that spells the character as the six characters `\u0000` + * (JSON already encoded to a string) is left alone: it holds no NUL. + */ + +const NUL = '\u0000'; + +/** `text` without its NUL characters. */ +export function stripNul(text: string): string { + return text.includes(NUL) ? text.replaceAll(NUL, '') : text; +} + +/** A bound parameter without NUL: a string, or each string of an array; any other value as it is. */ +export function nulFreeParameter(value: unknown): unknown { + if (typeof value === 'string') return stripNul(value); + if (Array.isArray(value)) return value.some((item) => typeof item === 'string') ? value.map(nulFreeParameter) : value; + return value; +} + +/** A value about to be serialized as JSON, with every string in it, object keys included, without NUL. */ +export function nulFreeJson(value: unknown): unknown { + if (typeof value === 'string') return stripNul(value); + if (Array.isArray(value)) return value.map(nulFreeJson); + if (value !== null && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype) { + return Object.fromEntries( + Object.entries(value as Record).map(([key, item]) => [stripNul(key), nulFreeJson(item)]), + ); + } + return value; +} + +/** `query` with its parameters passed through {@link nulFreeParameter}; the same object when none holds a NUL. */ +function nulFreeQuery(query: CompiledQuery): CompiledQuery { + const parameters = query.parameters.map(nulFreeParameter); + if (parameters.every((value, i) => value === query.parameters[i])) return query; + return { ...query, parameters }; +} + +/** + * `target` with `replaced` in place of some of its members. Every other member + * is the target's own, a method bound to the target, so a driver that reaches + * a connection's private state (Kysely's Postgres driver releases a + * connection through a private method) still finds it. + */ +function overriding(target: T, replaced: Partial): T { + return new Proxy(target, { + get(object, key) { + if (Object.prototype.hasOwnProperty.call(replaced, key)) return replaced[key as keyof T]; + const member: unknown = Reflect.get(object, key, object); + return typeof member === 'function' ? member.bind(object) : member; + }, + }); +} + +/** `driver`, whose connections send every query's parameters without NUL. */ +function nulFreeDriver(driver: Driver): Driver { + const wrapped = new WeakMap(); + const wrap = (connection: DatabaseConnection): DatabaseConnection => { + let proxy = wrapped.get(connection); + if (!proxy) { + proxy = overriding(connection, { + executeQuery: (query: CompiledQuery) => connection.executeQuery(nulFreeQuery(query)), + streamQuery: (query: CompiledQuery, chunkSize?: number) => + connection.streamQuery(nulFreeQuery(query), chunkSize), + }); + wrapped.set(connection, proxy); + } + return proxy; + }; + return overriding(driver, { acquireConnection: async () => wrap(await driver.acquireConnection()) }); +} + +/** `dialect`, whose driver sends every query's parameters without NUL; the rest is its own. */ +export function withNulFreeParameters(dialect: Dialect): Dialect { + return { + createAdapter: () => dialect.createAdapter(), + createDriver: () => nulFreeDriver(dialect.createDriver()), + createIntrospector: (db) => dialect.createIntrospector(db), + createQueryCompiler: () => dialect.createQueryCompiler(), + }; +} diff --git a/server/src/db/nul-safe-sqlite-platform.ts b/server/src/db/nul-safe-sqlite-platform.ts index f7398ad02..732714eca 100644 --- a/server/src/db/nul-safe-sqlite-platform.ts +++ b/server/src/db/nul-safe-sqlite-platform.ts @@ -1,3 +1,5 @@ +import { nulFreeJson, stripNul } from './nul-free'; +import type { TransformContext } from '@mikro-orm/core'; import { SqlitePlatform } from '@mikro-orm/sql'; /** @@ -5,55 +7,18 @@ import { SqlitePlatform } from '@mikro-orm/sql'; * surfaced by Plan 3c Task 4 review M1: MikroORM 7 never sends the driver a * parameterised statement. `AbstractSqlConnection.execute` calls * `platform.formatQuery(sql, params)` (`@mikro-orm/core`'s `Platform.formatQuery`) - * and hands better-sqlite3 the fully INLINED SQL text — `formatQuery` walks the - * `?` placeholders and splices in `this.quoteValue(param)` for each one, so - * every bound value this ORM ever writes is a literal in the statement string, - * not a bound parameter the driver escapes separately. `SqlitePlatform`'s own - * `quoteValue` (`@mikro-orm/sql`'s `AbstractSqlPlatform.quoteValue`, for - * anything that isn't a `Date`) delegates to `escape`, which wraps a string in - * single quotes and doubles embedded `'` — the standard SQL literal shape. + * and hands better-sqlite3 the fully INLINED SQL text: `formatQuery` walks the + * `?` placeholders and splices in `this.quoteValue(param)` for each one, and + * `quoteValue` delegates to `escape` for anything that isn't a `Date`. * - * A `\u0000` inside that string survives the JS-level quoting untouched (it - * is not `'`, so nothing escapes it), but the C string this literal eventually - * becomes IS NUL-terminated: SQLite's own tokenizer reads the statement text - * as a NUL-terminated buffer, so it stops scanning at the embedded `\0` — - * inside the still-open `'…'` literal, which is exactly why the failure mode - * is always "unterminated string", never anything an attacker could turn into - * an injection (verified in the Task 4 review: every NUL statement in this - * class fails closed with a syntax error, 500, never a different query). - * - * The legacy raw-SQL layer never had this problem: `better-sqlite3` binds `?` - * parameters as real, separately-transmitted values (no inlining, no NUL - * truncation), so a NUL byte in a user string round-tripped byte-for-byte — - * `GET /api/trips/:id/places?search=%00` was 200 and a place named - * `"nul\u0000name"` created (201) with its name intact. Once a route's - * statement moves off `DatabaseService`'s raw bind onto anything this ORM - * inlines (a QB read, `em.insert`, `nativeUpdate` — every one of them, - * regardless of domain), that same input 500s instead, byte-for-byte across - * every domain: this is corollary to rule 22, not specific to any one plan. - * - * The fix restores the round-trip AT THE PLATFORM, once, program-wide: a - * string containing `\u0000` renders not as one `'…'` literal but as a - * parenthesised `||` (SQL string concatenation) chain, splicing in SQLite's - * builtin `char(0)` — which returns a one-character string holding the NUL - * codepoint — everywhere the original string held one. `char()` is a core - * SQLite function (no extension, no `PRAGMA`), and `||` binds tighter than - * every arithmetic/comparison/logical operator that could otherwise appear - * around a bound value, so `(...)`'s parens are redundant for a scalar - * comparison — kept anyway so this expression composes safely as an - * IN-list element or the operand of anything else `escape()`'s caller - * splices it into (`AbstractSqlPlatform.escape` recurses through arrays - * with `value.map(v => this.escape(v)).join(', ')`, so a NUL inside one - * array element must stay a self-contained expression). Splitting on - * `\u0000` and re-quoting each piece with the platform's own `escape` (via - * `super.escape`) reuses the exact same `'`-doubling every non-NUL string - * already goes through, so a NUL next to a quote or a backslash (neither - * of which SQLite string literals treat specially) round-trips exactly as - * it did before this override existed. - * - * Deliberately scoped to `typeof value === 'string'` for the NUL half above: - * that IS the M1 fix, not a general re-implementation of `escape`/ - * `quoteValue`. + * **NUL.** A `\u0000` inside a string survives the JS-level quoting, but SQLite + * reads the statement text as a NUL-terminated buffer and stops at it, inside + * the still-open `'…'` literal, so every such statement failed with + * "unterminated string" (a 500, never a different query). This platform once + * spliced SQLite's `char(0)` into the literal to keep the character; TREK now + * stores no NUL on either engine (`nul-free.ts`: Postgres text cannot hold + * one), so `escape` drops it from every string, and `convertJsonToDatabaseValue` + * from every string of a JSON value, the way the Postgres platform does. * * **Rule 22 extension (Plan 3c Task 9, close-out review A H1):** non-finite * numbers hit the SAME inlining mechanism, and rule 21's `toRowId` guards do @@ -96,14 +61,15 @@ import { SqlitePlatform } from '@mikro-orm/sql'; */ export class NulSafeSqlitePlatform extends SqlitePlatform { override escape(value: unknown): string { - if (typeof value === 'string' && value.includes('\u0000')) { - const literals = value.split('\u0000').map((part) => super.escape(part)); - return `(${literals.join(' || char(0) || ')})`; - } + if (typeof value === 'string') return super.escape(stripNul(value)); if (typeof value === 'number' && !Number.isFinite(value)) { if (Number.isNaN(value)) return 'NULL'; return value > 0 ? '9e999' : '-9e999'; } return super.escape(value); } + + override convertJsonToDatabaseValue(value: unknown, context?: TransformContext): unknown { + return super.convertJsonToDatabaseValue(nulFreeJson(value), context); + } } diff --git a/server/src/db/orm-driver.ts b/server/src/db/orm-driver.ts index 02fc6743f..9abcae9f0 100644 --- a/server/src/db/orm-driver.ts +++ b/server/src/db/orm-driver.ts @@ -1,4 +1,5 @@ import { getRawConnection } from './database'; +import { withNulFreeParameters } from './nul-free'; import { NulSafeSqlitePlatform } from './nul-safe-sqlite-platform'; import type { Configuration } from '@mikro-orm/core'; import type { SqlitePlatform } from '@mikro-orm/sql'; @@ -57,13 +58,14 @@ export function createBoundSqliteDriver(getHandle: () => Database.Database): typ (this as unknown as { database: Database.Database }).database = handle; // Delegate the compiler/adapter/introspector to a throwaway stock dialect // (stateless factories); only `createDriver()` needs the non-closing swap. + // Every parameter goes out without NUL, as on Postgres (`nul-free.ts`). const stock = new SqliteDialect(config); - return { + return withNulFreeParameters({ createDriver: () => new NonClosingSqliteDriver(config), createQueryCompiler: () => stock.createQueryCompiler(), createAdapter: () => stock.createAdapter(), createIntrospector: (db) => stock.createIntrospector(db), - }; + }); } } diff --git a/server/src/nest/plugins/host/plugin-data.service.ts b/server/src/nest/plugins/host/plugin-data.service.ts index 5a75b1599..4a9869c2b 100644 --- a/server/src/nest/plugins/host/plugin-data.service.ts +++ b/server/src/nest/plugins/host/plugin-data.service.ts @@ -1,5 +1,6 @@ import { openDatabase } from '../../../db/connection'; import { PLUGIN_DATA_CAPS, type PluginDataStore } from '../../../db/engine/plugin-data'; +import { nulFreeParameter } from '../../../db/nul-free'; import { pluginDataDir, pluginDbFile, pluginsDataRoot } from '../paths'; import { splitAfterSemicolons } from './sql-script'; @@ -10,6 +11,19 @@ import path from 'node:path'; // The port, re-exported for the RPC kit, which may import from this file only (eslint.config.mjs). export type { PluginDataStore }; +/** + * A plugin's bound arguments without NUL, as TREK binds its own and as a + * plugin's schema on Postgres receives them (`db/nul-free.ts`): positional + * values, or the values of an object of named ones. + */ +function nulFreeArgs(args: readonly unknown[]): never[] { + return args.map((arg) => + arg !== null && typeof arg === 'object' && Object.getPrototypeOf(arg) === Object.prototype + ? Object.fromEntries(Object.entries(arg).map(([name, value]) => [name, nulFreeParameter(value)])) + : nulFreeParameter(arg), + ) as never[]; +} + /** * A plugin's own sqlite database (#plugins, db:own): the SQLite side of the * `PluginDataStore` port (`db/engine/plugin-data.ts`; the Postgres side is a @@ -142,7 +156,7 @@ export class PluginDataDb implements PluginDataStore { // clock is read between rows. It is read only once another row has come back, so // a result that has been read to its end is returned, however long that took. const rows: unknown[] = []; - for (const row of this.db.prepare(sql).iterate(...(args as never[]))) { + for (const row of this.db.prepare(sql).iterate(...nulFreeArgs(args))) { if (rows.length > 0) checkTime(); rows.push(row); if (rows.length > maxRows) throw new Error(`query returned more than ${maxRows} rows`); @@ -158,7 +172,7 @@ export class PluginDataDb implements PluginDataStore { exec(sql: string, args: unknown[] = []): { changes: number } { this.guard(sql); if (args.length > 0) { - const info = this.db.prepare(sql).run(...(args as never[])); + const info = this.db.prepare(sql).run(...nulFreeArgs(args)); return { changes: info.changes }; } this.runScript(sql, this.startBudget('exec')); @@ -242,7 +256,7 @@ export class PluginDataDb implements PluginDataStore { const results: Array<{ changes?: number; rows?: unknown[] }> = []; for (const op of batch) { const stmt = this.db.prepare(op.sql); - const args = (op.args ?? []) as never[]; + const args = nulFreeArgs(op.args ?? []); if (stmt.reader) { const rows: unknown[] = []; for (const row of stmt.iterate(...args)) { diff --git a/server/tests/integration/places-nul.test.ts b/server/tests/integration/places-nul.test.ts index 40b0242b7..ba72d6d03 100644 --- a/server/tests/integration/places-nul.test.ts +++ b/server/tests/integration/places-nul.test.ts @@ -1,13 +1,13 @@ /** - * M1 / program rule 22 (task-4-review.md): a NUL byte in a user string no - * longer 500s on SQLite. The platform override restores the legacy raw bind's - * byte-for-byte round-trip (base 94c6efbbc: 200/201; before that fix, every - * ORM-converted statement 500'd on NUL). + * A NUL character (U+0000) in a user string, on SQLite and on Postgres alike. * - * Moved out of places.test.ts so the rest of that suite runs on both engines: - * these two cases are about SQLite's value quoting, and Postgres text cannot - * hold U+0000 at all (tests/postgres/lane.ts, SQLITE_ONLY). The control - * characters 0x01-0x1F stay in places.test.ts, where they run on both. + * Postgres text cannot hold it, so a write or a filter carrying one used to + * answer a 500 there while SQLite stored it. TREK stores no NUL on either + * engine now (src/db/nul-free.ts): every string a statement carries loses it, + * so the same request answers the same status and leaves the same row on both + * engines (M1 / program rule 22 first made it a 201 on SQLite). The control + * characters 0x01-0x1F, which both engines store, are swept in + * places.test.ts. */ import { buildApp } from '../../src/bootstrap'; import { Places } from '../../src/db/entities/Places.entity'; @@ -19,6 +19,7 @@ import { makeUser } from '../helpers/factories/users'; import { resetDatabase } from '../helpers/reset-database'; import { resetRateLimits } from '../helpers/test-db'; import { MikroORM } from '@mikro-orm/core'; +import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { INestApplication } from '@nestjs/common'; import type { Application } from 'express'; @@ -44,34 +45,37 @@ afterAll(async () => { await nestApp.close(); }); -describe('M1 — NUL-safe value quoting on the SQLite platform (rule 22)', () => { - it("GET ?search=%00 is 200, not 500 (status parity — SQLite's LIKE pattern matcher itself iterates its RHS as NUL-terminated, an independent SQLite limitation this platform fix does not touch, so a NUL search matches everything on BOTH trees, not nothing)", async () => { +describe('a NUL character in a user string, on every engine', () => { + it('GET ?search=%00 is 200 and matches as an empty search', async () => { const { user } = await makeUser(orm); const trip = await makeTrip(orm, user.id); await makePlace(orm, trip.id, { name: 'Spot' }); - // A raw, pre-encoded `%00` in the URL — the review's exact literal - // request shape (`GET /api/trips/:id/places?search=%00`). `.query({...})` - // goes through superagent's own `qs` encoder, which is not guaranteed to - // round-trip a NUL character the same way; the literal query string is - // what Express/the route actually receives in production. + // A raw, pre-encoded `%00` in the URL: `.query({...})` goes through + // superagent's own encoder, which need not round-trip a NUL the same way. const res = await request(app).get(`/api/trips/${trip.id}/places?search=%00`).set('Cookie', authCookie(user.id)); expect(res.status).toBe(200); + expect(res.body.places.map((p: { name: string }) => p.name)).toEqual(['Spot']); }); - it('POST create with a NUL in the name is 201 and the name round-trips byte-for-byte', async () => { + it('POST create with a NUL in the name is 201 and stores the name without it', async () => { const { user } = await makeUser(orm); const trip = await makeTrip(orm, user.id); - const name = 'nul\u0000name'; const res = await request(app) .post(`/api/trips/${trip.id}/places`) .set('Cookie', authCookie(user.id)) - .send({ name }); + .send({ name: 'nul\u0000name' }); expect(res.status).toBe(201); - expect(res.body.place.name).toBe(name); + expect(res.body.place.name).toBe('nulname'); const stored = await findRow(orm, Places, { id: res.body.place.id }); - expect(stored?.name).toBe(name); + expect(stored?.name).toBe('nulname'); + }); + + it('a parameter of a Kysely statement loses its NUL too', async () => { + const kysely = (orm.em.getConnection() as AbstractSqlConnection).getClient(); + const row = await kysely.selectNoFrom((eb) => eb.val('a\u0000b').as('v')).executeTakeFirstOrThrow(); + expect(row.v).toBe('ab'); }); }); diff --git a/server/tests/integration/places.test.ts b/server/tests/integration/places.test.ts index d2c56f259..9dbf37b4b 100644 --- a/server/tests/integration/places.test.ts +++ b/server/tests/integration/places.test.ts @@ -1304,8 +1304,8 @@ describe('A-H1 / A-M1 / B-H1 — verifyTripAccess parses once with toRowId (Task // ───────────────────────────────────────────────────────────────────────────── // M1 / program rule 22 (task-4-review.md) — control characters in a user -// string round-trip on every engine. The NUL byte itself is SQLite's case and -// lives in places-nul-sqlite.test.ts: Postgres text cannot hold U+0000. +// string round-trip on every engine. The NUL byte, which neither engine +// stores, has its own suite: places-nul.test.ts. // ───────────────────────────────────────────────────────────────────────────── describe('M1 — control characters in a place name (rule 22)', () => { @@ -1313,7 +1313,7 @@ describe('M1 — control characters in a place name (rule 22)', () => { const { user } = await makeUser(orm); const trip = await makeTrip(orm, user.id); - // 0x00 is covered by places-nul-sqlite.test.ts; this sweeps the rest + // 0x00 is covered by places-nul.test.ts; this sweeps the rest // of the control-character range the review's fuzz asked for. for (let code = 0x01; code <= 0x1f; code++) { const name = `ctrl${String.fromCharCode(code)}char`; diff --git a/server/tests/postgres/lane.ts b/server/tests/postgres/lane.ts index e1afadd0a..d33f17d5c 100644 --- a/server/tests/postgres/lane.ts +++ b/server/tests/postgres/lane.ts @@ -73,10 +73,6 @@ export const SQLITE_ONLY: Readonly> = { 'tests/integration/legacy-upgrade-v205.test.ts': LEGACY_UPGRADE, 'tests/integration/legacy-upgrade-v242.test.ts': LEGACY_UPGRADE, 'tests/integration/legacy-upgrade-v244.test.ts': LEGACY_UPGRADE, - 'tests/integration/places-nul-sqlite.test.ts': - 'About SQLite value quoting of a NUL byte (U+0000) in a user string; Postgres text cannot hold that ' + - 'character, so the round-trip it asserts does not exist there. The other control characters run on ' + - 'both engines in places.test.ts.', }; const SNAPSHOT_BASELINE_FILE = path.join(__dirname, 'snapshot-suites-baseline.json'); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts index 832497dd0..1e6093dc5 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts @@ -80,14 +80,13 @@ describe('convertValue', () => { expect(convertValue('', 'float')).toMatchObject({ ok: false }); }); - it('MOVEVAL-005: text takes strings and numbers, and refuses a NUL Postgres cannot store or a BLOB', () => { + it('MOVEVAL-005: text takes strings and numbers, drops a NUL as every write does, and refuses a BLOB', () => { expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa')); expect(convertValue(7n, 'text')).toEqual(ok('7')); expect(convertValue(2.5, 'text')).toEqual(ok('2.5')); - expect(convertValue('a\u0000b', 'text')).toMatchObject({ - ok: false, - reason: expect.stringMatching(/NUL character/), - }); + expect(convertValue('a\u0000b\u0000', 'text')).toEqual(ok('ab')); + // The six characters of an escape are text like any other. + expect(convertValue('a\\u0000b', 'text')).toEqual(ok('a\\u0000b')); expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not text/), @@ -98,17 +97,17 @@ describe('convertValue', () => { }); }); - it('MOVEVAL-006: JSON must parse and hold no NUL, escaped or not', () => { + it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes', () => { expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}')); expect(convertValue(5n, 'json')).toEqual(ok('5')); expect(convertValue('{broken', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not valid JSON/), }); - expect(convertValue('"\\u0000"', 'json')).toMatchObject({ - ok: false, - reason: expect.stringMatching(/jsonb cannot store/), - }); + expect(convertValue('"\\u0000"', 'json')).toEqual(ok('""')); + expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json')).toEqual(ok('{"a":["xy",1]}')); + // A raw NUL is not valid JSON in the first place. + expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) }); expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not JSON text/), diff --git a/server/tests/unit/db/nul-free.test.ts b/server/tests/unit/db/nul-free.test.ts new file mode 100644 index 000000000..56655ebe4 --- /dev/null +++ b/server/tests/unit/db/nul-free.test.ts @@ -0,0 +1,129 @@ +/** + * TREK stores no NUL (U+0000) on either engine (src/db/nul-free.ts). These pin + * the pieces every engine's write path runs through; the integration suite + * tests/integration/places-nul.test.ts runs a write and a search through the + * whole app on SQLite and on Postgres. + */ +import { extendedQuery, unmarkedValues } from '../../../src/db/engine/postgres/extended-query'; +import { TrekPostgreSqlPlatform } from '../../../src/db/engine/postgres/postgres-driver'; +import { nulFreeJson, nulFreeParameter, stripNul, withNulFreeParameters } from '../../../src/db/nul-free'; + +import { + CompiledQuery, + type DatabaseConnection, + type Dialect, + type Driver, + type QueryResult, + SqliteAdapter, + SqliteIntrospector, + SqliteQueryCompiler, +} from 'kysely'; +import { describe, expect, it } from 'vitest'; + +const RELEASE = Symbol('release'); + +/** A connection that records what it was sent and keeps a private release, like Kysely's Postgres one. */ +class RecordingConnection implements DatabaseConnection { + readonly sent: CompiledQuery[] = []; + released = false; + #open = true; + + async executeQuery(query: CompiledQuery): Promise> { + if (!this.#open) throw new Error('closed'); + this.sent.push(query); + return { rows: [] }; + } + + async *streamQuery(query: CompiledQuery): AsyncIterableIterator> { + this.sent.push(query); + yield { rows: [] }; + } + + [RELEASE](): void { + this.released = this.#open; + } +} + +function recordingDialect(connection: RecordingConnection): Dialect { + const driver: Driver = { + init: async () => undefined, + acquireConnection: async () => connection, + beginTransaction: async (c) => { + await c.executeQuery(CompiledQuery.raw('begin')); + }, + commitTransaction: async () => undefined, + rollbackTransaction: async () => undefined, + releaseConnection: async (c) => { + (c as RecordingConnection)[RELEASE](); + }, + destroy: async () => undefined, + }; + return { + createAdapter: () => new SqliteAdapter(), + createDriver: () => driver, + createIntrospector: (db) => new SqliteIntrospector(db), + createQueryCompiler: () => new SqliteQueryCompiler(), + }; +} + +describe('NUL-free values', () => { + it('NULFREE-001: strips NUL from a string, from the strings of an array, and nothing else', () => { + expect(stripNul('a\u0000b\u0000')).toBe('ab'); + expect(stripNul('a\\u0000b')).toBe('a\\u0000b'); + const plain = 'no nul'; + expect(stripNul(plain)).toBe(plain); + expect(nulFreeParameter(['x\u0000', 2])).toEqual(['x', 2]); + const numbers = [1, 2]; + expect(nulFreeParameter(numbers)).toBe(numbers); + const bytes = Buffer.from([0, 1]); + expect(nulFreeParameter(bytes)).toBe(bytes); + const marker = { trekExtendedProtocol: true }; + expect(nulFreeParameter(marker)).toBe(marker); + expect(nulFreeParameter(null)).toBeNull(); + }); + + it('NULFREE-002: strips NUL from every string of a JSON value, keys included', () => { + expect(nulFreeJson({ 'a\u0000': ['b\u0000', 1, null, { c: 'd\u0000' }] })).toEqual({ + a: ['b', 1, null, { c: 'd' }], + }); + const date = new Date(0); + expect(nulFreeJson(date)).toBe(date); + }); + + it("NULFREE-003: a dialect's connections send every parameter without NUL and keep the rest of the query", async () => { + const connection = new RecordingConnection(); + const driver = withNulFreeParameters(recordingDialect(connection)).createDriver(); + const acquired = await driver.acquireConnection(); + expect(await driver.acquireConnection()).toBe(acquired); + + const clean = CompiledQuery.raw('select ?', ['clean']); + await acquired.executeQuery(clean); + expect(connection.sent[0]).toBe(clean); + + await acquired.executeQuery(CompiledQuery.raw('insert into t values (?, ?, ?)', ['a\u0000b', 3, ['c\u0000']])); + expect(connection.sent[1].sql).toBe('insert into t values (?, ?, ?)'); + expect(connection.sent[1].parameters).toEqual(['ab', 3, ['c']]); + + for await (const chunk of acquired.streamQuery(CompiledQuery.raw('select ?', ['s\u0000']), 10)) { + expect(chunk.rows).toEqual([]); + } + expect(connection.sent[2].parameters).toEqual(['s']); + + // The extended-protocol marker survives by identity. + await acquired.executeQuery(extendedQuery('select $1', ['m\u0000'])); + expect(unmarkedValues(connection.sent[3].parameters)).toEqual(['m']); + + // The driver's own methods get the wrapped connection and still reach its private state. + await driver.beginTransaction(acquired, {}); + expect(connection.sent[4].sql).toBe('begin'); + await driver.releaseConnection(acquired); + expect(connection.released).toBe(true); + }); + + it('NULFREE-004: the Postgres platform inlines strings and serializes JSON without NUL', () => { + const platform = new TrekPostgreSqlPlatform(); + expect(platform.escape("o\u0000'k")).toBe("'o''k'"); + expect(platform.formatQuery('select ?', ['x\u0000y'])).toBe("select 'xy'"); + expect(platform.convertJsonToDatabaseValue({ k: 'v\u0000' })).toBe('{"k":"v"}'); + }); +}); diff --git a/server/tests/unit/db/nul-safe-sqlite-platform.test.ts b/server/tests/unit/db/nul-safe-sqlite-platform.test.ts index 71ac63b75..765af718b 100644 --- a/server/tests/unit/db/nul-safe-sqlite-platform.test.ts +++ b/server/tests/unit/db/nul-safe-sqlite-platform.test.ts @@ -11,15 +11,15 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; /** * Plan 3c Task 4 review M1 / program rule 22: MikroORM 7 inlines every bound * value into the SQL text via `platform.quoteValue`/`escape` (see - * `NulSafeSqlitePlatform`'s own docstring for the full mechanism). These - * tests prove the override round-trips a NUL byte at every position SQLite - * itself will accept, by handing the platform's own generated SQL literal to - * a REAL `better-sqlite3` connection and reading the value back — not a - * string-shape assertion, an actual SQLite round-trip. + * `NulSafeSqlitePlatform`'s own docstring for the full mechanism). TREK stores + * no NUL on either engine (`src/db/nul-free.ts`), so the platform drops it from + * every string it inlines. These hand the platform's own SQL literal to a REAL + * `better-sqlite3` connection and read the value back. */ describe('NulSafeSqlitePlatform', () => { const platform = new NulSafeSqlitePlatform(); const db = new Database(':memory:'); + const withoutNul = (value: string) => value.replaceAll('\u0000', ''); function roundTrip(value: string): string { const literal = platform.escape(value); @@ -27,51 +27,29 @@ describe('NulSafeSqlitePlatform', () => { return row.v; } - it('round-trips a string with no NUL unchanged (baseline: unaffected by the override)', () => { + it('round-trips a string with no NUL unchanged', () => { expect(roundTrip('plain string')).toBe('plain string'); }); - it('round-trips a NUL in the middle of a string', () => { - const value = 'nul\u0000name'; - expect(roundTrip(value)).toBe(value); + it('drops a NUL wherever it stands, and a string of NULs becomes empty', () => { + expect(roundTrip('nul\u0000name')).toBe('nulname'); + expect(roundTrip('\u0000leading')).toBe('leading'); + expect(roundTrip('trailing\u0000')).toBe('trailing'); + expect(roundTrip('\u0000\u0000\u0000')).toBe(''); }); - it('round-trips a NUL at the very start of a string', () => { - const value = '\u0000leading'; - expect(roundTrip(value)).toBe(value); + it('keeps the quote and backslash handling of every other string next to a dropped NUL', () => { + for (const value of ["o'brien\u0000's café", 'back\\slash\u0000end\\', "a'\u0000b\\\u0000c''\u0000\\\\d"]) { + expect(roundTrip(value)).toBe(withoutNul(value)); + } }); - it('round-trips a NUL at the very end of a string', () => { - const value = 'trailing\u0000'; - expect(roundTrip(value)).toBe(value); - }); - - it('round-trips a string made ENTIRELY of NUL bytes', () => { - expect(roundTrip('\u0000')).toBe('\u0000'); - expect(roundTrip('\u0000\u0000\u0000')).toBe('\u0000\u0000\u0000'); - }); - - it('round-trips a NUL next to a single quote (the string-literal escape character)', () => { - const value = "o'brien\u0000's café"; - expect(roundTrip(value)).toBe(value); - }); - - it('round-trips a NUL next to a backslash (not special in a SQLite string literal)', () => { - const value = 'back\\slash\u0000end\\'; - expect(roundTrip(value)).toBe(value); - }); - - it('round-trips multiple NULs mixed with quotes and backslashes', () => { - const value = "a'\u0000b\\\u0000c''\u0000\\\\d"; - expect(roundTrip(value)).toBe(value); - }); - - it('fuzzes every 0x00-0x1F control character: none corrupt the round-trip and none throw', () => { + it('fuzzes every 0x00-0x1F control character: only NUL is dropped and none throw', () => { for (let code = 0x00; code <= 0x1f; code++) { const ch = String.fromCharCode(code); const value = `before${ch}after`; expect(() => roundTrip(value)).not.toThrow(); - expect(roundTrip(value)).toBe(value); + expect(roundTrip(value)).toBe(code === 0 ? 'beforeafter' : value); } }); @@ -81,14 +59,19 @@ describe('NulSafeSqlitePlatform', () => { expect(platform.escape(true)).toBe('true'); }); - it('formatQuery (the real call path AbstractSqlConnection.execute uses: formatQuery -> quoteValue -> escape) inlines a NUL-safe literal for a bound `?`, matched via a real table', () => { + it('formatQuery (the real call path AbstractSqlConnection.execute uses: formatQuery -> quoteValue -> escape) stores a value without its NUL and finds it by the same value', () => { db.exec('CREATE TABLE probe (name TEXT)'); const value = 'search\u0000term'; - const insertSql = platform.formatQuery('insert into probe (name) values (?)', [value]); - db.exec(insertSql); - const selectSql = platform.formatQuery('select name from probe where name = ?', [value]); - const row = db.prepare(selectSql).get() as { name: string } | undefined; - expect(row?.name).toBe(value); + db.exec(platform.formatQuery('insert into probe (name) values (?)', [value])); + const row = db.prepare(platform.formatQuery('select name from probe where name = ?', [value])).get() as + { name: string } | undefined; + expect(row?.name).toBe('searchterm'); + }); + + it('serializes a JSON value with every string in it, keys included, without NUL', () => { + expect(platform.convertJsonToDatabaseValue({ 'k\u0000': ['a\u0000b', 1, { n: 'x\u0000' }] })).toBe( + '{"k":["ab",1,{"n":"x"}]}', + ); }); /**