mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fix(db): store no NUL character on either engine
Postgres text cannot hold U+0000, so any write or filter carrying one (a request body, a query string, an import, a feed, a forwarded mail, a plugin's own SQL) answered a 500 on Postgres while SQLite stored it. Both engines now drop the character where every value leaves for the database (src/db/nul-free.ts): the MikroORM platforms when they inline a string or serialize a JSON value, the Kysely dialects for every parameter, a plugin's own SQLite file for its arguments, and the move from SQLite, which used to refuse such text. The same request now answers the same status and leaves the same row on both engines, so the places NUL suite runs on Postgres as well and leaves SQLITE_ONLY.
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import { nulFreeJson, stripNul, withNulFreeParameters } from '../../nul-free';
|
||||
import { sendMarkedQueriesExtended } from './extended-query';
|
||||
import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order';
|
||||
import { type Configuration } from '@mikro-orm/core';
|
||||
import { type Configuration, type TransformContext } from '@mikro-orm/core';
|
||||
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
|
||||
|
||||
/** The `pg` pool event a new client is handed to before anyone uses it. */
|
||||
@@ -59,7 +60,8 @@ class NumberTypesConnection extends PostgreSqlConnection {
|
||||
|
||||
/**
|
||||
* The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL
|
||||
* position (`null-order.ts`), on a pool whose every client sends the queries
|
||||
* position (`null-order.ts`), sending every parameter without NUL
|
||||
* (`nul-free.ts`), on a pool whose every client sends the queries
|
||||
* `extendedQuery` builds over the extended protocol (`extended-query.ts`).
|
||||
*/
|
||||
override createKyselyDialect(overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0]) {
|
||||
@@ -69,16 +71,31 @@ class NumberTypesConnection extends PostgreSqlConnection {
|
||||
return given.onPoolCreated?.(pool);
|
||||
};
|
||||
const withHook = { ...given, onPoolCreated } as Parameters<PostgreSqlConnection['createKyselyDialect']>[0];
|
||||
return withSqliteNullOrder(super.createKyselyDialect(withHook)) as ReturnType<
|
||||
return withNulFreeParameters(withSqliteNullOrder(super.createKyselyDialect(withHook))) as ReturnType<
|
||||
PostgreSqlConnection['createKyselyDialect']
|
||||
>;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres platform TREK runs on: ORDER BY with SQLite's NULL position
|
||||
* (`null-order.ts`), and no NUL in a string it inlines into a statement or a
|
||||
* JSON value it serializes, which Postgres text cannot hold (`nul-free.ts`).
|
||||
*/
|
||||
export class TrekPostgreSqlPlatform extends SqliteNullOrderPlatform {
|
||||
override escape(value: unknown): string {
|
||||
return super.escape(typeof value === 'string' ? stripNul(value) : value);
|
||||
}
|
||||
|
||||
override convertJsonToDatabaseValue(value: unknown, context?: TransformContext): unknown {
|
||||
return super.convertJsonToDatabaseValue(nulFreeJson(value), context);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
|
||||
* as numbers, and on a platform and a query compiler that sort NULLs where
|
||||
* SQLite does (`null-order.ts`). The parsers belong to this pool only:
|
||||
* as numbers, and on the platform above with a query compiler that sorts NULLs
|
||||
* where SQLite does (`null-order.ts`). The parsers belong to this pool only:
|
||||
* nothing global in `pg` changes.
|
||||
*/
|
||||
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
|
||||
@@ -89,6 +106,6 @@ export class TrekPostgreSqlDriver extends PostgreSqlDriver {
|
||||
// fields, which are not public, so they are replaced here, before anything
|
||||
// connects and before the configuration reads the platform.
|
||||
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
|
||||
(this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform();
|
||||
(this as unknown as { platform: TrekPostgreSqlPlatform }).platform = new TrekPostgreSqlPlatform();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { nulFreeJson, stripNul } from '../../../nul-free';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
/**
|
||||
@@ -7,7 +9,9 @@ import { createHash } from 'node:crypto';
|
||||
* SQLite stores a value by what it is, not by the column's declared type, so
|
||||
* a column may hold text where the schema says integer. The move never
|
||||
* guesses: a value the Postgres column cannot hold exactly is refused with
|
||||
* the reason, and the move stops before it commits anything.
|
||||
* the reason, and the move stops before it commits anything. The one
|
||||
* exception is a NUL character in text or JSON, which TREK stores on neither
|
||||
* engine (`db/nul-free.ts`): the move drops it, as every write does.
|
||||
*
|
||||
* The SQLite side is read with better-sqlite3's safe integers, so an integer
|
||||
* arrives as a `bigint` and keeps all 64 bits.
|
||||
@@ -29,7 +33,6 @@ const INT64_MAX = 2n ** 63n - 1n;
|
||||
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
|
||||
const DECIMAL_INTEGER = /^[+-]?\d+$/;
|
||||
const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
|
||||
const NUL = '\u0000';
|
||||
|
||||
/**
|
||||
* The kind of a Postgres column, from `information_schema.columns.data_type`.
|
||||
@@ -100,11 +103,7 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
}
|
||||
|
||||
function toText(value: Exclude<SqliteValue, null>): Converted {
|
||||
if (typeof value === 'string') {
|
||||
return value.includes(NUL)
|
||||
? refuse('the text holds a NUL character, which Postgres text cannot store')
|
||||
: { ok: true, value };
|
||||
}
|
||||
if (typeof value === 'string') return { ok: true, value: stripNul(value) };
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
return refuse(`${describe(value)} is not text`);
|
||||
}
|
||||
@@ -112,15 +111,14 @@ function toText(value: Exclude<SqliteValue, null>): Converted {
|
||||
function toJson(value: Exclude<SqliteValue, null>): Converted {
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
JSON.parse(value);
|
||||
parsed = JSON.parse(value);
|
||||
} catch {
|
||||
return refuse(`${describe(value)} is not valid JSON`);
|
||||
}
|
||||
// jsonb refuses the escaped NUL character as well as the raw one.
|
||||
if (value.includes(NUL) || value.includes('\\u0000'))
|
||||
return refuse('the JSON holds a NUL character, which jsonb cannot store');
|
||||
return { ok: true, value };
|
||||
// Valid JSON holds a NUL only escaped, which jsonb refuses as well: the value is written again without it.
|
||||
return { ok: true, value: value.includes('\\u0000') ? JSON.stringify(nulFreeJson(parsed)) : value };
|
||||
}
|
||||
|
||||
function toBytes(value: Exclude<SqliteValue, null>): Converted {
|
||||
|
||||
Reference in New Issue
Block a user