mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 11:36:05 +00:00
fix(db): close the move's remaining silent changes
An attack on the move to Postgres with real SQLite files found five values that changed without a refusal or a listing, or a valid file that failed: - Negative zero in a plugin column became 0: pg sends the number -0 as "0", and the checksum printed both as "0". It is sent as text and kept apart in the checksum on both sides. - Text whose bytes are not valid UTF-8 was read with each bad sequence replaced by U+FFFD, and the checksum was built from the replaced text. Text is now read as its bytes and decoded strictly in the file's encoding (UTF-8, or UTF-16 for a file created so); such a value is refused in TREK's and in plugin data, with --drop-nul too, and listed with its bytes. - --drop-nul serialized a JSON value again, so a large number lost digits and 1.0 became 1. Only the \u0000 escapes are removed now. - A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB column) failed the move, and DEFAULT 1.5 on an integer column would have been rounded. Such a default is left off and listed as not carried over. - A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM became one that compares exactly. It is still created, and the summary lists the collation it lost. Each case has a regression test that fails without its fix, on SQLite and on the Postgres lane.
This commit is contained in:
@@ -38,8 +38,9 @@ import type { Kysely } from 'kysely';
|
||||
* migrated by starting this TREK on it once, a newer one belongs to the
|
||||
* TREK that wrote it. Nothing else is checked by guess.
|
||||
* 2. Every plugin's data must move unchanged (`PLUGIN_RULES` in `values.ts`):
|
||||
* a value Postgres cannot hand back to the plugin as it is (a NUL, a
|
||||
* column whose values mix types) is listed, before anything is written.
|
||||
* a value Postgres cannot hand back to the plugin as it is (a NUL, text
|
||||
* whose bytes are not UTF-8, a column whose values mix types) is listed,
|
||||
* before anything is written.
|
||||
* 3. The Postgres database must hold no table and no plugin schema.
|
||||
* 4. The schema is created exactly as a first boot creates it (the engine's
|
||||
* bootstrap: the baseline and the migrations after its cut), without the
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source';
|
||||
import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema';
|
||||
import { portableDefault } from '../schema-adjust';
|
||||
import { PG_NOW_TEXT_DEFAULT, portableDefault } from '../schema-adjust';
|
||||
import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
|
||||
import type { ColumnKind } from './values';
|
||||
|
||||
@@ -21,7 +21,8 @@ import path from 'node:path';
|
||||
* migrations again over the copied tables. What has no Postgres counterpart
|
||||
* here (foreign keys between its tables, views, triggers, partial or
|
||||
* expression indexes, other defaults) is not created and is listed in the
|
||||
* summary. The file itself stays where it is, untouched.
|
||||
* summary, and so is a key SQLite compared with `NOCASE` or `RTRIM`, which
|
||||
* Postgres compares exactly. The file itself stays where it is, untouched.
|
||||
*
|
||||
* The rows are the plugin's, so none is changed on the way: every value must
|
||||
* come back from the Postgres store with the type and bytes it has in the
|
||||
@@ -69,9 +70,30 @@ export function pluginColumnKind(declaredType: string, storage: ReadonlySet<stri
|
||||
}
|
||||
|
||||
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
|
||||
const INTEGER_LITERAL = /^[+-]?\d+$/;
|
||||
const STRING_LITERAL = /^'(?:[^']|'')*'$/s;
|
||||
|
||||
/** The Postgres spelling of a column default, or null when it has none the move can trust. */
|
||||
/** A number literal as a default of `kind`, only where the column stores it as SQLite does. */
|
||||
function numberDefault(text: string, kind: ColumnKind): string | null {
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
// A fraction in a bigint column would be rounded, where SQLite keeps it.
|
||||
return INTEGER_LITERAL.test(text) ? text : null;
|
||||
case 'float':
|
||||
return text;
|
||||
case 'text':
|
||||
return `'${text}'`;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres spelling of a column default, or null when it has none the move
|
||||
* can trust: a default must give the plugin the value SQLite gives it, and must
|
||||
* be one Postgres takes for the column's type, so a default that fits neither
|
||||
* is listed as not carried over instead of failing the move or changing.
|
||||
*/
|
||||
export function pluginDefault(sql: string | null, kind: ColumnKind): string | null {
|
||||
if (sql === null) return null;
|
||||
const text = sql
|
||||
@@ -79,11 +101,13 @@ export function pluginDefault(sql: string | null, kind: ColumnKind): string | nu
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
const ported = portableDefault(text);
|
||||
if (ported !== text) return ported;
|
||||
// The time as text belongs in a text column; the seconds since 1970 fit any column but bytes.
|
||||
if (ported === PG_NOW_TEXT_DEFAULT) return kind === 'text' ? ported : null;
|
||||
if (ported !== text) return kind === 'bytes' ? null : ported;
|
||||
if (/^null$/i.test(text)) return 'null';
|
||||
if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text;
|
||||
if (NUMBER_LITERAL.test(text)) return numberDefault(text, kind);
|
||||
if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null;
|
||||
if (STRING_LITERAL.test(text)) return text;
|
||||
if (STRING_LITERAL.test(text)) return kind === 'text' ? text : null;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -113,6 +137,14 @@ function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, no
|
||||
}
|
||||
|
||||
function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null {
|
||||
// Postgres compares the copied text exactly, so a key SQLite compared ignoring case or trailing spaces
|
||||
// (`COLLATE NOCASE`, `RTRIM`) is created, or kept as the primary key, but enforces less than it did.
|
||||
if (index.collations.length > 0) {
|
||||
notes.push(
|
||||
`the collation ${index.collations.join(', ')} of the ${index.origin === 'pk' ? 'primary key' : `index ${index.name}`} ` +
|
||||
`on ${table} (Postgres compares its values exactly)`,
|
||||
);
|
||||
}
|
||||
if (index.origin === 'pk') return null;
|
||||
if (index.partial || index.columns === null) {
|
||||
notes.push(`the index ${index.name} on ${table} (partial or over an expression)`);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { nulFreeJson, stripNul } from '../../../nul-free';
|
||||
import { stripNul } from '../../../nul-free';
|
||||
import { InvalidUtf8Text } from '../../sqlite/sqlite-source';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
@@ -28,14 +29,17 @@ import { createHash } from 'node:crypto';
|
||||
* that changed a value fails the verification instead of passing it.
|
||||
*
|
||||
* The SQLite side is read with better-sqlite3's safe integers, so an integer
|
||||
* arrives as a `bigint` and keeps all 64 bits.
|
||||
* arrives as a `bigint` and keeps all 64 bits. A TEXT value whose bytes are
|
||||
* not valid UTF-8 arrives as an {@link InvalidUtf8Text} and is refused under
|
||||
* either set of rules: Postgres text cannot hold those bytes, and reading
|
||||
* them as text would replace them without a trace.
|
||||
*/
|
||||
|
||||
/** The Postgres column types the move writes, by what they hold. */
|
||||
export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes';
|
||||
|
||||
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */
|
||||
export type SqliteValue = null | bigint | number | string | Uint8Array;
|
||||
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT (valid UTF-8 or not) or BLOB. */
|
||||
export type SqliteValue = null | bigint | number | string | InvalidUtf8Text | Uint8Array;
|
||||
|
||||
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
|
||||
export type PostgresValue = null | string | number | Buffer;
|
||||
@@ -100,6 +104,7 @@ function refuse(reason: string): Converted {
|
||||
|
||||
function describe(value: SqliteValue): string {
|
||||
if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`;
|
||||
if (value instanceof InvalidUtf8Text) return describe(value.text);
|
||||
if (typeof value === 'string')
|
||||
return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`;
|
||||
return `the value ${String(value)}`;
|
||||
@@ -120,8 +125,21 @@ function toInteger(value: Exclude<SqliteValue, null>): Converted {
|
||||
return { ok: true, value: whole.toString() };
|
||||
}
|
||||
|
||||
/**
|
||||
* A double as the parameter Postgres takes. Negative zero is sent as text:
|
||||
* `pg` spells the number -0 as "0", which would drop its sign.
|
||||
*/
|
||||
function floatParameter(value: number): PostgresValue {
|
||||
return Object.is(value, -0) ? '-0' : value;
|
||||
}
|
||||
|
||||
/** A double as the compared text, negative zero kept apart from zero as Postgres prints it. */
|
||||
function floatText(value: number): string {
|
||||
return Object.is(value, -0) ? '-0' : String(value);
|
||||
}
|
||||
|
||||
function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
if (typeof value === 'number') return { ok: true, value };
|
||||
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
|
||||
if (typeof value === 'bigint') {
|
||||
const number = Number(value);
|
||||
if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`);
|
||||
@@ -129,7 +147,7 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
}
|
||||
if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) {
|
||||
const number = Number(value.trim());
|
||||
if (Number.isFinite(number)) return { ok: true, value: number };
|
||||
if (Number.isFinite(number)) return { ok: true, value: floatParameter(number) };
|
||||
}
|
||||
return refuse(`${describe(value)} is not a number`);
|
||||
}
|
||||
@@ -154,6 +172,15 @@ function jsonHoldsNul(value: unknown): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON text without its `\u0000` escapes, everything else exactly as written,
|
||||
* so dropping a NUL changes no number or spelling around it. Each escape is
|
||||
* read as a whole, so an escaped backslash followed by `u0000` stays.
|
||||
*/
|
||||
function dropJsonNul(text: string): string {
|
||||
return text.replace(/\\(?:u0000|[\s\S])/g, (escape) => (escape === '\\u0000' ? '' : escape));
|
||||
}
|
||||
|
||||
function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted {
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
|
||||
@@ -166,7 +193,7 @@ function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted
|
||||
// Valid JSON holds a NUL only escaped, which jsonb refuses as well.
|
||||
if (!jsonHoldsNul(parsed)) return { ok: true, value };
|
||||
if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`);
|
||||
return { ok: true, value: JSON.stringify(nulFreeJson(parsed)), droppedNul: true };
|
||||
return { ok: true, value: dropJsonNul(value), droppedNul: true };
|
||||
}
|
||||
|
||||
function toBytes(value: Exclude<SqliteValue, null>): Converted {
|
||||
@@ -188,7 +215,7 @@ const KIND_HOLDS: Record<ColumnKind, string> = {
|
||||
/** The SQLite storage class of a value, for a refusal. */
|
||||
function storageClass(value: Exclude<SqliteValue, null>): string {
|
||||
if (value instanceof Uint8Array) return 'a BLOB';
|
||||
if (typeof value === 'string') return 'TEXT';
|
||||
if (typeof value === 'string' || value instanceof InvalidUtf8Text) return 'TEXT';
|
||||
return typeof value === 'bigint' ? 'an INTEGER' : 'a REAL';
|
||||
}
|
||||
|
||||
@@ -213,7 +240,7 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
|
||||
}
|
||||
return { ok: true, value: value.toString() };
|
||||
case 'float':
|
||||
if (typeof value === 'number') return { ok: true, value };
|
||||
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
|
||||
if (typeof value !== 'bigint') return changed();
|
||||
return value > SAFE_MAX || value < -SAFE_MAX
|
||||
? refuse(`${describe(value)} has more digits than a double holds`)
|
||||
@@ -231,6 +258,11 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
|
||||
/** `value` as the Postgres column of `kind` takes it under `rules`, or the reason it cannot hold it exactly. */
|
||||
export function convertValue(value: SqliteValue, kind: ColumnKind, rules: ConvertRules = TREK_RULES): Converted {
|
||||
if (value === null) return { ok: true, value: null };
|
||||
if (value instanceof InvalidUtf8Text) {
|
||||
return refuse(
|
||||
`${describe(value)} holds bytes that are not valid UTF-8 (${hex(value.bytes)}), which Postgres text cannot hold`,
|
||||
);
|
||||
}
|
||||
if (rules.exact) return exactly(value, kind);
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
@@ -291,17 +323,19 @@ export function canonicalFromSource(
|
||||
): string | null {
|
||||
if (value === null) return null;
|
||||
if (value instanceof Uint8Array) return kind === 'bytes' ? hex(value) : `blob:${hex(value)}`;
|
||||
if (value instanceof InvalidUtf8Text) return `invalid-utf8:${hex(value.bytes)}`;
|
||||
const dropNul = rules.dropNul && !rules.exact;
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
return typeof value === 'bigint' ? value.toString() : BigInt(String(value).trim()).toString();
|
||||
case 'float':
|
||||
return typeof value === 'bigint' ? value.toString() : String(Number(String(value).trim()));
|
||||
if (typeof value === 'number') return floatText(value);
|
||||
return typeof value === 'bigint' ? value.toString() : floatText(Number(value.trim()));
|
||||
case 'text':
|
||||
return typeof value === 'string' && dropNul ? stripNul(value) : String(value);
|
||||
case 'json':
|
||||
if (typeof value !== 'string') return stableJson(String(value));
|
||||
return stableJson(dropNul ? JSON.stringify(nulFreeJson(JSON.parse(value))) : value);
|
||||
return stableJson(dropNul ? dropJsonNul(value) : value);
|
||||
case 'bytes':
|
||||
return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`;
|
||||
}
|
||||
@@ -317,7 +351,7 @@ export function canonicalFromPostgres(text: string | null, kind: ColumnKind): st
|
||||
case 'integer':
|
||||
return BigInt(text).toString();
|
||||
case 'float':
|
||||
return String(Number(text));
|
||||
return floatText(Number(text));
|
||||
case 'text':
|
||||
return text;
|
||||
case 'json':
|
||||
|
||||
Reference in New Issue
Block a user