fix(db): close the move's remaining silent changes

An attack on the move to Postgres with real SQLite files found five values
that changed without a refusal or a listing, or a valid file that failed:

- Negative zero in a plugin column became 0: pg sends the number -0 as "0",
  and the checksum printed both as "0". It is sent as text and kept apart in
  the checksum on both sides.
- Text whose bytes are not valid UTF-8 was read with each bad sequence
  replaced by U+FFFD, and the checksum was built from the replaced text.
  Text is now read as its bytes and decoded strictly in the file's encoding
  (UTF-8, or UTF-16 for a file created so); such a value is refused in
  TREK's and in plugin data, with --drop-nul too, and listed with its bytes.
- --drop-nul serialized a JSON value again, so a large number lost digits
  and 1.0 became 1. Only the \u0000 escapes are removed now.
- A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB
  column) failed the move, and DEFAULT 1.5 on an integer column would have
  been rounded. Such a default is left off and listed as not carried over.
- A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM
  became one that compares exactly. It is still created, and the summary
  lists the collation it lost.

Each case has a regression test that fails without its fix, on SQLite and
on the Postgres lane.
This commit is contained in:
Maurice
2026-11-17 23:30:35 +00:00
parent 636fc7d0a6
commit f2cfbc4041
9 changed files with 315 additions and 46 deletions
@@ -38,8 +38,9 @@ import type { Kysely } from 'kysely';
* migrated by starting this TREK on it once, a newer one belongs to the
* TREK that wrote it. Nothing else is checked by guess.
* 2. Every plugin's data must move unchanged (`PLUGIN_RULES` in `values.ts`):
* a value Postgres cannot hand back to the plugin as it is (a NUL, a
* column whose values mix types) is listed, before anything is written.
* a value Postgres cannot hand back to the plugin as it is (a NUL, text
* whose bytes are not UTF-8, a column whose values mix types) is listed,
* before anything is written.
* 3. The Postgres database must hold no table and no plugin schema.
* 4. The schema is created exactly as a first boot creates it (the engine's
* bootstrap: the baseline and the migrations after its cut), without the
@@ -1,6 +1,6 @@
import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source';
import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema';
import { portableDefault } from '../schema-adjust';
import { PG_NOW_TEXT_DEFAULT, portableDefault } from '../schema-adjust';
import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
import type { ColumnKind } from './values';
@@ -21,7 +21,8 @@ import path from 'node:path';
* migrations again over the copied tables. What has no Postgres counterpart
* here (foreign keys between its tables, views, triggers, partial or
* expression indexes, other defaults) is not created and is listed in the
* summary. The file itself stays where it is, untouched.
* summary, and so is a key SQLite compared with `NOCASE` or `RTRIM`, which
* Postgres compares exactly. The file itself stays where it is, untouched.
*
* The rows are the plugin's, so none is changed on the way: every value must
* come back from the Postgres store with the type and bytes it has in the
@@ -69,9 +70,30 @@ export function pluginColumnKind(declaredType: string, storage: ReadonlySet<stri
}
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const INTEGER_LITERAL = /^[+-]?\d+$/;
const STRING_LITERAL = /^'(?:[^']|'')*'$/s;
/** The Postgres spelling of a column default, or null when it has none the move can trust. */
/** A number literal as a default of `kind`, only where the column stores it as SQLite does. */
function numberDefault(text: string, kind: ColumnKind): string | null {
switch (kind) {
case 'integer':
// A fraction in a bigint column would be rounded, where SQLite keeps it.
return INTEGER_LITERAL.test(text) ? text : null;
case 'float':
return text;
case 'text':
return `'${text}'`;
default:
return null;
}
}
/**
* The Postgres spelling of a column default, or null when it has none the move
* can trust: a default must give the plugin the value SQLite gives it, and must
* be one Postgres takes for the column's type, so a default that fits neither
* is listed as not carried over instead of failing the move or changing.
*/
export function pluginDefault(sql: string | null, kind: ColumnKind): string | null {
if (sql === null) return null;
const text = sql
@@ -79,11 +101,13 @@ export function pluginDefault(sql: string | null, kind: ColumnKind): string | nu
.replace(/^\((.*)\)$/s, '$1')
.trim();
const ported = portableDefault(text);
if (ported !== text) return ported;
// The time as text belongs in a text column; the seconds since 1970 fit any column but bytes.
if (ported === PG_NOW_TEXT_DEFAULT) return kind === 'text' ? ported : null;
if (ported !== text) return kind === 'bytes' ? null : ported;
if (/^null$/i.test(text)) return 'null';
if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text;
if (NUMBER_LITERAL.test(text)) return numberDefault(text, kind);
if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null;
if (STRING_LITERAL.test(text)) return text;
if (STRING_LITERAL.test(text)) return kind === 'text' ? text : null;
return null;
}
@@ -113,6 +137,14 @@ function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, no
}
function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null {
// Postgres compares the copied text exactly, so a key SQLite compared ignoring case or trailing spaces
// (`COLLATE NOCASE`, `RTRIM`) is created, or kept as the primary key, but enforces less than it did.
if (index.collations.length > 0) {
notes.push(
`the collation ${index.collations.join(', ')} of the ${index.origin === 'pk' ? 'primary key' : `index ${index.name}`} ` +
`on ${table} (Postgres compares its values exactly)`,
);
}
if (index.origin === 'pk') return null;
if (index.partial || index.columns === null) {
notes.push(`the index ${index.name} on ${table} (partial or over an expression)`);
@@ -1,4 +1,5 @@
import { nulFreeJson, stripNul } from '../../../nul-free';
import { stripNul } from '../../../nul-free';
import { InvalidUtf8Text } from '../../sqlite/sqlite-source';
import { createHash } from 'node:crypto';
@@ -28,14 +29,17 @@ import { createHash } from 'node:crypto';
* that changed a value fails the verification instead of passing it.
*
* The SQLite side is read with better-sqlite3's safe integers, so an integer
* arrives as a `bigint` and keeps all 64 bits.
* arrives as a `bigint` and keeps all 64 bits. A TEXT value whose bytes are
* not valid UTF-8 arrives as an {@link InvalidUtf8Text} and is refused under
* either set of rules: Postgres text cannot hold those bytes, and reading
* them as text would replace them without a trace.
*/
/** The Postgres column types the move writes, by what they hold. */
export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes';
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */
export type SqliteValue = null | bigint | number | string | Uint8Array;
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT (valid UTF-8 or not) or BLOB. */
export type SqliteValue = null | bigint | number | string | InvalidUtf8Text | Uint8Array;
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
export type PostgresValue = null | string | number | Buffer;
@@ -100,6 +104,7 @@ function refuse(reason: string): Converted {
function describe(value: SqliteValue): string {
if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`;
if (value instanceof InvalidUtf8Text) return describe(value.text);
if (typeof value === 'string')
return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`;
return `the value ${String(value)}`;
@@ -120,8 +125,21 @@ function toInteger(value: Exclude<SqliteValue, null>): Converted {
return { ok: true, value: whole.toString() };
}
/**
* A double as the parameter Postgres takes. Negative zero is sent as text:
* `pg` spells the number -0 as "0", which would drop its sign.
*/
function floatParameter(value: number): PostgresValue {
return Object.is(value, -0) ? '-0' : value;
}
/** A double as the compared text, negative zero kept apart from zero as Postgres prints it. */
function floatText(value: number): string {
return Object.is(value, -0) ? '-0' : String(value);
}
function toFloat(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'number') return { ok: true, value };
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
if (typeof value === 'bigint') {
const number = Number(value);
if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`);
@@ -129,7 +147,7 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
}
if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) {
const number = Number(value.trim());
if (Number.isFinite(number)) return { ok: true, value: number };
if (Number.isFinite(number)) return { ok: true, value: floatParameter(number) };
}
return refuse(`${describe(value)} is not a number`);
}
@@ -154,6 +172,15 @@ function jsonHoldsNul(value: unknown): boolean {
return false;
}
/**
* JSON text without its `\u0000` escapes, everything else exactly as written,
* so dropping a NUL changes no number or spelling around it. Each escape is
* read as a whole, so an escaped backslash followed by `u0000` stays.
*/
function dropJsonNul(text: string): string {
return text.replace(/\\(?:u0000|[\s\S])/g, (escape) => (escape === '\\u0000' ? '' : escape));
}
function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted {
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
@@ -166,7 +193,7 @@ function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted
// Valid JSON holds a NUL only escaped, which jsonb refuses as well.
if (!jsonHoldsNul(parsed)) return { ok: true, value };
if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`);
return { ok: true, value: JSON.stringify(nulFreeJson(parsed)), droppedNul: true };
return { ok: true, value: dropJsonNul(value), droppedNul: true };
}
function toBytes(value: Exclude<SqliteValue, null>): Converted {
@@ -188,7 +215,7 @@ const KIND_HOLDS: Record<ColumnKind, string> = {
/** The SQLite storage class of a value, for a refusal. */
function storageClass(value: Exclude<SqliteValue, null>): string {
if (value instanceof Uint8Array) return 'a BLOB';
if (typeof value === 'string') return 'TEXT';
if (typeof value === 'string' || value instanceof InvalidUtf8Text) return 'TEXT';
return typeof value === 'bigint' ? 'an INTEGER' : 'a REAL';
}
@@ -213,7 +240,7 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
}
return { ok: true, value: value.toString() };
case 'float':
if (typeof value === 'number') return { ok: true, value };
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
if (typeof value !== 'bigint') return changed();
return value > SAFE_MAX || value < -SAFE_MAX
? refuse(`${describe(value)} has more digits than a double holds`)
@@ -231,6 +258,11 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
/** `value` as the Postgres column of `kind` takes it under `rules`, or the reason it cannot hold it exactly. */
export function convertValue(value: SqliteValue, kind: ColumnKind, rules: ConvertRules = TREK_RULES): Converted {
if (value === null) return { ok: true, value: null };
if (value instanceof InvalidUtf8Text) {
return refuse(
`${describe(value)} holds bytes that are not valid UTF-8 (${hex(value.bytes)}), which Postgres text cannot hold`,
);
}
if (rules.exact) return exactly(value, kind);
switch (kind) {
case 'integer':
@@ -291,17 +323,19 @@ export function canonicalFromSource(
): string | null {
if (value === null) return null;
if (value instanceof Uint8Array) return kind === 'bytes' ? hex(value) : `blob:${hex(value)}`;
if (value instanceof InvalidUtf8Text) return `invalid-utf8:${hex(value.bytes)}`;
const dropNul = rules.dropNul && !rules.exact;
switch (kind) {
case 'integer':
return typeof value === 'bigint' ? value.toString() : BigInt(String(value).trim()).toString();
case 'float':
return typeof value === 'bigint' ? value.toString() : String(Number(String(value).trim()));
if (typeof value === 'number') return floatText(value);
return typeof value === 'bigint' ? value.toString() : floatText(Number(value.trim()));
case 'text':
return typeof value === 'string' && dropNul ? stripNul(value) : String(value);
case 'json':
if (typeof value !== 'string') return stableJson(String(value));
return stableJson(dropNul ? JSON.stringify(nulFreeJson(JSON.parse(value))) : value);
return stableJson(dropNul ? dropJsonNul(value) : value);
case 'bytes':
return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`;
}
@@ -317,7 +351,7 @@ export function canonicalFromPostgres(text: string | null, kind: ColumnKind): st
case 'integer':
return BigInt(text).toString();
case 'float':
return String(Number(text));
return floatText(Number(text));
case 'text':
return text;
case 'json':
+81 -7
View File
@@ -9,9 +9,58 @@ import fs from 'node:fs';
* whole move sees the file as it was at one moment.
*
* Integers come back as `bigint` (better-sqlite3's safe integers), so a value
* past 2^53 keeps every digit.
* past 2^53 keeps every digit. Text is read as its bytes and decoded here, so
* a value whose bytes are not valid UTF-8 (which SQLite stores as it is
* given) arrives as an {@link InvalidUtf8Text} instead of with its bytes
* silently replaced by U+FFFD.
*/
/**
* A TEXT value whose bytes are not valid text in the file's encoding (UTF-8,
* or UTF-16 for a file created so): the bytes the file holds, and the text
* with each bad sequence as U+FFFD.
*/
export class InvalidUtf8Text {
constructor(
readonly bytes: Buffer,
readonly text: string,
) {}
/** The text, as a refusal names a row by it. */
toString(): string {
return this.text;
}
}
/** The decoder label of each text encoding a SQLite file can have (`pragma encoding`). */
const ENCODINGS: Record<string, string> = { 'UTF-8': 'utf-8', 'UTF-16le': 'utf-16le', 'UTF-16be': 'utf-16be' };
/**
* Decodes the TEXT values of a file in its encoding: a TEXT value cast to a
* BLOB is its bytes in that encoding. Strict, so bytes that are not valid
* text throw, and a leading byte order mark is kept as the text holds it.
*/
class TextReader {
private readonly strict: TextDecoder;
private readonly lenient: TextDecoder;
constructor(encoding: string) {
const label = ENCODINGS[encoding];
if (!label) throw new Error(`The SQLite file has the text encoding ${encoding}, which TREK cannot read.`);
this.strict = new TextDecoder(label, { fatal: true, ignoreBOM: true });
this.lenient = new TextDecoder(label, { ignoreBOM: true });
}
/** A TEXT value from its bytes. */
decode(bytes: Uint8Array): string | InvalidUtf8Text {
try {
return this.strict.decode(bytes);
} catch {
return new InvalidUtf8Text(Buffer.from(bytes), this.lenient.decode(bytes));
}
}
}
/** A column as `pragma_table_info` describes it. */
export interface SqliteColumn {
name: string;
@@ -33,6 +82,8 @@ export interface SqliteIndex {
partial: boolean;
/** The indexed columns in order; null when one of them is an expression. */
columns: string[] | null;
/** The collating sequences its keys compare with other than SQLite's default `BINARY` (`NOCASE`, `RTRIM`). */
collations: string[];
}
/** A row whose foreign key points at nothing, from `pragma_foreign_key_check`. */
@@ -51,10 +102,14 @@ function quoteSqliteIdentifier(name: string): string {
type SqliteHandle = ReturnType<typeof openDatabase>;
export class SqliteSource {
private readonly text: TextReader;
private constructor(
private readonly db: SqliteHandle,
readonly file: string,
) {}
) {
this.text = new TextReader(String(db.pragma('encoding', { simple: true })));
}
/**
* Opens `file` read-only and starts the read transaction. A file that does
@@ -141,10 +196,11 @@ export class SqliteSource {
}>;
return list.map((index) => {
const keys = this.db
.prepare('select cid, name from pragma_index_info(?) order by seqno')
.prepare('select cid, name, coll from pragma_index_xinfo(?) where key = 1 order by seqno')
.all(index.name) as Array<{
cid: bigint;
name: string | null;
coll: string;
}>;
const columns = keys.every((key) => key.cid >= 0n && key.name !== null)
? keys.map((key) => String(key.name))
@@ -155,6 +211,9 @@ export class SqliteSource {
origin: index.origin,
partial: index.partial !== 0n,
columns,
collations: [
...new Set(keys.map((key) => key.coll.toUpperCase()).filter((collation) => collation !== 'BINARY')),
],
};
});
}
@@ -180,13 +239,28 @@ export class SqliteSource {
);
}
/** The rows of `table`, one array of `columns` per row, read one at a time. */
rows(table: string, columns: readonly string[]): IterableIterator<unknown[]> {
const list = columns.map(quoteSqliteIdentifier).join(', ');
return this.db
/**
* The rows of `table`, one array of `columns` per row, read one at a time.
* Each column is read with its storage class, and TEXT as its bytes, which
* decoded in the file's encoding into a string or an {@link InvalidUtf8Text}.
*/
*rows(table: string, columns: readonly string[]): IterableIterator<unknown[]> {
const list = columns
.map(quoteSqliteIdentifier)
.map(
(column) =>
`typeof(${column}), case typeof(${column}) when 'text' then cast(${column} as blob) else ${column} end`,
)
.join(', ');
const rows = this.db
.prepare(`select ${list} from ${quoteSqliteIdentifier(table)}`)
.raw(true)
.iterate() as IterableIterator<unknown[]>;
for (const raw of rows) {
yield columns.map((_column, index) =>
raw[index * 2] === 'text' ? this.text.decode(raw[index * 2 + 1] as Uint8Array) : raw[index * 2 + 1],
);
}
}
/** The highest id each AUTOINCREMENT table ever handed out, which can be above the largest id left. */