mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fix(db): close the move's remaining silent changes
An attack on the move to Postgres with real SQLite files found five values that changed without a refusal or a listing, or a valid file that failed: - Negative zero in a plugin column became 0: pg sends the number -0 as "0", and the checksum printed both as "0". It is sent as text and kept apart in the checksum on both sides. - Text whose bytes are not valid UTF-8 was read with each bad sequence replaced by U+FFFD, and the checksum was built from the replaced text. Text is now read as its bytes and decoded strictly in the file's encoding (UTF-8, or UTF-16 for a file created so); such a value is refused in TREK's and in plugin data, with --drop-nul too, and listed with its bytes. - --drop-nul serialized a JSON value again, so a large number lost digits and 1.0 became 1. Only the \u0000 escapes are removed now. - A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB column) failed the move, and DEFAULT 1.5 on an integer column would have been rounded. Such a default is left off and listed as not carried over. - A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM became one that compares exactly. It is still created, and the summary lists the collation it lost. Each case has a regression test that fails without its fix, on SQLite and on the Postgres lane.
This commit is contained in:
@@ -38,8 +38,9 @@ import type { Kysely } from 'kysely';
|
||||
* migrated by starting this TREK on it once, a newer one belongs to the
|
||||
* TREK that wrote it. Nothing else is checked by guess.
|
||||
* 2. Every plugin's data must move unchanged (`PLUGIN_RULES` in `values.ts`):
|
||||
* a value Postgres cannot hand back to the plugin as it is (a NUL, a
|
||||
* column whose values mix types) is listed, before anything is written.
|
||||
* a value Postgres cannot hand back to the plugin as it is (a NUL, text
|
||||
* whose bytes are not UTF-8, a column whose values mix types) is listed,
|
||||
* before anything is written.
|
||||
* 3. The Postgres database must hold no table and no plugin schema.
|
||||
* 4. The schema is created exactly as a first boot creates it (the engine's
|
||||
* bootstrap: the baseline and the migrations after its cut), without the
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source';
|
||||
import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema';
|
||||
import { portableDefault } from '../schema-adjust';
|
||||
import { PG_NOW_TEXT_DEFAULT, portableDefault } from '../schema-adjust';
|
||||
import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
|
||||
import type { ColumnKind } from './values';
|
||||
|
||||
@@ -21,7 +21,8 @@ import path from 'node:path';
|
||||
* migrations again over the copied tables. What has no Postgres counterpart
|
||||
* here (foreign keys between its tables, views, triggers, partial or
|
||||
* expression indexes, other defaults) is not created and is listed in the
|
||||
* summary. The file itself stays where it is, untouched.
|
||||
* summary, and so is a key SQLite compared with `NOCASE` or `RTRIM`, which
|
||||
* Postgres compares exactly. The file itself stays where it is, untouched.
|
||||
*
|
||||
* The rows are the plugin's, so none is changed on the way: every value must
|
||||
* come back from the Postgres store with the type and bytes it has in the
|
||||
@@ -69,9 +70,30 @@ export function pluginColumnKind(declaredType: string, storage: ReadonlySet<stri
|
||||
}
|
||||
|
||||
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
|
||||
const INTEGER_LITERAL = /^[+-]?\d+$/;
|
||||
const STRING_LITERAL = /^'(?:[^']|'')*'$/s;
|
||||
|
||||
/** The Postgres spelling of a column default, or null when it has none the move can trust. */
|
||||
/** A number literal as a default of `kind`, only where the column stores it as SQLite does. */
|
||||
function numberDefault(text: string, kind: ColumnKind): string | null {
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
// A fraction in a bigint column would be rounded, where SQLite keeps it.
|
||||
return INTEGER_LITERAL.test(text) ? text : null;
|
||||
case 'float':
|
||||
return text;
|
||||
case 'text':
|
||||
return `'${text}'`;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Postgres spelling of a column default, or null when it has none the move
|
||||
* can trust: a default must give the plugin the value SQLite gives it, and must
|
||||
* be one Postgres takes for the column's type, so a default that fits neither
|
||||
* is listed as not carried over instead of failing the move or changing.
|
||||
*/
|
||||
export function pluginDefault(sql: string | null, kind: ColumnKind): string | null {
|
||||
if (sql === null) return null;
|
||||
const text = sql
|
||||
@@ -79,11 +101,13 @@ export function pluginDefault(sql: string | null, kind: ColumnKind): string | nu
|
||||
.replace(/^\((.*)\)$/s, '$1')
|
||||
.trim();
|
||||
const ported = portableDefault(text);
|
||||
if (ported !== text) return ported;
|
||||
// The time as text belongs in a text column; the seconds since 1970 fit any column but bytes.
|
||||
if (ported === PG_NOW_TEXT_DEFAULT) return kind === 'text' ? ported : null;
|
||||
if (ported !== text) return kind === 'bytes' ? null : ported;
|
||||
if (/^null$/i.test(text)) return 'null';
|
||||
if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text;
|
||||
if (NUMBER_LITERAL.test(text)) return numberDefault(text, kind);
|
||||
if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null;
|
||||
if (STRING_LITERAL.test(text)) return text;
|
||||
if (STRING_LITERAL.test(text)) return kind === 'text' ? text : null;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -113,6 +137,14 @@ function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, no
|
||||
}
|
||||
|
||||
function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null {
|
||||
// Postgres compares the copied text exactly, so a key SQLite compared ignoring case or trailing spaces
|
||||
// (`COLLATE NOCASE`, `RTRIM`) is created, or kept as the primary key, but enforces less than it did.
|
||||
if (index.collations.length > 0) {
|
||||
notes.push(
|
||||
`the collation ${index.collations.join(', ')} of the ${index.origin === 'pk' ? 'primary key' : `index ${index.name}`} ` +
|
||||
`on ${table} (Postgres compares its values exactly)`,
|
||||
);
|
||||
}
|
||||
if (index.origin === 'pk') return null;
|
||||
if (index.partial || index.columns === null) {
|
||||
notes.push(`the index ${index.name} on ${table} (partial or over an expression)`);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { nulFreeJson, stripNul } from '../../../nul-free';
|
||||
import { stripNul } from '../../../nul-free';
|
||||
import { InvalidUtf8Text } from '../../sqlite/sqlite-source';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
@@ -28,14 +29,17 @@ import { createHash } from 'node:crypto';
|
||||
* that changed a value fails the verification instead of passing it.
|
||||
*
|
||||
* The SQLite side is read with better-sqlite3's safe integers, so an integer
|
||||
* arrives as a `bigint` and keeps all 64 bits.
|
||||
* arrives as a `bigint` and keeps all 64 bits. A TEXT value whose bytes are
|
||||
* not valid UTF-8 arrives as an {@link InvalidUtf8Text} and is refused under
|
||||
* either set of rules: Postgres text cannot hold those bytes, and reading
|
||||
* them as text would replace them without a trace.
|
||||
*/
|
||||
|
||||
/** The Postgres column types the move writes, by what they hold. */
|
||||
export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes';
|
||||
|
||||
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */
|
||||
export type SqliteValue = null | bigint | number | string | Uint8Array;
|
||||
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT (valid UTF-8 or not) or BLOB. */
|
||||
export type SqliteValue = null | bigint | number | string | InvalidUtf8Text | Uint8Array;
|
||||
|
||||
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
|
||||
export type PostgresValue = null | string | number | Buffer;
|
||||
@@ -100,6 +104,7 @@ function refuse(reason: string): Converted {
|
||||
|
||||
function describe(value: SqliteValue): string {
|
||||
if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`;
|
||||
if (value instanceof InvalidUtf8Text) return describe(value.text);
|
||||
if (typeof value === 'string')
|
||||
return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`;
|
||||
return `the value ${String(value)}`;
|
||||
@@ -120,8 +125,21 @@ function toInteger(value: Exclude<SqliteValue, null>): Converted {
|
||||
return { ok: true, value: whole.toString() };
|
||||
}
|
||||
|
||||
/**
|
||||
* A double as the parameter Postgres takes. Negative zero is sent as text:
|
||||
* `pg` spells the number -0 as "0", which would drop its sign.
|
||||
*/
|
||||
function floatParameter(value: number): PostgresValue {
|
||||
return Object.is(value, -0) ? '-0' : value;
|
||||
}
|
||||
|
||||
/** A double as the compared text, negative zero kept apart from zero as Postgres prints it. */
|
||||
function floatText(value: number): string {
|
||||
return Object.is(value, -0) ? '-0' : String(value);
|
||||
}
|
||||
|
||||
function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
if (typeof value === 'number') return { ok: true, value };
|
||||
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
|
||||
if (typeof value === 'bigint') {
|
||||
const number = Number(value);
|
||||
if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`);
|
||||
@@ -129,7 +147,7 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
}
|
||||
if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) {
|
||||
const number = Number(value.trim());
|
||||
if (Number.isFinite(number)) return { ok: true, value: number };
|
||||
if (Number.isFinite(number)) return { ok: true, value: floatParameter(number) };
|
||||
}
|
||||
return refuse(`${describe(value)} is not a number`);
|
||||
}
|
||||
@@ -154,6 +172,15 @@ function jsonHoldsNul(value: unknown): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON text without its `\u0000` escapes, everything else exactly as written,
|
||||
* so dropping a NUL changes no number or spelling around it. Each escape is
|
||||
* read as a whole, so an escaped backslash followed by `u0000` stays.
|
||||
*/
|
||||
function dropJsonNul(text: string): string {
|
||||
return text.replace(/\\(?:u0000|[\s\S])/g, (escape) => (escape === '\\u0000' ? '' : escape));
|
||||
}
|
||||
|
||||
function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted {
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
|
||||
@@ -166,7 +193,7 @@ function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted
|
||||
// Valid JSON holds a NUL only escaped, which jsonb refuses as well.
|
||||
if (!jsonHoldsNul(parsed)) return { ok: true, value };
|
||||
if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`);
|
||||
return { ok: true, value: JSON.stringify(nulFreeJson(parsed)), droppedNul: true };
|
||||
return { ok: true, value: dropJsonNul(value), droppedNul: true };
|
||||
}
|
||||
|
||||
function toBytes(value: Exclude<SqliteValue, null>): Converted {
|
||||
@@ -188,7 +215,7 @@ const KIND_HOLDS: Record<ColumnKind, string> = {
|
||||
/** The SQLite storage class of a value, for a refusal. */
|
||||
function storageClass(value: Exclude<SqliteValue, null>): string {
|
||||
if (value instanceof Uint8Array) return 'a BLOB';
|
||||
if (typeof value === 'string') return 'TEXT';
|
||||
if (typeof value === 'string' || value instanceof InvalidUtf8Text) return 'TEXT';
|
||||
return typeof value === 'bigint' ? 'an INTEGER' : 'a REAL';
|
||||
}
|
||||
|
||||
@@ -213,7 +240,7 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
|
||||
}
|
||||
return { ok: true, value: value.toString() };
|
||||
case 'float':
|
||||
if (typeof value === 'number') return { ok: true, value };
|
||||
if (typeof value === 'number') return { ok: true, value: floatParameter(value) };
|
||||
if (typeof value !== 'bigint') return changed();
|
||||
return value > SAFE_MAX || value < -SAFE_MAX
|
||||
? refuse(`${describe(value)} has more digits than a double holds`)
|
||||
@@ -231,6 +258,11 @@ function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted
|
||||
/** `value` as the Postgres column of `kind` takes it under `rules`, or the reason it cannot hold it exactly. */
|
||||
export function convertValue(value: SqliteValue, kind: ColumnKind, rules: ConvertRules = TREK_RULES): Converted {
|
||||
if (value === null) return { ok: true, value: null };
|
||||
if (value instanceof InvalidUtf8Text) {
|
||||
return refuse(
|
||||
`${describe(value)} holds bytes that are not valid UTF-8 (${hex(value.bytes)}), which Postgres text cannot hold`,
|
||||
);
|
||||
}
|
||||
if (rules.exact) return exactly(value, kind);
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
@@ -291,17 +323,19 @@ export function canonicalFromSource(
|
||||
): string | null {
|
||||
if (value === null) return null;
|
||||
if (value instanceof Uint8Array) return kind === 'bytes' ? hex(value) : `blob:${hex(value)}`;
|
||||
if (value instanceof InvalidUtf8Text) return `invalid-utf8:${hex(value.bytes)}`;
|
||||
const dropNul = rules.dropNul && !rules.exact;
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
return typeof value === 'bigint' ? value.toString() : BigInt(String(value).trim()).toString();
|
||||
case 'float':
|
||||
return typeof value === 'bigint' ? value.toString() : String(Number(String(value).trim()));
|
||||
if (typeof value === 'number') return floatText(value);
|
||||
return typeof value === 'bigint' ? value.toString() : floatText(Number(value.trim()));
|
||||
case 'text':
|
||||
return typeof value === 'string' && dropNul ? stripNul(value) : String(value);
|
||||
case 'json':
|
||||
if (typeof value !== 'string') return stableJson(String(value));
|
||||
return stableJson(dropNul ? JSON.stringify(nulFreeJson(JSON.parse(value))) : value);
|
||||
return stableJson(dropNul ? dropJsonNul(value) : value);
|
||||
case 'bytes':
|
||||
return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`;
|
||||
}
|
||||
@@ -317,7 +351,7 @@ export function canonicalFromPostgres(text: string | null, kind: ColumnKind): st
|
||||
case 'integer':
|
||||
return BigInt(text).toString();
|
||||
case 'float':
|
||||
return String(Number(text));
|
||||
return floatText(Number(text));
|
||||
case 'text':
|
||||
return text;
|
||||
case 'json':
|
||||
|
||||
@@ -9,9 +9,58 @@ import fs from 'node:fs';
|
||||
* whole move sees the file as it was at one moment.
|
||||
*
|
||||
* Integers come back as `bigint` (better-sqlite3's safe integers), so a value
|
||||
* past 2^53 keeps every digit.
|
||||
* past 2^53 keeps every digit. Text is read as its bytes and decoded here, so
|
||||
* a value whose bytes are not valid UTF-8 (which SQLite stores as it is
|
||||
* given) arrives as an {@link InvalidUtf8Text} instead of with its bytes
|
||||
* silently replaced by U+FFFD.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A TEXT value whose bytes are not valid text in the file's encoding (UTF-8,
|
||||
* or UTF-16 for a file created so): the bytes the file holds, and the text
|
||||
* with each bad sequence as U+FFFD.
|
||||
*/
|
||||
export class InvalidUtf8Text {
|
||||
constructor(
|
||||
readonly bytes: Buffer,
|
||||
readonly text: string,
|
||||
) {}
|
||||
|
||||
/** The text, as a refusal names a row by it. */
|
||||
toString(): string {
|
||||
return this.text;
|
||||
}
|
||||
}
|
||||
|
||||
/** The decoder label of each text encoding a SQLite file can have (`pragma encoding`). */
|
||||
const ENCODINGS: Record<string, string> = { 'UTF-8': 'utf-8', 'UTF-16le': 'utf-16le', 'UTF-16be': 'utf-16be' };
|
||||
|
||||
/**
|
||||
* Decodes the TEXT values of a file in its encoding: a TEXT value cast to a
|
||||
* BLOB is its bytes in that encoding. Strict, so bytes that are not valid
|
||||
* text throw, and a leading byte order mark is kept as the text holds it.
|
||||
*/
|
||||
class TextReader {
|
||||
private readonly strict: TextDecoder;
|
||||
private readonly lenient: TextDecoder;
|
||||
|
||||
constructor(encoding: string) {
|
||||
const label = ENCODINGS[encoding];
|
||||
if (!label) throw new Error(`The SQLite file has the text encoding ${encoding}, which TREK cannot read.`);
|
||||
this.strict = new TextDecoder(label, { fatal: true, ignoreBOM: true });
|
||||
this.lenient = new TextDecoder(label, { ignoreBOM: true });
|
||||
}
|
||||
|
||||
/** A TEXT value from its bytes. */
|
||||
decode(bytes: Uint8Array): string | InvalidUtf8Text {
|
||||
try {
|
||||
return this.strict.decode(bytes);
|
||||
} catch {
|
||||
return new InvalidUtf8Text(Buffer.from(bytes), this.lenient.decode(bytes));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A column as `pragma_table_info` describes it. */
|
||||
export interface SqliteColumn {
|
||||
name: string;
|
||||
@@ -33,6 +82,8 @@ export interface SqliteIndex {
|
||||
partial: boolean;
|
||||
/** The indexed columns in order; null when one of them is an expression. */
|
||||
columns: string[] | null;
|
||||
/** The collating sequences its keys compare with other than SQLite's default `BINARY` (`NOCASE`, `RTRIM`). */
|
||||
collations: string[];
|
||||
}
|
||||
|
||||
/** A row whose foreign key points at nothing, from `pragma_foreign_key_check`. */
|
||||
@@ -51,10 +102,14 @@ function quoteSqliteIdentifier(name: string): string {
|
||||
type SqliteHandle = ReturnType<typeof openDatabase>;
|
||||
|
||||
export class SqliteSource {
|
||||
private readonly text: TextReader;
|
||||
|
||||
private constructor(
|
||||
private readonly db: SqliteHandle,
|
||||
readonly file: string,
|
||||
) {}
|
||||
) {
|
||||
this.text = new TextReader(String(db.pragma('encoding', { simple: true })));
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens `file` read-only and starts the read transaction. A file that does
|
||||
@@ -141,10 +196,11 @@ export class SqliteSource {
|
||||
}>;
|
||||
return list.map((index) => {
|
||||
const keys = this.db
|
||||
.prepare('select cid, name from pragma_index_info(?) order by seqno')
|
||||
.prepare('select cid, name, coll from pragma_index_xinfo(?) where key = 1 order by seqno')
|
||||
.all(index.name) as Array<{
|
||||
cid: bigint;
|
||||
name: string | null;
|
||||
coll: string;
|
||||
}>;
|
||||
const columns = keys.every((key) => key.cid >= 0n && key.name !== null)
|
||||
? keys.map((key) => String(key.name))
|
||||
@@ -155,6 +211,9 @@ export class SqliteSource {
|
||||
origin: index.origin,
|
||||
partial: index.partial !== 0n,
|
||||
columns,
|
||||
collations: [
|
||||
...new Set(keys.map((key) => key.coll.toUpperCase()).filter((collation) => collation !== 'BINARY')),
|
||||
],
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -180,13 +239,28 @@ export class SqliteSource {
|
||||
);
|
||||
}
|
||||
|
||||
/** The rows of `table`, one array of `columns` per row, read one at a time. */
|
||||
rows(table: string, columns: readonly string[]): IterableIterator<unknown[]> {
|
||||
const list = columns.map(quoteSqliteIdentifier).join(', ');
|
||||
return this.db
|
||||
/**
|
||||
* The rows of `table`, one array of `columns` per row, read one at a time.
|
||||
* Each column is read with its storage class, and TEXT as its bytes, which
|
||||
* decoded in the file's encoding into a string or an {@link InvalidUtf8Text}.
|
||||
*/
|
||||
*rows(table: string, columns: readonly string[]): IterableIterator<unknown[]> {
|
||||
const list = columns
|
||||
.map(quoteSqliteIdentifier)
|
||||
.map(
|
||||
(column) =>
|
||||
`typeof(${column}), case typeof(${column}) when 'text' then cast(${column} as blob) else ${column} end`,
|
||||
)
|
||||
.join(', ');
|
||||
const rows = this.db
|
||||
.prepare(`select ${list} from ${quoteSqliteIdentifier(table)}`)
|
||||
.raw(true)
|
||||
.iterate() as IterableIterator<unknown[]>;
|
||||
for (const raw of rows) {
|
||||
yield columns.map((_column, index) =>
|
||||
raw[index * 2] === 'text' ? this.text.decode(raw[index * 2 + 1] as Uint8Array) : raw[index * 2 + 1],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** The highest id each AUTOINCREMENT table ever handed out, which can be above the largest id left. */
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
/**
|
||||
* PGMOVENUL-001..004: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
||||
* PGMOVENUL-001..005: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
||||
* changes nothing silently, on the lane's server. A plugin value holding a NUL
|
||||
* refuses the move before anything is written; a NUL in TREK's own text
|
||||
* refuses it unless `dropNul` (`--drop-nul`) is given; a unique key that
|
||||
* dropping a NUL breaks is named as that, with the constraint Postgres
|
||||
* reports; and a move with `dropNul` lists every value it changed and stores
|
||||
* the value TREK would have written.
|
||||
* or bytes that are not UTF-8 refuses the move before anything is written; a
|
||||
* NUL in TREK's own text refuses it unless `dropNul` (`--drop-nul`) is given;
|
||||
* a unique key that dropping a NUL breaks is named as that, with the
|
||||
* constraint Postgres reports; a move with `dropNul` lists every value it
|
||||
* changed and stores the value TREK would have written, JSON with nothing
|
||||
* else changed; and TREK's own text that is not UTF-8 is refused either way.
|
||||
*/
|
||||
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
||||
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
||||
@@ -101,7 +102,7 @@ beforeAll(async () => {
|
||||
const plugin = new Database(path.join(dir, 'plugins-data', 'nul-notes', 'plugin.db'));
|
||||
plugin.exec(
|
||||
'create table notes (k text primary key, v text);' +
|
||||
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two');",
|
||||
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two'), ('bytes', cast(x'ff41' as text));",
|
||||
);
|
||||
plugin.close();
|
||||
await emptyTarget();
|
||||
@@ -114,11 +115,14 @@ afterAll(async () => {
|
||||
});
|
||||
|
||||
describe('the move changes nothing silently', () => {
|
||||
it("PGMOVENUL-001: a plugin's NUL refuses the move before anything is written", async () => {
|
||||
it("PGMOVENUL-001: a plugin's NUL or bytes that are not UTF-8 refuse the move before anything is written", async () => {
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
/1 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
|
||||
/2 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
|
||||
);
|
||||
await expect(failure).rejects.toThrow(
|
||||
'plugin nul-notes: notes row k=bytes, column v: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
fs.rmSync(path.join(dir, 'plugins-data'), { recursive: true, force: true });
|
||||
@@ -149,11 +153,33 @@ describe('the move changes nothing silently', () => {
|
||||
|
||||
it('PGMOVENUL-004: with --drop-nul and no collision, each changed value is listed and stored without the NUL', async () => {
|
||||
handle.exec(`update users set username = 'mover' || char(0) || 'b' where id = ${movedId}`);
|
||||
// JSON loses the escaped NUL and nothing else: the digits of a large number and the spelling 1.0 stay.
|
||||
handle.exec(
|
||||
`update addons set config = '{"note":"a\\u0000b","big":12345678901234567890,"ratio":1.0}' where id = 'collab'`,
|
||||
);
|
||||
const summary = await move(true);
|
||||
expect(summary.nulDropped).toEqual([`users row id=${movedId}, column username`]);
|
||||
expect([...summary.nulDropped].sort()).toEqual([
|
||||
'addons row id=collab, column config',
|
||||
`users row id=${movedId}, column username`,
|
||||
]);
|
||||
const stored = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
||||
run<{ username: string }>('select username from users where id = ?', [movedId]),
|
||||
);
|
||||
expect(stored).toEqual([{ username: 'moverb' }]);
|
||||
const config = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
||||
run<{ config: string }>("select config::text as config from addons where id = 'collab'"),
|
||||
);
|
||||
expect(config).toEqual([{ config: '{"big": 12345678901234567890, "note": "ab", "ratio": 1.0}' }]);
|
||||
});
|
||||
|
||||
it("PGMOVENUL-005: TREK's own text whose bytes are not UTF-8 is refused, with --drop-nul too", async () => {
|
||||
await emptyTarget();
|
||||
handle.exec("insert into app_settings (key, value) values ('move-bytes', cast(x'ff41' as text))");
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
'app_settings row key=move-bytes, column value: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -127,9 +127,10 @@ function seedPluginData(root: string): void {
|
||||
'create unique index notes_body on notes (body);' +
|
||||
"insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" +
|
||||
"delete from notes where body = 'gone';" +
|
||||
// Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them.
|
||||
// Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them,
|
||||
// negative zero (which a column without a type keeps) with its sign.
|
||||
'create table docs (id integer primary key, payload blob, n);' +
|
||||
`insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5);`,
|
||||
`insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5), (3, 'signed', -0.0);`,
|
||||
);
|
||||
plugin.close();
|
||||
}
|
||||
@@ -212,7 +213,7 @@ describe('moving a SQLite install to Postgres', () => {
|
||||
schema: 'trek_plugin_move_notes',
|
||||
tables: [
|
||||
expect.objectContaining({ label: '_plugin_migrations', rows: 1 }),
|
||||
expect.objectContaining({ label: 'docs', rows: 2 }),
|
||||
expect.objectContaining({ label: 'docs', rows: 3 }),
|
||||
expect.objectContaining({ label: 'notes', rows: 2 }),
|
||||
],
|
||||
}),
|
||||
@@ -289,6 +290,7 @@ describe('moving a SQLite install to Postgres', () => {
|
||||
expect(await store.query('select id, payload, n from docs order by id')).toEqual([
|
||||
{ id: 1, payload: '{"a":1}', n: 5 },
|
||||
{ id: 2, payload: 'plain', n: 2.5 },
|
||||
{ id: 3, payload: 'signed', n: -0 },
|
||||
]);
|
||||
// The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out.
|
||||
expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false });
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVE-001..025: the move of a SQLite install into Postgres
|
||||
* MOVE-001..026: the move of a SQLite install into Postgres
|
||||
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
|
||||
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
|
||||
* the refusals before anything is written, the copy in foreign key order with
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
import type { AppEnv } from '../../../../../../src/app-config';
|
||||
import { CLEAR_SCHEMA_SQL } from '../../../../../../src/db/engine/postgres/clear-schema';
|
||||
import { PG_NOW_TEXT_DEFAULT } from '../../../../../../src/db/engine/postgres/schema-adjust';
|
||||
import {
|
||||
runMoveCommand,
|
||||
USAGE,
|
||||
@@ -34,7 +35,11 @@ import {
|
||||
pluginColumnKind,
|
||||
pluginDefault,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data';
|
||||
import { sqliteInstallHoldsAccounts, SqliteSource } from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
||||
import {
|
||||
InvalidUtf8Text,
|
||||
sqliteInstallHoldsAccounts,
|
||||
SqliteSource,
|
||||
} from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
||||
import type { AnyOrm } from '../../../../../../src/db/orm';
|
||||
import { FakePostgres, fakeMoveOrm, type FakeTableDef } from '../../../../../helpers/fake-postgres-move';
|
||||
import { PostgreSqlPlatform } from '@mikro-orm/postgresql';
|
||||
@@ -487,6 +492,15 @@ describe('the pieces', () => {
|
||||
expect(pluginDefault("strftime('%s','now')", 'integer')).toBe(
|
||||
'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)',
|
||||
);
|
||||
// A default the column's Postgres type would refuse, or store as another value, is not carried over.
|
||||
expect(pluginDefault('0', 'bytes')).toBeNull();
|
||||
expect(pluginDefault("'a'", 'bytes')).toBeNull();
|
||||
expect(pluginDefault('1.5', 'integer')).toBeNull();
|
||||
expect(pluginDefault('1.5', 'float')).toBe('1.5');
|
||||
expect(pluginDefault("'abc'", 'integer')).toBeNull();
|
||||
expect(pluginDefault('CURRENT_TIMESTAMP', 'integer')).toBeNull();
|
||||
expect(pluginDefault('CURRENT_TIMESTAMP', 'text')).toBe(PG_NOW_TEXT_DEFAULT);
|
||||
expect(pluginDefault("strftime('%s','now')", 'bytes')).toBeNull();
|
||||
|
||||
expect(batchRows(2)).toBe(500);
|
||||
expect(batchRows(200)).toBe(150);
|
||||
@@ -524,6 +538,57 @@ describe('the pieces', () => {
|
||||
source.close();
|
||||
expect(fs.readFileSync(path.join(dir, 'travel.db')).equals(before)).toBe(true);
|
||||
});
|
||||
|
||||
it('MOVE-026: text is read by its bytes, so bytes that are not UTF-8 are never replaced unseen', () => {
|
||||
const file = sqlite(
|
||||
path.join(dir, 'bytes.db'),
|
||||
'create table t (v);' +
|
||||
"insert into t values (cast(x'ff41' as text)), (cast(x'f09080' as text)), (cast(x'efbbbf41' as text)), " +
|
||||
"(cast(x'efbfbd' as text)), ('a' || char(0) || 'b'), (''), (x''), (7), (2.5), (null);",
|
||||
);
|
||||
const source = SqliteSource.open(file);
|
||||
const values = [...source.rows('t', ['v'])].map(([value]) => value);
|
||||
source.close();
|
||||
expect(values[0]).toEqual(new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA'));
|
||||
expect(values[1]).toBeInstanceOf(InvalidUtf8Text);
|
||||
expect(String(values[1])).toBe('\uFFFD');
|
||||
// A byte order mark and a real U+FFFD are text like any other, kept as they are.
|
||||
expect(values.slice(2)).toEqual(['\uFEFFA', '\uFFFD', 'a\u0000b', '', Buffer.alloc(0), 7n, 2.5, null]);
|
||||
});
|
||||
|
||||
it('MOVE-028: text of a file created as UTF-16 is read in that encoding, its bad sequences refused like bad UTF-8', () => {
|
||||
const file = sqlite(
|
||||
path.join(dir, 'utf16.db'),
|
||||
"pragma encoding = 'UTF-16le'; create table t (v);" +
|
||||
"insert into t values ('h\u00e9llo \u{1F600}'), (cast(x'00d841' || x'00' as text)), ('');",
|
||||
);
|
||||
const source = SqliteSource.open(file);
|
||||
const values = [...source.rows('t', ['v'])].map(([value]) => value);
|
||||
source.close();
|
||||
expect(values[0]).toBe('h\u00e9llo \u{1F600}');
|
||||
expect(values[1]).toEqual(new InvalidUtf8Text(Buffer.from([0x00, 0xd8, 0x41, 0x00]), '\uFFFDA'));
|
||||
expect(values[2]).toBe('');
|
||||
});
|
||||
|
||||
it('MOVE-027: a plugin key SQLite compared with NOCASE or RTRIM is listed, since Postgres compares it exactly', async () => {
|
||||
const pluginDir = path.join(dir, 'plugins-data', 'move-tags');
|
||||
fs.mkdirSync(pluginDir, { recursive: true });
|
||||
sqlite(
|
||||
path.join(pluginDir, 'plugin.db'),
|
||||
`create table tags (name text primary key collate nocase, label text, slug text collate rtrim, note text);
|
||||
create unique index tags_label on tags (label collate nocase, slug);
|
||||
create index tags_note on tags (note);
|
||||
insert into tags values ('Alpha', 'a', 's', 'n');`,
|
||||
);
|
||||
const summary = await moveSqliteToPostgres(options());
|
||||
expect(summary.plugins[0].notCarried).toEqual([
|
||||
'the collation NOCASE, RTRIM of the index tags_label on tags (Postgres compares its values exactly)',
|
||||
'the collation NOCASE of the primary key on tags (Postgres compares its values exactly)',
|
||||
]);
|
||||
const statements = pg.statements.join('\n');
|
||||
expect(statements).toContain('create unique index on "trek_plugin_move_tags"."tags" ("label", "slug")');
|
||||
expect(statements).toContain('create index on "trek_plugin_move_tags"."tags" ("note")');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runMoveCommand', () => {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVEVAL-001..014: how a SQLite value becomes the value a Postgres column of
|
||||
* MOVEVAL-001..017: how a SQLite value becomes the value a Postgres column of
|
||||
* the move's target takes, under TREK's rules and under a plugin's, the text
|
||||
* both sides are compared in, and the order-free table checksum
|
||||
* (src/db/engine/postgres/sqlite-move/values.ts).
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
TableChecksum,
|
||||
type ColumnKind,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/values';
|
||||
import { InvalidUtf8Text } from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
@@ -262,3 +263,35 @@ describe("a plugin's values (PLUGIN_RULES)", () => {
|
||||
expect(canonicalFromSource('{"k\\u0000":1}', 'json', DROP_NUL)).toBe(canonicalFromPostgres('{"k": 1}', 'json'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('values a conversion could change unseen', () => {
|
||||
it('MOVEVAL-015: negative zero keeps its sign on the way and in the compared text', () => {
|
||||
// `pg` spells the number -0 as "0", so it is sent as text.
|
||||
expect(convertValue(-0, 'float')).toEqual(ok('-0'));
|
||||
expect(convertValue('-0', 'float')).toEqual(ok('-0'));
|
||||
expect(convertValue(-0, 'float', PLUGIN_RULES)).toEqual(ok('-0'));
|
||||
expect(convertValue(0, 'float', PLUGIN_RULES)).toEqual(ok(0));
|
||||
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).toBe(canonicalFromPostgres('-0', 'float'));
|
||||
expect(canonicalFromSource('-0', 'float')).toBe(canonicalFromPostgres('-0', 'float'));
|
||||
// A zero that lost its sign on the way fails the verification.
|
||||
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('0', 'float'));
|
||||
});
|
||||
|
||||
it('MOVEVAL-016: dropping a NUL from JSON changes nothing else in it', () => {
|
||||
const written = '{"a\\u0000":"x\\u0000y","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
|
||||
const dropped = '{"a":"xy","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
|
||||
expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true });
|
||||
expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json'));
|
||||
});
|
||||
|
||||
it('MOVEVAL-017: text whose bytes are not UTF-8 is refused under either rules, for every column kind', () => {
|
||||
const invalid = new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA');
|
||||
const reason = 'the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41), which Postgres text cannot hold';
|
||||
for (const kind of ['integer', 'float', 'text', 'json', 'bytes'] as const) {
|
||||
expect(convertValue(invalid, kind)).toEqual({ ok: false, reason });
|
||||
expect(convertValue(invalid, kind, DROP_NUL)).toEqual({ ok: false, reason });
|
||||
expect(convertValue(invalid, kind, PLUGIN_RULES)).toEqual({ ok: false, reason });
|
||||
}
|
||||
expect(canonicalFromSource(invalid, 'text')).not.toBe(canonicalFromPostgres('\uFFFDA', 'text'));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user