From bc6728678b16014e1facc26588976d3fbfaf11af Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 23:28:01 +0100 Subject: [PATCH] fix(db): let the move to Postgres change nothing silently The move stripped NUL from a plugin's own text, merging keys that were distinct, and turned a plugin's values into another type (text in a BLOB column into bytea, numbers in an untyped column into text). Its checksum was built from the converted values, so both passed as verified. A plugin's data now moves exactly as the Postgres plugin store would hand it back, or not at all: a NUL, a value of another storage class than its column, or an integer past 2^53 refuses the move before anything is written, listing plugin, table, row and column. A column left open by SQLite's affinity takes the kind its values need, so text in a BLOB column stays text. TREK's own text holding a NUL from an older install is refused by default too. --drop-nul drops it, lists every changed value in the summary, and names a unique key that dropping a NUL broke. Every table's checksum is now built from the values read from SQLite, so a conversion that changes a value fails the verification. --- .../db/engine/postgres/sqlite-move/catalog.ts | 24 ++- .../db/engine/postgres/sqlite-move/command.ts | 16 +- .../db/engine/postgres/sqlite-move/copy.ts | 106 ++++++++++-- .../db/engine/postgres/sqlite-move/move.ts | 88 ++++++++-- .../postgres/sqlite-move/plugin-data.ts | 28 ++- .../db/engine/postgres/sqlite-move/values.ts | 161 +++++++++++++++--- server/tests/helpers/fake-postgres-move.ts | 14 +- .../engine/postgres/sqlite-move/move.test.ts | 155 ++++++++++++++++- .../engine/postgres/sqlite-move/order.test.ts | 34 +++- .../postgres/sqlite-move/values.test.ts | 126 +++++++++++--- 10 files changed, 655 insertions(+), 97 deletions(-) diff --git a/server/src/db/engine/postgres/sqlite-move/catalog.ts b/server/src/db/engine/postgres/sqlite-move/catalog.ts index 7881cea0a..11c6460fb 100644 --- a/server/src/db/engine/postgres/sqlite-move/catalog.ts +++ b/server/src/db/engine/postgres/sqlite-move/catalog.ts @@ -23,6 +23,8 @@ interface TargetTable { name: string; columns: TargetColumn[]; primaryKey: string[]; + /** Every unique index (the primary key's among them) over plain columns, by name. */ + uniqueKeys: Array<{ name: string; columns: string[] }>; } export interface TargetSchema { @@ -57,9 +59,24 @@ export async function readTargetSchema(trx: Trx): Promise { "and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name", ); + // A unique constraint is backed by an index of its name, so the index names what a refusal reports. + const uniqueIndexes = await run<{ table_name: string; index_name: string; columns: string[] }>( + trx, + 'select t.relname as table_name, i.relname as index_name, array_agg(a.attname::text order by k.ord) as columns ' + + 'from pg_index x join pg_class i on i.oid = x.indexrelid join pg_class t on t.oid = x.indrelid ' + + 'cross join lateral unnest(x.indkey) with ordinality as k(attnum, ord) ' + + 'join pg_attribute a on a.attrelid = x.indrelid and a.attnum = k.attnum ' + + 'where x.indisunique and t.relnamespace = current_schema()::regnamespace group by t.relname, i.relname', + ); + const tables = new Map(); for (const column of columns) { - const table = tables.get(column.table_name) ?? { name: column.table_name, columns: [], primaryKey: [] }; + const table = tables.get(column.table_name) ?? { + name: column.table_name, + columns: [], + primaryKey: [], + uniqueKeys: [], + }; table.columns.push({ name: column.column_name, dataType: column.data_type, @@ -67,6 +84,9 @@ export async function readTargetSchema(trx: Trx): Promise { }); tables.set(column.table_name, table); } + for (const index of uniqueIndexes) { + tables.get(index.table_name)?.uniqueKeys.push({ name: index.index_name, columns: index.columns }); + } const foreignKeys: ForeignKey[] = []; for (const constraint of constraints) { if (constraint.kind === 'p') { @@ -154,6 +174,8 @@ export function planTrekTables(target: TargetSchema, source: SourceTables): Trek columns, deferred, primaryKey: table.primaryKey, + exact: false, + uniqueKeys: table.uniqueKeys, }); } const sequences = target.sequences diff --git a/server/src/db/engine/postgres/sqlite-move/command.ts b/server/src/db/engine/postgres/sqlite-move/command.ts index 1d461dada..dcc46b08a 100644 --- a/server/src/db/engine/postgres/sqlite-move/command.ts +++ b/server/src/db/engine/postgres/sqlite-move/command.ts @@ -14,12 +14,18 @@ import path from 'node:path'; * The command around the move (`src/move-to-postgres.ts` runs it): reads * TREK_DB_URL as the server does, takes the SQLite file from the data * directory or `--from=`, the plugin data from the plugins' data - * directory, runs the move and prints its summary. Answers the exit code. + * directory, runs the move and prints its summary. `--drop-nul` lets the move + * drop NUL characters from TREK's own text instead of refusing them. Answers + * the exit code. */ export const USAGE = - 'Usage: move-to-postgres [--from=]\n' + - 'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.'; + 'Usage: move-to-postgres [--from=] [--drop-nul]\n' + + 'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.\n' + + ' --from= the SQLite file to copy\n' + + " --drop-nul drop NUL characters (U+0000) from TREK's own text, which Postgres cannot hold and TREK drops on\n" + + ' its next write anyway, instead of refusing the move; the summary lists every value changed.\n' + + " A plugin's own data is never changed: a NUL there refuses the move either way."; export interface MoveCommandIo { env: () => AppEnv; @@ -51,13 +57,14 @@ const defaultIo: MoveCommandIo = { }; const FROM = '--from='; +const DROP_NUL = '--drop-nul'; async function run(argv: readonly string[], io: MoveCommandIo): Promise { if (argv.includes('--help') || argv.includes('-h')) { io.out(USAGE); return 0; } - const unknown = argv.find((arg) => !arg.startsWith(FROM)); + const unknown = argv.find((arg) => !arg.startsWith(FROM) && arg !== DROP_NUL); if (unknown !== undefined) throw new MoveRefusal(`Unknown argument ${unknown}.\n${USAGE}`); const env = io.env(); if (env.db.engine !== 'postgres') @@ -77,6 +84,7 @@ async function run(argv: readonly string[], io: MoveCommandIo): Promise known: knownMigrationNames(), baseline: readPostgresBaseline, log: (line) => io.out(` ${line}`), + dropNul: argv.includes(DROP_NUL), }); io.out(''); io.out(renderSummary(summary, target)); diff --git a/server/src/db/engine/postgres/sqlite-move/copy.ts b/server/src/db/engine/postgres/sqlite-move/copy.ts index d454c321d..5f6c79f52 100644 --- a/server/src/db/engine/postgres/sqlite-move/copy.ts +++ b/server/src/db/engine/postgres/sqlite-move/copy.ts @@ -2,10 +2,13 @@ import type { SqliteSource } from '../../sqlite/sqlite-source'; import { beyondSafeInteger, canonicalFromPostgres, - canonicalFromSent, + canonicalFromSource, convertValue, + NUL_REASON, + PLUGIN_RULES, TableChecksum, type ColumnKind, + type ConvertRules, type PostgresValue, type SqliteValue, } from './values'; @@ -17,7 +20,7 @@ import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; * inside the move's transaction. The same steps serve TREK's own tables and a * plugin's: read the rows one at a time, convert each value to its column's * type, insert in batches, then read the table back and compare its row count - * and checksum with what was sent. + * and checksum with the rows read from the SQLite file. */ export type Trx = Transaction; @@ -29,6 +32,8 @@ const MAX_BATCH_PARAMS = 30_000; const FETCH_ROWS = 2_000; /** The most refused values a refusal lists one by one. */ export const MAX_LISTED_PROBLEMS = 20; +/** Postgres's SQLSTATE for a row that would break a unique key. */ +const UNIQUE_VIOLATION = '23505'; /** One column of a table the move copies. */ interface CopyColumn { @@ -49,6 +54,17 @@ export interface TableSpec { deferred: string[]; /** The primary key's columns, which the deferred update finds a row by. */ primaryKey: string[]; + /** A plugin's own table, whose values must arrive exactly as stored (`PLUGIN_RULES` in `values.ts`). */ + exact: boolean; + /** The unique keys of the Postgres table (the primary key among them), by constraint or index name. */ + uniqueKeys: Array<{ name: string; columns: string[] }>; +} + +/** A value from which `--drop-nul` removed NUL characters, for the summary. */ +interface DroppedNul { + /** The table, row and column, named as a refusal names a value. */ + where: string; + column: string; } /** A row's deferred values, kept until every table is in. */ @@ -65,6 +81,10 @@ export interface CopiedTable { /** Values the Postgres column cannot hold exactly, listed up to `MAX_LISTED_PROBLEMS`; the move refuses on any. */ problems: string[]; problemCount: number; + /** How many of the refused values hold a NUL character. */ + nulRefused: number; + /** Every value `--drop-nul` changed. */ + nulDropped: DroppedNul[]; /** Integer values beyond 2^53, which TREK's Postgres driver refuses to read back. */ unsafeIntegers: number; deferredRows: DeferredRow[]; @@ -120,26 +140,64 @@ export function insertSql(spec: Pick, rows: num return `insert into ${spec.target} (${names}) values ${tuples.join(', ')}`; } -async function insertBatch(trx: Trx, spec: TableSpec, batch: PostgresValue[][]): Promise { +/** Lines as a refusal lists them, and how many more there are when `total` is above what is listed. */ +export function listLines(lines: readonly string[], total = lines.length): string { + const shown = lines.map((line) => ` - ${line}`); + if (total > lines.length) shown.push(` - and ${total - lines.length} more`); + return shown.join('\n'); +} + +/** + * Why Postgres refused rows of a table. A unique key broken in a table whose + * values lost NUL characters to `--drop-nul` is named as that: two values + * that differed only by a NUL are equal once it is gone, which the SQLite + * file's own key never saw. + */ +export function refusedRowsMessage(spec: TableSpec, err: unknown, dropped: readonly DroppedNul[]): string { + const message = `Postgres refused rows of ${spec.label}: ${describePgError(err)}`; + const { code, constraint } = (err ?? {}) as { code?: unknown; constraint?: unknown }; + if (code !== UNIQUE_VIOLATION || dropped.length === 0) return message; + const key = spec.uniqueKeys.find((candidate) => candidate.name === constraint); + const suspects = key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped; + if (suspects.length === 0) return message; + const where = key ? `its column(s) ${key.columns.join(', ')}` : spec.label; + return ( + `${message}\nThe key is broken because --drop-nul removed NUL characters (U+0000) from ${suspects.length} ` + + `value(s) of ${where}, and a value that differed from another only by a NUL equals it once the NUL is gone:\n` + + `${listLines( + suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where), + suspects.length, + )}\n` + + 'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.' + ); +} + +async function insertBatch( + trx: Trx, + spec: TableSpec, + batch: PostgresValue[][], + dropped: readonly DroppedNul[], +): Promise { if (batch.length === 0) return; try { await run(trx, insertSql(spec, batch.length), batch.flat()); } catch (err) { - throw new Error(`Postgres refused rows of ${spec.label}: ${describePgError(err)}`, { cause: err }); + throw new Error(refusedRowsMessage(spec, err, dropped), { cause: err }); } } /** * Copies the rows of one table. Every value is converted first; once one * cannot be, nothing more is inserted, but the rest of the table is still read - * so the refusal can list every problem at once. With `insert: false` it only - * reads and converts, once an earlier table was refused. + * so the refusal can list every problem at once. Without a transaction, or + * with `insert: false` (once an earlier table was refused), it only reads and + * converts, which is how the checks before anything is written run it. */ export async function copyTable( - trx: Trx, + trx: Trx | null, source: SqliteSource, spec: TableSpec, - options: { insert: boolean } = { insert: true }, + options: { insert?: boolean; dropNul?: boolean } = {}, ): Promise { const result: CopiedTable = { spec, @@ -147,18 +205,26 @@ export async function copyTable( checksum: '', problems: [], problemCount: 0, + nulRefused: 0, + nulDropped: [], unsafeIntegers: 0, deferredRows: [], }; + const rules: ConvertRules = spec.exact ? PLUGIN_RULES : { exact: false, dropNul: options.dropNul === true }; const checksum = new TableChecksum(); const deferredAt = spec.deferred.map((name) => spec.columns.findIndex((column) => column.name === name)); const keyAt = spec.primaryKey.map((name) => spec.columns.findIndex((column) => column.name === name)); const size = batchRows(spec.columns.length); - /** How a refusal names a row: by its primary key, or by its place in the table without one. */ + /** + * How a refusal names a row: by its primary key, or by its place in the + * table without one. A NUL in a key is spelled `\u0000`, so the line can be + * read and copied. + */ const rowName = (raw: unknown[]): string => { if (keyAt.length === 0 || keyAt.includes(-1)) return `row ${result.rows}`; - return `row ${spec.primaryKey.map((name, index) => `${name}=${String(raw[keyAt[index]])}`).join(', ')}`; + const key = (index: number) => String(raw[keyAt[index]]).replaceAll('\u0000', '\\u0000'); + return `row ${spec.primaryKey.map((name, index) => `${name}=${key(index)}`).join(', ')}`; }; /** One SQLite row as the values sent to Postgres, or null once a value was refused. */ @@ -166,21 +232,25 @@ export async function copyTable( result.rows++; const values: PostgresValue[] = []; spec.columns.forEach((column, index) => { - const converted = convertValue(raw[index] as SqliteValue, column.kind); + const converted = convertValue(raw[index] as SqliteValue, column.kind, rules); + const where = `${spec.label} ${rowName(raw)}, column ${column.name}`; // Narrowed by `in`: the server compiles without strictNullChecks, where `ok` does not narrow the union. if (!('reason' in converted)) { values.push(converted.value); if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++; + if (converted.droppedNul) result.nulDropped.push({ where, column: column.name }); return; } values.push(null); result.problemCount++; - if (result.problems.length < MAX_LISTED_PROBLEMS) { - result.problems.push(`${spec.label} ${rowName(raw)}, column ${column.name}: ${converted.reason}`); - } + if (converted.reason.includes(NUL_REASON)) result.nulRefused++; + if (result.problems.length < MAX_LISTED_PROBLEMS) result.problems.push(`${where}: ${converted.reason}`); }); if (result.problemCount > 0) return null; - checksum.add(values.map((value, index) => canonicalFromSent(value, spec.columns[index].kind))); + // From the SQLite values, not the converted ones, so a conversion that changed a value cannot verify itself. + checksum.add( + spec.columns.map((column, index) => canonicalFromSource(raw[index] as SqliteValue, column.kind, rules)), + ); if (deferredAt.length > 0) { const later = deferredAt.map((index) => values[index]); if (later.some((value) => value !== null)) { @@ -209,7 +279,9 @@ export async function copyTable( const pump = async (): Promise => { const { batch, done } = nextBatch(); // Always awaited, so the next batch starts on a fresh stack even when nothing is inserted. - await (options.insert && result.problemCount === 0 ? insertBatch(trx, spec, batch) : Promise.resolve()); + await (trx && options.insert !== false && result.problemCount === 0 + ? insertBatch(trx, spec, batch, result.nulDropped) + : Promise.resolve()); if (!done) await pump(); }; try { @@ -243,7 +315,7 @@ export async function applyDeferred(trx: Trx, copied: CopiedTable): Promise { const checksum = new TableChecksum(); diff --git a/server/src/db/engine/postgres/sqlite-move/move.ts b/server/src/db/engine/postgres/sqlite-move/move.ts index 79687cb6b..7fcdaaa69 100644 --- a/server/src/db/engine/postgres/sqlite-move/move.ts +++ b/server/src/db/engine/postgres/sqlite-move/move.ts @@ -11,6 +11,7 @@ import { applyDeferred, copyTable, inOrder, + listLines, MAX_LISTED_PROBLEMS, readBack, run, @@ -36,18 +37,27 @@ import type { Kysely } from 'kysely'; * 1. The SQLite file must be at this build's migration level: an older one is * migrated by starting this TREK on it once, a newer one belongs to the * TREK that wrote it. Nothing else is checked by guess. - * 2. The Postgres database must hold no table and no plugin schema. - * 3. The schema is created exactly as a first boot creates it (the engine's + * 2. Every plugin's data must move unchanged (`PLUGIN_RULES` in `values.ts`): + * a value Postgres cannot hand back to the plugin as it is (a NUL, a + * column whose values mix types) is listed, before anything is written. + * 3. The Postgres database must hold no table and no plugin schema. + * 4. The schema is created exactly as a first boot creates it (the engine's * bootstrap: the baseline and the migrations after its cut), without the * seeders, since every row comes from the file. - * 4. In one transaction: the rows the bootstrap seeded are cleared, every + * 5. In one transaction: the rows the bootstrap seeded are cleared, every * table is filled in foreign key order, each plugin's `plugin.db` goes into * its schema, the sequences are moved past the copied ids, and every table - * is read back and compared with what was sent (row count and a checksum - * over all rows). Only a copy that matches is committed. - * 5. Anything that fails leaves the database as it was found: the + * is read back and compared with the SQLite file (row count and a checksum + * over all rows, built from the values read from SQLite). Only a copy that + * matches is committed. + * 6. Anything that fails leaves the database as it was found: the * transaction rolls back and the schema the bootstrap created is dropped. * + * Nothing is changed silently. A NUL character in TREK's own text, which TREK + * no longer stores and drops on its next write, refuses the move unless the + * operator passes `dropNul` (`--drop-nul`); then each value that lost one is + * listed in the summary. + * * The SQLite files are opened read-only and never written. Uploads are not * the database's: they stay where the storage driver keeps them. */ @@ -67,6 +77,8 @@ export interface MoveOptions { baseline: () => PostgresBaseline; /** Where progress lines go. */ log: (line: string) => void; + /** Drop NUL characters from TREK's own text and JSON instead of refusing them (`--drop-nul`). */ + dropNul?: boolean; } interface MovedTable { @@ -80,10 +92,24 @@ export interface MoveSummary { tables: MovedTable[]; plugins: Array<{ pluginId: string; schema: string; tables: MovedTable[]; notCarried: string[] }>; sequences: number; + /** Every value of TREK's own tables from which `--drop-nul` removed NUL characters: table, row and column. */ + nulDropped: string[]; warnings: string[]; seconds: number; } +/** What the operator can do about a NUL in TREK's own data, for a refusal that lists one. */ +const TREK_NUL_REMEDY = + 'TREK no longer stores this character and drops it on its next write. Run the move again with --drop-nul ' + + 'to drop it now; the summary then lists every value it changed.'; + +/** What the operator can do about plugin data that cannot move unchanged. */ +const PLUGIN_DATA_REMEDY = + "A plugin's own data moves exactly as the plugin stored it, or not at all: Postgres text cannot hold a NUL " + + 'character (U+0000), and each column becomes one Postgres type, which hands every value back to the plugin as ' + + "that type. Change those values in the plugin's plugin.db (through the plugin, or with the sqlite3 tool while " + + 'TREK is stopped), or uninstall the plugin and remove its data directory, then run the move again.'; + /** * Why the SQLite file is not at this build's migration level, or null when it * is: it must have recorded exactly the migrations this build ships. @@ -109,12 +135,6 @@ export function migrationLevelProblem(recorded: readonly string[] | null, known: return null; } -function listed(problems: readonly string[], total: number): string { - const lines = problems.map((problem) => ` - ${problem}`); - if (total > problems.length) lines.push(` - and ${total - problems.length} more`); - return lines.join('\n'); -} - /** Refuses a database that holds a table or a plugin schema: the move fills an empty database only. */ async function refuseNonEmptyTarget(db: Kysely): Promise { const tables = await run<{ table_name: string }>( @@ -147,6 +167,27 @@ function planPlugins(root: string, warnings: string[]): Array<{ source: SqliteSo return plugins; } +/** + * Reads every plugin table and checks that each value moves unchanged, before + * anything is written: a refusal lists plugin, table, row and column of each + * value that would not. + */ +async function checkPluginData(plugins: ReadonlyArray<{ source: SqliteSource; plan: PluginPlan }>): Promise { + const checked: CopiedTable[] = []; + await inOrder(plugins, (plugin) => + inOrder(plugin.plan.specs, async (spec) => { + checked.push(await copyTable(null, plugin.source, spec)); + }), + ); + const total = checked.reduce((sum, table) => sum + table.problemCount, 0); + if (total === 0) return; + const problems = checked.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS); + throw new MoveRefusal( + `${total} value(s) in plugin data cannot move to Postgres unchanged; nothing was written:\n` + + `${listLines(problems, total)}\n${PLUGIN_DATA_REMEDY}`, + ); +} + /** Compares a copied table with what Postgres holds now: the SQLite count, the rows sent and the checksum. */ async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promise { const back = await readBack(trx, copied.spec); @@ -154,7 +195,7 @@ async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promi return `${copied.spec.label}: the SQLite file holds ${sqliteCount} rows, ${copied.rows} were sent, Postgres holds ${back.rows}`; } if (back.checksum !== copied.checksum) - return `${copied.spec.label}: the rows Postgres holds differ from the rows sent`; + return `${copied.spec.label}: the rows Postgres holds differ from the rows of the SQLite file`; return null; } @@ -173,7 +214,7 @@ async function copyAll( const plan = planTrekTables(await readTargetSchema(trx), sourceTables); if (plan.problems.length > 0) { throw new MoveRefusal( - `The SQLite file and this TREK's Postgres schema differ:\n${listed(plan.problems, plan.problems.length)}`, + `The SQLite file and this TREK's Postgres schema differ:\n${listLines(plan.problems, plan.problems.length)}`, ); } // The bootstrap's own rows (default settings, addons, providers) give way to the file's. @@ -181,10 +222,12 @@ async function copyAll( const copied: CopiedTable[] = []; let refused = 0; + let nulRefused = 0; const copyOne = (from: SqliteSource) => async (spec: CopiedTable['spec']) => { options.log(`Copying ${spec.label}`); - const result = await copyTable(trx, from, spec, { insert: refused === 0 }); + const result = await copyTable(trx, from, spec, { insert: refused === 0, dropNul: options.dropNul }); refused += result.problemCount; + nulRefused += result.nulRefused; copied.push(result); }; await inOrder(plan.specs, copyOne(source)); @@ -202,9 +245,11 @@ async function copyAll( }); if (refused > 0) { const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS); + const nul = nulRefused > 0 ? `${nulRefused} of them hold a NUL character (U+0000). ${TREK_NUL_REMEDY}\n` : ''; throw new MoveRefusal( `${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` + - `${listed(problems, refused)}\nCorrect those rows (or ask for help with them), then run the move again.`, + `${listLines(problems, refused)}\n${nul}` + + 'Correct those rows (or ask for help with them), then run the move again.', ); } @@ -228,7 +273,7 @@ async function copyAll( }); if (mismatches.length > 0) { throw new Error( - `The copy does not match the SQLite file; nothing was written:\n${listed(mismatches, mismatches.length)}`, + `The copy does not match the SQLite file; nothing was written:\n${listLines(mismatches, mismatches.length)}`, ); } @@ -245,6 +290,7 @@ async function copyAll( tables: (pluginTables.get(pluginPlan.pluginId) ?? []).map(moved), notCarried: pluginPlan.notCarried, })); + summary.nulDropped = copied.flatMap((table) => table.nulDropped.map((value) => value.where)); const unsafe = copied.filter((table) => table.unsafeIntegers > 0); for (const table of unsafe) { summary.warnings.push( @@ -279,6 +325,7 @@ export async function moveSqliteToPostgres(options: MoveOptions): Promise 0) { throw new MoveRefusal( `${options.sqliteFile} holds rows whose reference points at nothing, which Postgres refuses:\n` + - listed( + listLines( orphans.map((row) => `${row.table} row ${String(row.rowid)} refers to a missing row of ${row.parent}`), orphans.length, ), ); } plugins.push(...planPlugins(options.pluginsDataDir, summary.warnings)); + await checkPluginData(plugins); await refuseNonEmptyTarget(db); try { @@ -350,6 +398,10 @@ export function renderSummary(summary: MoveSummary, target: string): string { ); for (const item of plugin.notCarried) lines.push(` not carried over: ${item}`); } + if (summary.nulDropped.length > 0) { + lines.push(` --drop-nul removed NUL characters (U+0000) from ${summary.nulDropped.length} value(s):`); + for (const value of summary.nulDropped) lines.push(` ${value}`); + } for (const warning of summary.warnings) lines.push(` Warning: ${warning}`); lines.push( '', diff --git a/server/src/db/engine/postgres/sqlite-move/plugin-data.ts b/server/src/db/engine/postgres/sqlite-move/plugin-data.ts index 9108f78cf..4ae67f0d3 100644 --- a/server/src/db/engine/postgres/sqlite-move/plugin-data.ts +++ b/server/src/db/engine/postgres/sqlite-move/plugin-data.ts @@ -22,6 +22,11 @@ import path from 'node:path'; * here (foreign keys between its tables, views, triggers, partial or * expression indexes, other defaults) is not created and is listed in the * summary. The file itself stays where it is, untouched. + * + * The rows are the plugin's, so none is changed on the way: every value must + * come back from the Postgres store with the type and bytes it has in the + * file, or the move refuses before it writes anything and lists the value + * (`PLUGIN_RULES` in `values.ts`, checked by `checkPluginData` in `move.ts`). */ /** The plugin store's own record of the migrations it ran, created exactly as the store creates it. */ @@ -40,21 +45,27 @@ const PG_TYPE: Record = { /** * The kind of a plugin's column, by SQLite's affinity rules (the declared * type's text decides, in this order: INT, then CHAR/CLOB/TEXT, then BLOB or - * none, then REAL/FLOA/DOUB, else NUMERIC). Where the affinity leaves the - * storage open (none, or NUMERIC), the values the column holds decide, and - * text is the answer that holds every one of them. + * none, then REAL/FLOA/DOUB, else NUMERIC). An INTEGER, TEXT or REAL affinity + * names the type the plugin asked for. Where the affinity leaves the storage + * open (BLOB or none, or NUMERIC), the values the column holds decide: the + * kind that hands every one of them back to the plugin as it reads it from + * SQLite (an INTEGER or a REAL as a number, TEXT as a string, a BLOB as a + * Buffer). A column whose values no single kind hands back like that still + * gets one, and the move refuses each value it would change + * (`PLUGIN_RULES` in `values.ts`); nothing is converted to fit. */ export function pluginColumnKind(declaredType: string, storage: ReadonlySet): ColumnKind { const declared = declaredType.toUpperCase(); const only = (...classes: string[]) => storage.size > 0 && [...storage].every((value) => classes.includes(value)); if (declared.includes('INT')) return 'integer'; if (/CHAR|CLOB|TEXT/.test(declared)) return 'text'; - if (declared.includes('BLOB')) return only('blob', 'text') ? 'bytes' : 'text'; - if (/REAL|FLOA|DOUB/.test(declared)) return only('integer', 'real') || storage.size === 0 ? 'float' : 'text'; + const blob = declared.includes('BLOB') || declared.trim() === ''; + if (!blob && /REAL|FLOA|DOUB/.test(declared)) return 'float'; if (only('integer')) return 'integer'; if (only('integer', 'real')) return 'float'; if (only('blob')) return 'bytes'; - return 'text'; + // Empty, or text, or a mix: a column declared BLOB with no values keeps bytes for what the plugin writes later. + return storage.size === 0 && declared.includes('BLOB') ? 'bytes' : 'text'; } const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i; @@ -143,6 +154,8 @@ export function planPluginData(pluginId: string, source: SqliteSource): PluginPl ], deferred: [], primaryKey: ['id'], + exact: true, + uniqueKeys: [], }); continue; } @@ -168,6 +181,9 @@ export function planPluginData(pluginId: string, source: SqliteSource): PluginPl columns: columns.map((column, index) => ({ name: column.name, kind: kinds[index] })), deferred: [], primaryKey: keys.map((key) => key.name), + // A plugin's values are never changed (`PLUGIN_RULES`), so no key of its tables can break on the way. + exact: true, + uniqueKeys: [], }); // The identity's sequence is named by Postgres; resolveIdentitySequences reads it once the table exists. if (rowid) plan.sequences.push({ table, target, column: rowid, sequence: '' }); diff --git a/server/src/db/engine/postgres/sqlite-move/values.ts b/server/src/db/engine/postgres/sqlite-move/values.ts index b7cd41152..536b022d2 100644 --- a/server/src/db/engine/postgres/sqlite-move/values.ts +++ b/server/src/db/engine/postgres/sqlite-move/values.ts @@ -9,9 +9,23 @@ import { createHash } from 'node:crypto'; * SQLite stores a value by what it is, not by the column's declared type, so * a column may hold text where the schema says integer. The move never * guesses: a value the Postgres column cannot hold exactly is refused with - * the reason, and the move stops before it commits anything. The one - * exception is a NUL character in text or JSON, which TREK stores on neither - * engine (`db/nul-free.ts`): the move drops it, as every write does. + * the reason, and the move stops before it commits anything. + * + * Two sets of rules decide what "exactly" means ({@link ConvertRules}): + * - TREK's own tables follow TREK's storage model, so a whole number stored + * as text reaches an integer column as that number. A NUL character in text + * or JSON, which TREK has stopped storing on either engine + * (`db/nul-free.ts`), is refused unless the operator asked for it to be + * dropped (`--drop-nul`); each value that loses one is reported. + * - A plugin's own data is the plugin's, so a value must come back from the + * Postgres plugin store as the plugin's SQLite file hands it back today: + * an INTEGER or a REAL as a number, TEXT as a string, a BLOB as a Buffer, + * with the same bytes. Nothing changes its storage class, and a NUL, which + * the Postgres store cannot hold, is refused. + * + * The compared text of a row is built from the value read from SQLite + * ({@link canonicalFromSource}), never from the value sent, so a conversion + * that changed a value fails the verification instead of passing it. * * The SQLite side is read with better-sqlite3's safe integers, so an integer * arrives as a `bigint` and keeps all 64 bits. @@ -26,13 +40,31 @@ export type SqliteValue = null | bigint | number | string | Uint8Array; /** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */ export type PostgresValue = null | string | number | Buffer; -type Converted = { ok: true; value: PostgresValue } | { ok: false; reason: string }; +/** A converted value, and whether `--drop-nul` removed NUL characters from it. */ +type Converted = { ok: true; value: PostgresValue; droppedNul?: true } | { ok: false; reason: string }; + +/** How strictly a value must arrive; see the header. */ +export interface ConvertRules { + /** A plugin's own data: the value keeps its storage class and its bytes, and a NUL is refused. */ + exact: boolean; + /** TREK's own data: a NUL in text or JSON is dropped (and reported) instead of refused. */ + dropNul: boolean; +} + +/** TREK's own tables, without `--drop-nul`. */ +const TREK_RULES: ConvertRules = { exact: false, dropNul: false }; +/** A plugin's own tables. */ +export const PLUGIN_RULES: ConvertRules = { exact: true, dropNul: false }; const INT64_MIN = -(2n ** 63n); const INT64_MAX = 2n ** 63n - 1n; const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER); const DECIMAL_INTEGER = /^[+-]?\d+$/; const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i; +const NUL = '\u0000'; + +/** Why a value holding a NUL character is refused. */ +export const NUL_REASON = 'holds a NUL character (U+0000), which Postgres text cannot hold'; /** * The kind of a Postgres column, from `information_schema.columns.data_type`. @@ -102,13 +134,27 @@ function toFloat(value: Exclude): Converted { return refuse(`${describe(value)} is not a number`); } -function toText(value: Exclude): Converted { - if (typeof value === 'string') return { ok: true, value: stripNul(value) }; +function toText(value: Exclude, dropNul: boolean): Converted { + if (typeof value === 'string') { + if (!value.includes(NUL)) return { ok: true, value }; + if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON}`); + return { ok: true, value: stripNul(value), droppedNul: true }; + } if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; return refuse(`${describe(value)} is not text`); } -function toJson(value: Exclude): Converted { +/** Whether a parsed JSON value holds a NUL in any string, object keys included. */ +function jsonHoldsNul(value: unknown): boolean { + if (typeof value === 'string') return value.includes(NUL); + if (Array.isArray(value)) return value.some(jsonHoldsNul); + if (value !== null && typeof value === 'object') { + return Object.entries(value).some(([key, item]) => key.includes(NUL) || jsonHoldsNul(item)); + } + return false; +} + +function toJson(value: Exclude, dropNul: boolean): Converted { if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`); let parsed: unknown; @@ -117,8 +163,10 @@ function toJson(value: Exclude): Converted { } catch { return refuse(`${describe(value)} is not valid JSON`); } - // Valid JSON holds a NUL only escaped, which jsonb refuses as well: the value is written again without it. - return { ok: true, value: value.includes('\\u0000') ? JSON.stringify(nulFreeJson(parsed)) : value }; + // Valid JSON holds a NUL only escaped, which jsonb refuses as well. + if (!jsonHoldsNul(parsed)) return { ok: true, value }; + if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`); + return { ok: true, value: JSON.stringify(nulFreeJson(parsed)), droppedNul: true }; } function toBytes(value: Exclude): Converted { @@ -128,18 +176,71 @@ function toBytes(value: Exclude): Converted { return refuse(`${describe(value)} is not a BLOB`); } -/** `value` as the Postgres column of `kind` takes it, or the reason it cannot hold it exactly. */ -export function convertValue(value: SqliteValue, kind: ColumnKind): Converted { +/** What a column of `kind` holds on Postgres, for a refusal. */ +const KIND_HOLDS: Record = { + integer: 'integers (bigint)', + float: 'numbers (double precision)', + text: 'text', + json: 'JSON (jsonb)', + bytes: 'bytes (bytea)', +}; + +/** The SQLite storage class of a value, for a refusal. */ +function storageClass(value: Exclude): string { + if (value instanceof Uint8Array) return 'a BLOB'; + if (typeof value === 'string') return 'TEXT'; + return typeof value === 'bigint' ? 'an INTEGER' : 'a REAL'; +} + +/** + * A plugin's value as its Postgres column takes it, only where the Postgres + * plugin store hands it back to the plugin as the SQLite file did: a `bigint` + * column reads as a number (and one past 2^53 is refused, `parseInt8` in + * `postgres-driver.ts`), `double precision` as a number, `text` as a string + * and `bytea` as a Buffer. + */ +function exactly(value: Exclude, kind: ColumnKind): Converted { + const changed = () => + refuse( + `${describe(value)} is stored as ${storageClass(value)}, but the column holds ${KIND_HOLDS[kind]} on Postgres, ` + + 'so the plugin would read it back as another type', + ); + switch (kind) { + case 'integer': + if (typeof value !== 'bigint') return changed(); + if (value > SAFE_MAX || value < -SAFE_MAX) { + return refuse(`${describe(value)} is beyond 2^53, which the Postgres plugin store refuses to read`); + } + return { ok: true, value: value.toString() }; + case 'float': + if (typeof value === 'number') return { ok: true, value }; + if (typeof value !== 'bigint') return changed(); + return value > SAFE_MAX || value < -SAFE_MAX + ? refuse(`${describe(value)} has more digits than a double holds`) + : { ok: true, value: Number(value) }; + case 'text': + if (typeof value !== 'string') return changed(); + return value.includes(NUL) ? refuse(`${describe(value)} ${NUL_REASON}`) : { ok: true, value }; + case 'bytes': + return value instanceof Uint8Array ? { ok: true, value: Buffer.from(value) } : changed(); + case 'json': + return toJson(value, false); + } +} + +/** `value` as the Postgres column of `kind` takes it under `rules`, or the reason it cannot hold it exactly. */ +export function convertValue(value: SqliteValue, kind: ColumnKind, rules: ConvertRules = TREK_RULES): Converted { if (value === null) return { ok: true, value: null }; + if (rules.exact) return exactly(value, kind); switch (kind) { case 'integer': return toInteger(value); case 'float': return toFloat(value); case 'text': - return toText(value); + return toText(value, rules.dropNul); case 'json': - return toJson(value); + return toJson(value, rules.dropNul); case 'bytes': return toBytes(value); } @@ -168,23 +269,41 @@ export function stableJson(text: string): string { return JSON.stringify(sort(JSON.parse(text))); } +/** Bytes as the compared text: their hex digits. */ +function hex(bytes: Uint8Array): string { + return Buffer.from(bytes).toString('hex'); +} + /** - * The text a value is compared in, from the value the move sent to Postgres. - * `canonicalFromPostgres` must answer the same for what Postgres stored. + * The text a value is compared in, from the value read from the SQLite file + * (not from what the move sent), by the rules it was converted under. + * `canonicalFromPostgres` must answer the same for what Postgres stored, so a + * conversion that changed a value, or a value that changed on the way, fails + * the verification. The one change the rules allow is `--drop-nul` on TREK's + * own text and JSON, which the operator asked for and the summary lists. A + * value of a storage class its kind does not take is refused before it is + * sent; here it answers a marked text that matches nothing Postgres prints. */ -export function canonicalFromSent(value: PostgresValue, kind: ColumnKind): string | null { +export function canonicalFromSource( + value: SqliteValue, + kind: ColumnKind, + rules: ConvertRules = TREK_RULES, +): string | null { if (value === null) return null; + if (value instanceof Uint8Array) return kind === 'bytes' ? hex(value) : `blob:${hex(value)}`; + const dropNul = rules.dropNul && !rules.exact; switch (kind) { case 'integer': - return BigInt(String(value)).toString(); + return typeof value === 'bigint' ? value.toString() : BigInt(String(value).trim()).toString(); case 'float': - return String(Number(value)); + return typeof value === 'bigint' ? value.toString() : String(Number(String(value).trim())); case 'text': - return String(value); + return typeof value === 'string' && dropNul ? stripNul(value) : String(value); case 'json': - return stableJson(String(value)); + if (typeof value !== 'string') return stableJson(String(value)); + return stableJson(dropNul ? JSON.stringify(nulFreeJson(JSON.parse(value))) : value); case 'bytes': - return Buffer.isBuffer(value) ? value.toString('hex') : Buffer.from(String(value), 'utf8').toString('hex'); + return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`; } } diff --git a/server/tests/helpers/fake-postgres-move.ts b/server/tests/helpers/fake-postgres-move.ts index 62c43ecab..8641887e0 100644 --- a/server/tests/helpers/fake-postgres-move.ts +++ b/server/tests/helpers/fake-postgres-move.ts @@ -25,6 +25,8 @@ export interface FakeTableDef { foreignKeys?: Array<{ columns: string[]; references: string }>; /** Columns a sequence fills. */ serial?: string[]; + /** Unique indexes besides the primary key's (`_pkey`). */ + unique?: Array<{ name: string; columns: string[] }>; } interface FakeTable { @@ -55,6 +57,8 @@ export class FakePostgres { pluginSchemas: string[] = []; /** A statement matching this fails, as Postgres would refuse it. */ failOn: RegExp | null = null; + /** What the refusal of `failOn` carries besides its message: by default a broken key without a SQLSTATE. */ + failWith: Record = { detail: 'Key (id)=(1)', constraint: 'fake_key' }; /** Rows the read-back answers in place of the stored ones, by table: a copy that came out wrong. */ tamper = new Map unknown[][]>(); private cursor: Array> = []; @@ -81,7 +85,7 @@ export class FakePostgres { const params = [...query.parameters]; this.statements.push(sql); if (this.failOn?.test(sql)) { - throw Object.assign(new Error('refused by the fake'), { detail: 'Key (id)=(1)', constraint: 'fake_key' }); + throw Object.assign(new Error('refused by the fake'), this.failWith); } return { rows: this.answer(sql, params) as R[] }; } @@ -123,6 +127,14 @@ export class FakePostgres { })), ]); } + if (sql.includes('from pg_index')) { + return this.schema.flatMap((table) => [ + ...(table.primaryKey.length > 0 + ? [{ table_name: table.name, index_name: `${table.name}_pkey`, columns: table.primaryKey }] + : []), + ...(table.unique ?? []).map((key) => ({ table_name: table.name, index_name: key.name, columns: key.columns })), + ]); + } if (sql.includes("pg_get_serial_sequence(format('%I'")) { return this.schema.flatMap((table) => (table.serial ?? []).map((column) => ({ diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts index f4073092c..0b24f746a 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts @@ -1,5 +1,5 @@ /** - * MOVE-001..016: the move of a SQLite install into Postgres + * MOVE-001..025: the move of a SQLite install into Postgres * (src/db/engine/postgres/sqlite-move/), over real SQLite files and a * stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts): * the refusals before anything is written, the copy in foreign key order with @@ -15,7 +15,12 @@ import { USAGE, type MoveCommandIo, } from '../../../../../../src/db/engine/postgres/sqlite-move/command'; -import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy'; +import { + batchRows, + describePgError, + insertSql, + refusedRowsMessage, +} from '../../../../../../src/db/engine/postgres/sqlite-move/copy'; import { migrationLevelProblem, MoveRefusal, @@ -82,6 +87,7 @@ const SCHEMA: FakeTableDef[] = [ ], primaryKey: ['id'], serial: ['id'], + unique: [{ name: 'users_name_unique', columns: ['name'] }], }, ]; @@ -308,6 +314,134 @@ describe('moveSqliteToPostgres', () => { }); }); +describe('nothing is changed silently', () => { + /** A plugin's `plugin.db` under the plugins' data directory. */ + function pluginDb(pluginId: string, sql: string): void { + fs.mkdirSync(path.join(dir, 'plugins-data', pluginId), { recursive: true }); + sqlite(path.join(dir, 'plugins-data', pluginId, 'plugin.db'), sql); + } + + it("MOVE-021: a plugin's NUL or a value that would change type refuses the move before anything is written", async () => { + pluginDb( + 'nul-notes', + `create table notes (id integer primary key, tag text, body text); + insert into notes values (1, 'team' || char(0) || 'a', 'secret' || char(0) || 'payload'), (2, 'teama', 'other'); + create table kv (k text primary key, v); + insert into kv values ('a', 5), ('b', 1.5), ('c', 'x');`, + ); + const failure = moveSqliteToPostgres(options()); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + const message = await failure.catch((err: Error) => err.message); + expect(message).toContain( + '4 value(s) in plugin data cannot move to Postgres unchanged; nothing was written:\n' + + ' - plugin nul-notes: kv row k=a, column v: the value 5 is stored as an INTEGER, but the column holds text ' + + 'on Postgres, so the plugin would read it back as another type\n' + + ' - plugin nul-notes: kv row k=b, column v: the value 1.5 is stored as a REAL', + ); + expect(message).toContain( + ' - plugin nul-notes: notes row id=1, column tag: the text "team\\u0000a" holds a NUL character (U+0000), ' + + 'which Postgres text cannot hold\n' + + ' - plugin nul-notes: notes row id=1, column body: the text "secret\\u0000payload" holds a NUL character', + ); + expect(message).toMatch(/A plugin's own data moves exactly as the plugin stored it, or not at all/); + // Refused before the target was even looked at: nothing created, nothing inserted. + expect(pg.statements).toEqual([]); + }); + + it('MOVE-022: a BLOB column holding text stays text, and a column holding only numbers stays numbers', async () => { + pluginDb( + 'typed-notes', + `create table docs (id integer primary key, payload blob, raw blob, data); + insert into docs values (1, '{"a":1}', x'00ff', 5), (2, 'plain', null, 1.5);`, + ); + const summary = await moveSqliteToPostgres(options()); + expect(pg.statements).toContain( + 'create table "trek_plugin_typed_notes"."docs" ("id" bigint generated by default as identity, ' + + '"payload" text, "raw" bytea, "data" double precision, primary key ("id"))', + ); + expect(pg.rowsOf('"trek_plugin_typed_notes"."docs"')).toEqual([ + ['1', '{"a":1}', Buffer.from([0, 255]), 5], + ['2', 'plain', null, 1.5], + ]); + expect(summary.plugins[0].tables).toEqual([{ label: 'docs', rows: 2, checksum: expect.any(String) }]); + }); + + it("MOVE-023: a NUL in TREK's own text is refused by default, and with dropNul dropped and listed", async () => { + sqlite( + path.join(dir, 'travel.db'), + `update users set name = 'Ad' || char(0) || 'a' where id = 1; + update trips set meta = '{"k\\u0000":1}' where id = 1;`, + ); + const failure = moveSqliteToPostgres(options()); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + const message = await failure.catch((err: Error) => err.message); + expect(message).toContain( + '2 value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n' + + ' - users row id=1, column name: the text "Ad\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold\n' + + ' - trips row id=1, column meta: the text "{\\"k\\\\u0000\\":1}" holds a NUL character (U+0000), which ' + + 'Postgres text cannot hold (escaped as \\u0000)\n' + + '2 of them hold a NUL character (U+0000). TREK no longer stores this character and drops it on its next write. ' + + 'Run the move again with --drop-nul', + ); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + + const summary = await moveSqliteToPostgres(options({ dropNul: true })); + expect(pg.rowsOf('"users"')).toEqual([ + ['1', 'Ada'], + ['2', 'Grace'], + ]); + expect(pg.rowsOf('"trips"')[0][3]).toBe('{"k":1}'); + expect(summary.nulDropped).toEqual(['users row id=1, column name', 'trips row id=1, column meta']); + expect(renderSummary(summary, 'postgres://db/trek')).toContain( + ' --drop-nul removed NUL characters (U+0000) from 2 value(s):\n' + + ' users row id=1, column name\n' + + ' trips row id=1, column meta', + ); + }); + + it('MOVE-024: a unique key that dropping a NUL broke is named as that, never as a bare duplicate', async () => { + sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 2;"); + pg.failOn = /^insert into "users"/; + pg.failWith = { + code: '23505', + constraint: 'users_name_unique', + detail: 'Key (name)=(Ada) already exists.', + }; + await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow( + 'Postgres refused rows of users: refused by the fake; constraint users_name_unique; Key (name)=(Ada) already ' + + 'exists.\nThe key is broken because --drop-nul removed NUL characters (U+0000) from 1 value(s) of its ' + + 'column(s) name, and a value that differed from another only by a NUL equals it once the NUL is gone:\n' + + ' - users row id=2, column name\n' + + 'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.', + ); + // A broken key whose columns lost no NUL is reported as Postgres reported it. + pg.failWith = { code: '23505', constraint: 'users_pkey', detail: 'Key (id)=(2) already exists.' }; + await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow( + /^Postgres refused rows of users: refused by the fake; constraint users_pkey; Key \(id\)=\(2\) already exists\.$/, + ); + }); + + it('MOVE-025: a refused key is explained by every dropped NUL of the table when the key is not known', () => { + const spec = { + source: 't', + target: '"t"', + label: 't', + columns: [], + deferred: [], + primaryKey: [], + exact: false, + uniqueKeys: [], + }; + const dropped = [{ where: 't row 1, column a', column: 'a' }]; + const err = Object.assign(new Error('duplicate key'), { code: '23505', constraint: 'gone' }); + expect(refusedRowsMessage(spec, err, dropped)).toMatch( + /from 1 value\(s\) of t, and a value .*\n {2}- t row 1, column a\n/, + ); + expect(refusedRowsMessage(spec, new Error('other'), dropped)).toBe('Postgres refused rows of t: other'); + expect(refusedRowsMessage(spec, err, [])).toBe('Postgres refused rows of t: duplicate key; constraint gone'); + }); +}); + describe('migrationLevelProblem', () => { it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => { const known = new Set([M1, M2]); @@ -327,12 +461,19 @@ describe('the pieces', () => { expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer'); expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text'); expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes'); - expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text'); + expect(pluginColumnKind('BLOB', classes('integer'))).toBe('integer'); + // Text in a column declared BLOB stays text: the plugin reads a string back, not a Buffer. + expect(pluginColumnKind('BLOB', classes('text'))).toBe('text'); + expect(pluginColumnKind('BLOB', classes('blob', 'text'))).toBe('text'); + expect(pluginColumnKind('BLOB', classes())).toBe('bytes'); expect(pluginColumnKind('DOUBLE', classes())).toBe('float'); - expect(pluginColumnKind('REAL', classes('text'))).toBe('text'); + // A REAL column keeps its type; text in it is refused by the copy, not turned into text with the numbers. + expect(pluginColumnKind('REAL', classes('text'))).toBe('float'); expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer'); expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float'); expect(pluginColumnKind('', classes('blob'))).toBe('bytes'); + expect(pluginColumnKind('', classes())).toBe('text'); + expect(pluginColumnKind('', classes('integer', 'text'))).toBe('text'); expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text'); expect(pluginDefault(null, 'text')).toBeNull(); @@ -394,6 +535,7 @@ describe('runMoveCommand', () => { tables: [{ label: 'users', rows: 2, checksum: 'x' }], plugins: [], sequences: 1, + nulDropped: [], warnings: ['careful'], seconds: 0.4, }; @@ -421,6 +563,7 @@ describe('runMoveCommand', () => { const help = io(); expect(await runMoveCommand(['--help'], help.io)).toBe(0); expect(help.out).toEqual([USAGE]); + expect(USAGE).toMatch(/\[--drop-nul\][\s\S]*--drop-nul +drop NUL characters \(U\+0000\) from TREK's own text/); const unknown = io(); expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1); expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/); @@ -436,8 +579,12 @@ describe('runMoveCommand', () => { expect.objectContaining({ sqliteFile: path.join(dir, 'other.db'), pluginsDataDir: expect.stringMatching(/plugins-data$/), + dropNul: false, }), ); + const dropping = io(); + expect(await runMoveCommand(['--drop-nul'], dropping.io)).toBe(0); + expect(dropping.io.move).toHaveBeenCalledWith(expect.objectContaining({ dropNul: true })); const printed = run.out.join('\n'); expect(printed).toContain('postgres://trek:***@db:5432/trek'); expect(printed).not.toContain('secret'); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts index c8515e4e8..f55727f70 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts @@ -63,7 +63,12 @@ describe('planCopyOrder', () => { describe('planTrekTables', () => { const target: TargetSchema = { tables: [ - { name: 'mikro_orm_migrations', columns: [{ name: 'name', dataType: 'text', nullable: false }], primaryKey: [] }, + { + name: 'mikro_orm_migrations', + columns: [{ name: 'name', dataType: 'text', nullable: false }], + primaryKey: [], + uniqueKeys: [], + }, { name: 'messages', columns: [ @@ -71,6 +76,7 @@ describe('planTrekTables', () => { { name: 'reply_to', dataType: 'bigint', nullable: true }, ], primaryKey: ['id'], + uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }], }, { name: 'users', @@ -79,6 +85,7 @@ describe('planTrekTables', () => { { name: 'avatar', dataType: 'bytea', nullable: true }, ], primaryKey: ['id'], + uniqueKeys: [], }, ], foreignKeys: [{ table: 'messages', columns: ['reply_to'], references: 'messages' }], @@ -96,7 +103,13 @@ describe('planTrekTables', () => { ); expect(plan.problems).toEqual([]); expect(plan.specs.map((spec) => spec.target)).toEqual(['"messages"', '"users"']); - expect(plan.specs[0]).toMatchObject({ deferred: ['reply_to'], primaryKey: ['id'], label: 'messages' }); + expect(plan.specs[0]).toMatchObject({ + deferred: ['reply_to'], + primaryKey: ['id'], + label: 'messages', + exact: false, + uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }], + }); expect(plan.specs[1].columns).toEqual([ { name: 'id', kind: 'integer' }, { name: 'avatar', kind: 'bytes' }, @@ -119,8 +132,9 @@ describe('planTrekTables', () => { { name: 'sent_at', dataType: 'timestamp with time zone', nullable: true }, ], primaryKey: ['id'], + uniqueKeys: [], }, - { name: 'extra', columns: [], primaryKey: [] }, + { name: 'extra', columns: [], primaryKey: [], uniqueKeys: [] }, ], }; const plan = planTrekTables( @@ -143,7 +157,7 @@ describe('planTrekTables', () => { }); describe('readTargetSchema', () => { - it('MOVEORD-007: reads the columns, keys and sequences of the current schema', async () => { + it('MOVEORD-007: reads the columns, keys, unique keys and sequences of the current schema', async () => { const pg = new FakePostgres([ { name: 'trips', @@ -154,6 +168,7 @@ describe('readTargetSchema', () => { primaryKey: ['id'], foreignKeys: [{ columns: ['user_id'], references: 'users' }], serial: ['id'], + unique: [{ name: 'trips_user_id_unique', columns: ['user_id'] }], }, { name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] }, ]); @@ -166,8 +181,17 @@ describe('readTargetSchema', () => { { name: 'id', dataType: 'bigint', nullable: false }, { name: 'user_id', dataType: 'bigint', nullable: true }, ], + uniqueKeys: [ + { name: 'trips_pkey', columns: ['id'] }, + { name: 'trips_user_id_unique', columns: ['user_id'] }, + ], + }, + { + name: 'users', + primaryKey: ['id'], + columns: [{ name: 'id', dataType: 'bigint', nullable: false }], + uniqueKeys: [{ name: 'users_pkey', columns: ['id'] }], }, - { name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }] }, ]); expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]); expect(schema.sequences).toEqual([{ table: 'trips', column: 'id', sequence: 'public.trips_id_seq' }]); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts index 1e6093dc5..2164e2ac6 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts @@ -1,14 +1,16 @@ /** - * MOVEVAL-001..010: how a SQLite value becomes the value a Postgres column of - * the move's target takes, the text both sides are compared in, and the - * order-free table checksum (src/db/engine/postgres/sqlite-move/values.ts). + * MOVEVAL-001..014: how a SQLite value becomes the value a Postgres column of + * the move's target takes, under TREK's rules and under a plugin's, the text + * both sides are compared in, and the order-free table checksum + * (src/db/engine/postgres/sqlite-move/values.ts). */ import { beyondSafeInteger, canonicalFromPostgres, - canonicalFromSent, + canonicalFromSource, columnKind, convertValue, + PLUGIN_RULES, stableJson, TableChecksum, type ColumnKind, @@ -19,6 +21,7 @@ import path from 'node:path'; import { describe, expect, it } from 'vitest'; const ok = (value: unknown) => ({ ok: true, value }); +const DROP_NUL = { exact: false, dropNul: true }; describe('columnKind', () => { it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => { @@ -80,11 +83,15 @@ describe('convertValue', () => { expect(convertValue('', 'float')).toMatchObject({ ok: false }); }); - it('MOVEVAL-005: text takes strings and numbers, drops a NUL as every write does, and refuses a BLOB', () => { + it('MOVEVAL-005: text takes strings and numbers, refuses a NUL unless told to drop it, and refuses a BLOB', () => { expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa')); expect(convertValue(7n, 'text')).toEqual(ok('7')); expect(convertValue(2.5, 'text')).toEqual(ok('2.5')); - expect(convertValue('a\u0000b\u0000', 'text')).toEqual(ok('ab')); + expect(convertValue('a\u0000b\u0000', 'text')).toEqual({ + ok: false, + reason: 'the text "a\\u0000b\\u0000" holds a NUL character (U+0000), which Postgres text cannot hold', + }); + expect(convertValue('a\u0000b\u0000', 'text', DROP_NUL)).toEqual({ ok: true, value: 'ab', droppedNul: true }); // The six characters of an escape are text like any other. expect(convertValue('a\\u0000b', 'text')).toEqual(ok('a\\u0000b')); expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({ @@ -97,15 +104,25 @@ describe('convertValue', () => { }); }); - it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes', () => { + it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes only when told to', () => { expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}')); expect(convertValue(5n, 'json')).toEqual(ok('5')); expect(convertValue('{broken', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not valid JSON/), }); - expect(convertValue('"\\u0000"', 'json')).toEqual(ok('""')); - expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json')).toEqual(ok('{"a":["xy",1]}')); + expect(convertValue('"\\u0000"', 'json')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/holds a NUL character \(U\+0000\), .* \(escaped as \\u0000\)$/), + }); + expect(convertValue('"\\u0000"', 'json', DROP_NUL)).toEqual({ ok: true, value: '""', droppedNul: true }); + expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json', DROP_NUL)).toEqual({ + ok: true, + value: '{"a":["xy",1]}', + droppedNul: true, + }); + // An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is. + expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"')); // A raw NUL is not valid JSON in the first place. expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) }); expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({ @@ -130,24 +147,33 @@ describe('convertValue', () => { }); describe('the compared text', () => { - const cases: Array<[ColumnKind, Parameters[0], string]> = [ - ['integer', '-12', '-12'], + const cases: Array<[ColumnKind, Parameters[0], string]> = [ + ['integer', ' -12 ', '-12'], + ['integer', 9007199254740993n, '9007199254740993'], + ['integer', 42, '42'], ['float', 0.30000000000000004, '0.30000000000000004'], ['float', 1e21, '1e+21'], + ['float', 12n, '12'], + ['float', '3.5e2', '350'], ['text', 'São Paulo', 'São Paulo'], + ['text', 7n, '7'], ['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'], - ['bytes', Buffer.from([0, 171]), '\\x00ab'], + ['json', 5n, '5'], + ['bytes', new Uint8Array([0, 171]), '\\x00ab'], ]; - it.each(cases)('MOVEVAL-009: %s values compare equal as sent and as Postgres prints them', (kind, sent, printed) => { - expect(canonicalFromSent(sent, kind)).toBe(canonicalFromPostgres(printed, kind)); - expect(canonicalFromSent(null, kind)).toBeNull(); - expect(canonicalFromPostgres(null, kind)).toBeNull(); - }); + it.each(cases)( + 'MOVEVAL-009: %s values compare equal as read from SQLite and as Postgres prints them', + (kind, read, printed) => { + expect(canonicalFromSource(read, kind)).toBe(canonicalFromPostgres(printed, kind)); + expect(canonicalFromSource(null, kind)).toBeNull(); + expect(canonicalFromPostgres(null, kind)).toBeNull(); + }, + ); - it('MOVEVAL-009b: text sent for a BLOB compares as its bytes, and a float as Postgres spells it', () => { - expect(canonicalFromSent('ab', 'bytes')).toBe('6162'); - expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSent(1e20, 'float')); + it('MOVEVAL-009b: text read for a BLOB compares as its bytes, and a float as Postgres spells it', () => { + expect(canonicalFromSource('ab', 'bytes')).toBe('6162'); + expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSource(1e20, 'float')); expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain'); expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}'); }); @@ -176,3 +202,63 @@ describe('the compared text', () => { expect(sum([['null']])).not.toBe(sum([[null]])); }); }); + +describe("a plugin's values (PLUGIN_RULES)", () => { + const exact = (value: Parameters[0], kind: ColumnKind) => + convertValue(value, kind, PLUGIN_RULES); + const changed = + /is stored as .*, but the column holds .* on Postgres, so the plugin would read it back as another type/; + + it('MOVEVAL-011: a value keeps its storage class and its bytes, as the Postgres plugin store hands it back', () => { + expect(exact(42n, 'integer')).toEqual(ok('42')); + expect(exact(1.5, 'float')).toEqual(ok(1.5)); + expect(exact(3n, 'float')).toEqual(ok(3)); + expect(exact('{"a":1}', 'text')).toEqual(ok('{"a":1}')); + expect(exact(new Uint8Array([0, 1]), 'bytes')).toEqual(ok(Buffer.from([0, 1]))); + expect(exact(null, 'text')).toEqual(ok(null)); + }); + + it('MOVEVAL-012: a value that would come back as another type is refused, not converted', () => { + expect(exact(1.5, 'text')).toEqual({ + ok: false, + reason: + 'the value 1.5 is stored as a REAL, but the column holds text on Postgres, ' + + 'so the plugin would read it back as another type', + }); + expect(exact(5n, 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact('{"a":1}', 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact(' 7', 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact(2.5, 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact('2.5', 'float')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact(new Uint8Array([1]), 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) }); + expect(exact(9007199254740993n, 'integer')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/beyond 2\^53, which the Postgres plugin store refuses to read/), + }); + expect(exact(-9007199254740993n, 'float')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/more digits than a double holds/), + }); + }); + + it('MOVEVAL-013: a NUL in a plugin value is refused, never dropped, whatever the operator asked for', () => { + expect(exact('team\u0000a', 'text')).toEqual({ + ok: false, + reason: 'the text "team\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold', + }); + expect(convertValue('team\u0000a', 'text', { exact: true, dropNul: true })).toMatchObject({ ok: false }); + }); + + it('MOVEVAL-014: the compared text comes from the SQLite value, so a value that changed on the way does not match', () => { + // What Postgres would print had the NUL been dropped, or the type changed, on the way. + expect(canonicalFromSource('team\u0000a', 'text', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('teama', 'text')); + expect(canonicalFromSource('team\u0000a', 'text', { exact: true, dropNul: true })).not.toBe('teama'); + expect(canonicalFromSource(new Uint8Array([0x61]), 'text', PLUGIN_RULES)).not.toBe( + canonicalFromPostgres('a', 'text'), + ); + expect(canonicalFromSource(5n, 'bytes')).not.toBe(canonicalFromPostgres('\\x35', 'bytes')); + // The one change allowed is the one the operator asked for on TREK's own data. + expect(canonicalFromSource('team\u0000a', 'text', DROP_NUL)).toBe(canonicalFromPostgres('teama', 'text')); + expect(canonicalFromSource('{"k\\u0000":1}', 'json', DROP_NUL)).toBe(canonicalFromPostgres('{"k": 1}', 'json')); + }); +});