diff --git a/server/src/db/engine/postgres/sqlite-move/catalog.ts b/server/src/db/engine/postgres/sqlite-move/catalog.ts index 11c6460fb..94cabf20b 100644 --- a/server/src/db/engine/postgres/sqlite-move/catalog.ts +++ b/server/src/db/engine/postgres/sqlite-move/catalog.ts @@ -1,4 +1,4 @@ -import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy'; +import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx, type UniqueKey } from './copy'; import { planCopyOrder, type ForeignKey } from './order'; import { columnKind } from './values'; @@ -23,8 +23,8 @@ interface TargetTable { name: string; columns: TargetColumn[]; primaryKey: string[]; - /** Every unique index (the primary key's among them) over plain columns, by name. */ - uniqueKeys: Array<{ name: string; columns: string[] }>; + /** Every unique index (the primary key's among them), by name, expression indexes included. */ + uniqueKeys: UniqueKey[]; } export interface TargetSchema { @@ -59,14 +59,30 @@ export async function readTargetSchema(trx: Trx): Promise { "and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name", ); - // A unique constraint is backed by an index of its name, so the index names what a refusal reports. - const uniqueIndexes = await run<{ table_name: string; index_name: string; columns: string[] }>( + // A unique constraint is backed by an index of its name, so the index names what a refusal reports. Its key + // is read as Postgres spells it. `indkey` holds 0 for a key that is an expression (`lower(email)`), which + // marks the index as an expression index; the columns an expression reads are the Var nodes of `indexprs`, + // listed after the plain key columns. The partial index's predicate and the INCLUDE columns are left out, + // since neither decides whether two rows collide. + const uniqueIndexes = await run<{ + table_name: string; + index_name: string; + keys: string[]; + columns: string[]; + expression: boolean; + }>( trx, - 'select t.relname as table_name, i.relname as index_name, array_agg(a.attname::text order by k.ord) as columns ' + + 'select t.relname as table_name, i.relname as index_name, ' + + '0 = any ((x.indkey::int2[])[0:x.indnkeyatts - 1]) as expression, ' + + 'array(select pg_get_indexdef(x.indexrelid, k, true) from generate_series(1, x.indnkeyatts::int) as k ' + + 'order by k) as keys, ' + + 'array(select a.attname::text from pg_attribute a where a.attrelid = x.indrelid and a.attnum > 0 and (' + + 'a.attnum = any ((x.indkey::int2[])[0:x.indnkeyatts - 1]) or a.attnum::text in (select m[1] from ' + + "regexp_matches(coalesce(x.indexprs::text, ''), ':varattno ([0-9]+)', 'g') as m)) " + + 'order by coalesce(array_position((x.indkey::int2[])[0:x.indnkeyatts - 1], a.attnum), ' + + 'x.indnkeyatts + a.attnum)) as columns ' + 'from pg_index x join pg_class i on i.oid = x.indexrelid join pg_class t on t.oid = x.indrelid ' + - 'cross join lateral unnest(x.indkey) with ordinality as k(attnum, ord) ' + - 'join pg_attribute a on a.attrelid = x.indrelid and a.attnum = k.attnum ' + - 'where x.indisunique and t.relnamespace = current_schema()::regnamespace group by t.relname, i.relname', + 'where x.indisunique and t.relnamespace = current_schema()::regnamespace order by t.relname, i.relname', ); const tables = new Map(); @@ -85,7 +101,12 @@ export async function readTargetSchema(trx: Trx): Promise { tables.set(column.table_name, table); } for (const index of uniqueIndexes) { - tables.get(index.table_name)?.uniqueKeys.push({ name: index.index_name, columns: index.columns }); + tables.get(index.table_name)?.uniqueKeys.push({ + name: index.index_name, + key: index.keys.join(', '), + columns: index.columns, + expression: index.expression, + }); } const foreignKeys: ForeignKey[] = []; for (const constraint of constraints) { diff --git a/server/src/db/engine/postgres/sqlite-move/command.ts b/server/src/db/engine/postgres/sqlite-move/command.ts index dcc46b08a..f77ec7d66 100644 --- a/server/src/db/engine/postgres/sqlite-move/command.ts +++ b/server/src/db/engine/postgres/sqlite-move/command.ts @@ -5,7 +5,8 @@ import type { AnyOrm } from '../../../orm'; import { buildOrmConfig, resolveEngine } from '../../index'; import { redactUrl } from '../../redact-url'; import { readPostgresBaseline } from '../baseline'; -import { MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions } from './move'; +import { moveSqliteToPostgres, renderSummary, type MoveOptions } from './move'; +import { MoveRefusal } from './refusal'; import { MikroORM } from '@mikro-orm/core'; import path from 'node:path'; @@ -23,8 +24,9 @@ export const USAGE = 'Usage: move-to-postgres [--from=] [--drop-nul]\n' + 'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.\n' + ' --from= the SQLite file to copy\n' + - " --drop-nul drop NUL characters (U+0000) from TREK's own text, which Postgres cannot hold and TREK drops on\n" + - ' its next write anyway, instead of refusing the move; the summary lists every value changed.\n' + + " --drop-nul drop NUL characters (U+0000) from TREK's own text and JSON (raw or escaped as \\u0000), which\n" + + ' Postgres cannot hold and TREK drops on its next write anyway, instead of refusing the move;\n' + + ' the summary lists every value changed.\n' + " A plugin's own data is never changed: a NUL there refuses the move either way."; export interface MoveCommandIo { diff --git a/server/src/db/engine/postgres/sqlite-move/copy.ts b/server/src/db/engine/postgres/sqlite-move/copy.ts index 5f6c79f52..b34d0edbb 100644 --- a/server/src/db/engine/postgres/sqlite-move/copy.ts +++ b/server/src/db/engine/postgres/sqlite-move/copy.ts @@ -1,4 +1,5 @@ import type { SqliteSource } from '../../sqlite/sqlite-source'; +import { FIX_IN_PLUGIN_DB, FIX_IN_SQLITE_FILE, MoveRefusal } from './refusal'; import { beyondSafeInteger, canonicalFromPostgres, @@ -35,6 +36,20 @@ export const MAX_LISTED_PROBLEMS = 20; /** Postgres's SQLSTATE for a row that would break a unique key. */ const UNIQUE_VIOLATION = '23505'; +/** + * A unique index of a Postgres table. `key` is its key as Postgres spells it + * (`username`, or `lower(email)` for an expression), `columns` every column + * the key reads, so a broken key can be traced to the values that changed. + * `expression` is whether any part of the key is an expression rather than a + * plain column. + */ +export interface UniqueKey { + name: string; + key: string; + columns: string[]; + expression: boolean; +} + /** One column of a table the move copies. */ interface CopyColumn { name: string; @@ -57,7 +72,7 @@ export interface TableSpec { /** A plugin's own table, whose values must arrive exactly as stored (`PLUGIN_RULES` in `values.ts`). */ exact: boolean; /** The unique keys of the Postgres table (the primary key among them), by constraint or index name. */ - uniqueKeys: Array<{ name: string; columns: string[] }>; + uniqueKeys: UniqueKey[]; } /** A value from which `--drop-nul` removed NUL characters, for the summary. */ @@ -65,6 +80,12 @@ interface DroppedNul { /** The table, row and column, named as a refusal names a value. */ where: string; column: string; + /** + * The text without its NUL characters, as Postgres spells it in a broken + * key's detail, or null where it spells it otherwise (JSON, which jsonb + * reformats). + */ + text: string | null; } /** A row's deferred values, kept until every table is in. */ @@ -147,29 +168,106 @@ export function listLines(lines: readonly string[], total = lines.length): strin return shown.join('\n'); } +/** The SQLSTATE of a Postgres error, when it carries one. */ +function sqlState(err: unknown): string { + const { code } = (err ?? {}) as { code?: unknown }; + return typeof code === 'string' ? code : ''; +} + /** - * Why Postgres refused rows of a table. A unique key broken in a table whose - * values lost NUL characters to `--drop-nul` is named as that: two values - * that differed only by a NUL are equal once it is gone, which the SQLite - * file's own key never saw. + * Whether Postgres refused rows over a constraint (SQLSTATE class 23: a + * unique, check, not-null or foreign key violation). Its detail names the + * key or the failing row, which the operator corrects in the SQLite file. + */ +function brokeConstraint(err: unknown): boolean { + return sqlState(err).startsWith('23'); +} + +/** + * Whether Postgres refused a value its column cannot hold (SQLSTATE class + * 22). `values.ts` refuses every such value by table, row and column before + * anything is written, so one that reaches Postgres is a gap in the move, not + * something the operator can find in the file: Postgres does not say which + * row of a batch it was. + */ +function valueRefused(err: unknown): boolean { + return sqlState(err).startsWith('22'); +} + +/** + * The key value of a unique violation's detail (`Key (lower(email))=(ab@x.test) + * already exists.`), lower-cased, or null when the detail has another shape + * (none, or the server's messages are in another language). + */ +function duplicateKeyValue(err: unknown): string | null { + const { detail } = (err ?? {}) as { detail?: unknown }; + if (typeof detail !== 'string') return null; + const match = /^Key \(.*?\)=\((.*)\) already exists\.$/s.exec(detail); + return match ? match[1].toLowerCase() : null; +} + +/** A unique key as a refusal names it: its columns, and for an expression also the key as Postgres spells it. */ +function describeKey(key: UniqueKey): string { + const columns = `column(s) ${key.columns.join(', ')}`; + return key.expression ? `${columns} (read by its unique key ${key.key})` : columns; +} + +/** + * Why Postgres refused rows of a table, and what to do. A broken unique key + * is two rows the SQLite file held apart: it may lack that index, compare + * differently, or have held two values that differed only by a NUL until + * `--drop-nul` removed it. Only Postgres's detail tells which, so the refusal + * never asserts a cause: it names the key and the value, and lists the values + * `--drop-nul` changed in the columns the key reads (for an expression index, + * the columns its expression reads) whose text the duplicate value holds. + * That test ignores case, so a key over lower() still finds them; it may keep + * a value that is not one of the two rows, but never leaves out one that is. + * Any other broken constraint is a row Postgres names in its detail. A value + * its column refuses is a gap in the move, reported as one. */ export function refusedRowsMessage(spec: TableSpec, err: unknown, dropped: readonly DroppedNul[]): string { const message = `Postgres refused rows of ${spec.label}: ${describePgError(err)}`; - const { code, constraint } = (err ?? {}) as { code?: unknown; constraint?: unknown }; - if (code !== UNIQUE_VIOLATION || dropped.length === 0) return message; + if (valueRefused(err)) { + return ( + `${message}\nThe move let through a value of ${spec.label} that Postgres refuses. It should have refused ` + + 'that value by table, row and column before writing anything, so this is a bug in TREK: please report it ' + + 'with this message.' + ); + } + if (!brokeConstraint(err)) return message; + const fix = spec.exact ? FIX_IN_PLUGIN_DB : FIX_IN_SQLITE_FILE; + if (sqlState(err) !== UNIQUE_VIOLATION) { + return ( + `${message}\nA row of ${spec.label} breaks a rule of its Postgres table that the SQLite file did not hold ` + + `it to; the detail above names the failing row or the value it refers to. Correct or remove that row ${fix}` + ); + } + const { constraint } = (err ?? {}) as { constraint?: unknown }; const key = spec.uniqueKeys.find((candidate) => candidate.name === constraint); - const suspects = key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped; - if (suspects.length === 0) return message; - const where = key ? `its column(s) ${key.columns.join(', ')}` : spec.label; - return ( - `${message}\nThe key is broken because --drop-nul removed NUL characters (U+0000) from ${suspects.length} ` + - `value(s) of ${where}, and a value that differed from another only by a NUL equals it once the NUL is gone:\n` + - `${listLines( - suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where), - suspects.length, - )}\n` + - 'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.' + const keyValue = duplicateKeyValue(err); + const suspects = (key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped).filter( + (value) => keyValue === null || value.text === null || keyValue.includes(value.text.toLowerCase()), ); + const differently = key?.expression ? ", or compare differently: SQLite's lower() changes only A to Z" : ''; + const lines = [ + message, + `Two rows of ${spec.label} share ${key ? `a value of its ${describeKey(key)}` : 'a unique key'}, which ` + + `Postgres holds unique and the SQLite file did not (it may lack that index${differently}). The detail ` + + 'above names the value.', + ]; + if (suspects.length > 0) { + lines.push( + `--drop-nul removed NUL characters (U+0000) from ${suspects.length} value(s) of ` + + `${key ? `its ${describeKey(key)}` : spec.label} that may hold it; a value that differed from another ` + + 'only by a NUL equals it once the NUL is gone, so check these first:', + listLines( + suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where), + suspects.length, + ), + ); + } + lines.push(`Change or remove one of those rows ${fix}`); + return lines.join('\n'); } async function insertBatch( @@ -182,7 +280,8 @@ async function insertBatch( try { await run(trx, insertSql(spec, batch.length), batch.flat()); } catch (err) { - throw new Error(refusedRowsMessage(spec, err, dropped), { cause: err }); + const message = refusedRowsMessage(spec, err, dropped); + throw brokeConstraint(err) ? new MoveRefusal(message, { cause: err }) : new Error(message, { cause: err }); } } @@ -238,7 +337,10 @@ export async function copyTable( if (!('reason' in converted)) { values.push(converted.value); if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++; - if (converted.droppedNul) result.nulDropped.push({ where, column: column.name }); + if (converted.droppedNul) { + const text = column.kind === 'text' ? String(converted.value) : null; + result.nulDropped.push({ where, column: column.name, text }); + } return; } values.push(null); diff --git a/server/src/db/engine/postgres/sqlite-move/move.ts b/server/src/db/engine/postgres/sqlite-move/move.ts index 4e94ca3ed..b49faadbd 100644 --- a/server/src/db/engine/postgres/sqlite-move/move.ts +++ b/server/src/db/engine/postgres/sqlite-move/move.ts @@ -26,6 +26,7 @@ import { resolveIdentitySequences, type PluginPlan, } from './plugin-data'; +import { FIX_IN_SQLITE_FILE, MoveRefusal } from './refusal'; import type { AbstractSqlConnection } from '@mikro-orm/sql'; import type { Kysely } from 'kysely'; @@ -63,9 +64,6 @@ import type { Kysely } from 'kysely'; * the database's: they stay where the storage driver keeps them. */ -/** A refusal the operator can act on: printed without a stack. */ -export class MoveRefusal extends Error {} - export interface MoveOptions { /** The SQLite file to move: `data/travel.db` by default. */ sqliteFile: string; @@ -247,10 +245,11 @@ async function copyAll( if (refused > 0) { const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS); const nul = nulRefused > 0 ? `${nulRefused} of them hold a NUL character (U+0000). ${TREK_NUL_REMEDY}\n` : ''; + const others = + refused > nulRefused ? `${nulRefused > 0 ? 'Correct the others' : 'Correct them'} ${FIX_IN_SQLITE_FILE}` : ''; throw new MoveRefusal( `${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` + - `${listLines(problems, refused)}\n${nul}` + - 'Correct those rows (or ask for help with them), then run the move again.', + `${listLines(problems, refused)}\n${nul}${others}`.trimEnd(), ); } diff --git a/server/src/db/engine/postgres/sqlite-move/refusal.ts b/server/src/db/engine/postgres/sqlite-move/refusal.ts new file mode 100644 index 000000000..8e31b618f --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/refusal.ts @@ -0,0 +1,17 @@ +/** + * A refusal the operator can act on: the move stops because of what the + * SQLite file holds, and the message says what to change before running it + * again. The command prints it without a stack, where any other error is a + * failure of the move itself. + */ +export class MoveRefusal extends Error {} + +/** Where and how the operator corrects a row of TREK's own data, after "Correct them" or "Change or remove ...". */ +export const FIX_IN_SQLITE_FILE = + 'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped), then run the move again.'; + +/** The same for a row of a plugin's own data, which lives in the plugin's own SQLite file. */ +export const FIX_IN_PLUGIN_DB = + "in the plugin's plugin.db (through the plugin, or with the sqlite3 tool while TREK is stopped), then run the " + + 'move again.'; diff --git a/server/tests/helpers/fake-postgres-move.ts b/server/tests/helpers/fake-postgres-move.ts index 8641887e0..4b0a1eb0e 100644 --- a/server/tests/helpers/fake-postgres-move.ts +++ b/server/tests/helpers/fake-postgres-move.ts @@ -25,8 +25,12 @@ export interface FakeTableDef { foreignKeys?: Array<{ columns: string[]; references: string }>; /** Columns a sequence fills. */ serial?: string[]; - /** Unique indexes besides the primary key's (`_pkey`). */ - unique?: Array<{ name: string; columns: string[] }>; + /** + * Unique indexes besides the primary key's (`
_pkey`): the columns the + * key reads, and for an expression index its key as Postgres spells it and + * `expression`. + */ + unique?: Array<{ name: string; columns: string[]; keys?: string[]; expression?: boolean }>; } interface FakeTable { @@ -130,9 +134,23 @@ export class FakePostgres { if (sql.includes('from pg_index')) { return this.schema.flatMap((table) => [ ...(table.primaryKey.length > 0 - ? [{ table_name: table.name, index_name: `${table.name}_pkey`, columns: table.primaryKey }] + ? [ + { + table_name: table.name, + index_name: `${table.name}_pkey`, + keys: table.primaryKey, + columns: table.primaryKey, + expression: false, + }, + ] : []), - ...(table.unique ?? []).map((key) => ({ table_name: table.name, index_name: key.name, columns: key.columns })), + ...(table.unique ?? []).map((key) => ({ + table_name: table.name, + index_name: key.name, + keys: key.keys ?? key.columns, + columns: key.columns, + expression: key.expression ?? false, + })), ]); } if (sql.includes("pg_get_serial_sequence(format('%I'")) { diff --git a/server/tests/postgres/sqlite-move-nul.test.ts b/server/tests/postgres/sqlite-move-nul.test.ts index 838d8f3b3..de14ade52 100644 --- a/server/tests/postgres/sqlite-move-nul.test.ts +++ b/server/tests/postgres/sqlite-move-nul.test.ts @@ -1,16 +1,27 @@ /** - * PGMOVENUL-001..005: the move into Postgres (`src/db/engine/postgres/sqlite-move/`) + * PGMOVENUL-001..010: the move into Postgres (`src/db/engine/postgres/sqlite-move/`) * changes nothing silently, on the lane's server. A plugin value holding a NUL * or bytes that are not UTF-8 refuses the move before anything is written; a * NUL in TREK's own text refuses it unless `dropNul` (`--drop-nul`) is given; - * a unique key that dropping a NUL breaks is named as that, with the + * a unique key that dropping a NUL may have broken lists that value, with the * constraint Postgres reports; a move with `dropNul` lists every value it * changed and stores the value TREK would have written, JSON with nothing * else changed; and TREK's own text that is not UTF-8 is refused either way. + * A key over an expression (`lower(email)`) that a dropped NUL breaks lists + * only the dropped values of the columns its expression reads, and a raw NUL + * in JSON text is refused as a NUL and dropped like an escaped one. Two rows + * a Postgres key holds equal while the SQLite file held them apart are a + * refusal that names the key and says what to change, and a value that lost + * a NUL in a column of that key is listed only when the duplicate value holds + * it. The catalog names a key over an expression as one, and a plain key over + * a quoted column by that column. */ import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; -import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { readTargetSchema } from '../../src/db/engine/postgres/sqlite-move/catalog'; +import { inOrder, run } from '../../src/db/engine/postgres/sqlite-move/copy'; +import { moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { MoveRefusal } from '../../src/db/engine/postgres/sqlite-move/refusal'; import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; import { knownMigrationNames } from '../../src/db/known-migrations'; import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm'; @@ -18,6 +29,7 @@ import { createBoundSqliteDriver } from '../../src/db/orm-driver'; import { makeUser } from '../helpers/factories/users'; import { withAdmin } from './databases'; import { MikroORM, type Options } from '@mikro-orm/core'; +import type { AbstractSqlConnection } from '@mikro-orm/sql'; import Database from 'better-sqlite3'; import fs from 'node:fs'; @@ -140,12 +152,12 @@ describe('the move changes nothing silently', () => { expect(await tablesInTarget()).toBe(0); }); - it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks is named, with the constraint', async () => { + it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks lists that value, with the constraint', async () => { await expect(move(true)).rejects.toThrow( new RegExp( 'Postgres refused rows of users: .*constraint users_username_unique[\\s\\S]*' + - 'The key is broken because --drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its ' + - `column\\(s\\) username,[\\s\\S]* - users row id=${movedId}, column username\\n`, + '--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) username ' + + `that may hold it;[\\s\\S]* - users row id=${movedId}, column username\\n`, ), ); expect(await tablesInTarget()).toBe(0); @@ -181,5 +193,132 @@ describe('the move changes nothing silently', () => { 'app_settings row key=move-bytes, column value: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)', ); expect(await tablesInTarget()).toBe(0); + handle.exec("delete from app_settings where key = 'move-bytes'"); + }); + + it('PGMOVENUL-006: a key over an expression that a dropped NUL breaks lists only the values its expression reads', async () => { + // The row loses a NUL in username and in email; only email is read by lower(email), the key that breaks. + handle.exec( + "update users set email = 'Same@x.test' where username = 'movera';" + + `update users set username = 'mover' || char(0) || 'c', email = 'same' || char(0) || '@x.test' where id = ${movedId}`, + ); + const failure = move(true); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + new RegExp( + 'Postgres refused rows of users: .*constraint idx_users_email_lower; ' + + 'Key \\(lower\\(email\\)\\)=\\(same@x\\.test\\) already exists\\.\\n' + + 'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' + + '[^\\n]*\\n' + + '--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) email ' + + '\\(read by its unique key lower\\(email\\)\\) that may hold it; [^\\n]*\\n' + + ` - users row id=${movedId}, column email\\n` + + 'Change or remove one of those rows', + ), + ); + expect(await tablesInTarget()).toBe(0); + }); + + it("PGMOVENUL-007: a raw NUL in TREK's JSON is refused as a NUL, and --drop-nul drops it like an escaped one", async () => { + handle.exec( + `update users set email = 'mover@x.test' where id = ${movedId};` + + `update addons set config = '{"note":"a' || char(0) || 'b",' || char(0) || '"ratio":1.0}' where id = 'collab'`, + ); + const failure = move(); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + 'addons row id=collab, column config: the text "{\\"note\\":\\"a\\u0000b\\",\\u0000\\"ratio\\":1.0}" holds a ' + + 'NUL character (U+0000), which Postgres text cannot hold', + ); + await expect(failure).rejects.toThrow(/Run the move again with --drop-nul/); + expect(await tablesInTarget()).toBe(0); + + const summary = await move(true); + expect(summary.nulDropped).toContain('addons row id=collab, column config'); + const config = await withAdmin(process.env.TREK_DB_URL as string, (run) => + run<{ config: string }>("select config::text as config from addons where id = 'collab'"), + ); + expect(config).toEqual([{ config: '{"note": "ab", "ratio": 1.0}' }]); + }); + + it('PGMOVENUL-008: two rows the SQLite file held apart but a Postgres key does not are a refusal that says what to do', async () => { + await emptyTarget(); + // An install whose case-insensitive email index is still deferred: two accounts differ only in case. + handle.exec( + 'drop index idx_users_email_lower;' + + "update users set email = 'Dup@x.test' where username = 'movera';" + + `update users set email = 'dup@x.test' where id = ${movedId}`, + ); + const failure = move(true); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + new RegExp( + 'constraint idx_users_email_lower; Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' + + 'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' + + 'which Postgres holds unique and the SQLite file did not [\\s\\S]*Change or remove one of those rows in the ' + + 'SQLite file \\(through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped\\), then run the move again\\.$', + ), + ); + expect(await tablesInTarget()).toBe(0); + }); + + it('PGMOVENUL-009: a value that lost a NUL is not blamed for a duplicate that does not hold it', async () => { + await emptyTarget(); + // The two accounts of PGMOVENUL-008 still differ only in case; a third address loses a NUL and collides with nothing. + const { user } = await makeUser(sqliteOrm, { username: 'mover-three' }); + handle.exec(`update users set email = 'other' || char(0) || '@x.test' where id = ${user.id}`); + const failure = move(true); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + new RegExp( + 'Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' + + "Two rows of users share [^\\n]*SQLite's lower\\(\\) changes only A to Z\\)\\. The detail above names the " + + 'value\\.\\nChange or remove one of those rows', + ), + ); + await expect(failure).rejects.not.toThrow(/--drop-nul/); + expect(await tablesInTarget()).toBe(0); + }); + + it('PGMOVENUL-010: the catalog names a key over an expression as one, and a plain key over a quoted column by it', async () => { + const orm = await MikroORM.init({ + ...buildOrmConfig(resolveEngine()), + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + }); + const db = (orm.em.getConnection() as AbstractSqlConnection).getClient(); + const rollback = new Error('roll back'); + let keys: unknown; + try { + await db.transaction().execute(async (trx) => { + // Rolled back below, so the schema never outlives the test. + await inOrder( + [ + 'create schema move_catalog_probe', + 'set local search_path to move_catalog_probe', + 'create table p (id bigint primary key, "userId" text, email text, note text)', + 'create unique index p_user on p ("userId")', + 'create unique index p_email on p (lower(email), "userId") include (note) where note is not null', + ], + async (statement) => { + await run(trx, statement); + }, + ); + const schema = await readTargetSchema(trx); + keys = schema.tables.find((table) => table.name === 'p')?.uniqueKeys; + throw rollback; + }); + } catch (err) { + if (err !== rollback) throw err; + } finally { + await orm.close(true); + } + expect(keys).toEqual([ + { name: 'p_email', key: 'lower(email), "userId"', columns: ['userId', 'email'], expression: true }, + { name: 'p_pkey', key: 'id', columns: ['id'], expression: false }, + { name: 'p_user', key: '"userId"', columns: ['userId'], expression: false }, + ]); }); }); diff --git a/server/tests/postgres/sqlite-move.test.ts b/server/tests/postgres/sqlite-move.test.ts index 884f6dcd4..52b474092 100644 --- a/server/tests/postgres/sqlite-move.test.ts +++ b/server/tests/postgres/sqlite-move.test.ts @@ -15,7 +15,8 @@ import { buildApp } from '../../src/bootstrap'; import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; import { PluginDataSlots, poolClients, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; -import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { MoveRefusal } from '../../src/db/engine/postgres/sqlite-move/refusal'; import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; import { AppSettings } from '../../src/db/entities/AppSettings.entity'; import { BudgetItems } from '../../src/db/entities/BudgetItems.entity'; diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts index c57ab169c..d74055e77 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts @@ -1,5 +1,5 @@ /** - * MOVE-001..026: the move of a SQLite install into Postgres + * MOVE-001..032: the move of a SQLite install into Postgres * (src/db/engine/postgres/sqlite-move/), over real SQLite files and a * stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts): * the refusals before anything is written, the copy in foreign key order with @@ -24,7 +24,6 @@ import { } from '../../../../../../src/db/engine/postgres/sqlite-move/copy'; import { migrationLevelProblem, - MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions, @@ -35,6 +34,7 @@ import { pluginColumnKind, pluginDefault, } from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data'; +import { MoveRefusal } from '../../../../../../src/db/engine/postgres/sqlite-move/refusal'; import { InvalidUtf8Text, sqliteInstallHoldsAccounts, @@ -404,7 +404,7 @@ describe('nothing is changed silently', () => { ); }); - it('MOVE-024: a unique key that dropping a NUL broke is named as that, never as a bare duplicate', async () => { + it('MOVE-024: a unique key a dropped NUL may have broken lists that value, never asserting the cause', async () => { sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 2;"); pg.failOn = /^insert into "users"/; pg.failWith = { @@ -412,17 +412,200 @@ describe('nothing is changed silently', () => { constraint: 'users_name_unique', detail: 'Key (name)=(Ada) already exists.', }; - await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow( + const broken = moveSqliteToPostgres(options({ dropNul: true })); + await expect(broken).rejects.toBeInstanceOf(MoveRefusal); + await expect(broken).rejects.toThrow( 'Postgres refused rows of users: refused by the fake; constraint users_name_unique; Key (name)=(Ada) already ' + - 'exists.\nThe key is broken because --drop-nul removed NUL characters (U+0000) from 1 value(s) of its ' + - 'column(s) name, and a value that differed from another only by a NUL equals it once the NUL is gone:\n' + + 'exists.\nTwo rows of users share a value of its column(s) name, which Postgres holds unique and the SQLite ' + + 'file did not (it may lack that index). The detail above names the value.\n' + + '--drop-nul removed NUL characters (U+0000) from 1 value(s) of its column(s) name that may hold it; a value ' + + 'that differed from another only by a NUL equals it once the NUL is gone, so check these first:\n' + ' - users row id=2, column name\n' + - 'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.', + 'Change or remove one of those rows ' + + 'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped), then run the move again.', ); - // A broken key whose columns lost no NUL is reported as Postgres reported it. + // A broken key whose columns lost no NUL is a duplicate the SQLite file held apart: named, with what to do. pg.failWith = { code: '23505', constraint: 'users_pkey', detail: 'Key (id)=(2) already exists.' }; - await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow( - /^Postgres refused rows of users: refused by the fake; constraint users_pkey; Key \(id\)=\(2\) already exists\.$/, + const duplicate = moveSqliteToPostgres(options({ dropNul: true })); + await expect(duplicate).rejects.toBeInstanceOf(MoveRefusal); + await expect(duplicate).rejects.toThrow( + 'Postgres refused rows of users: refused by the fake; constraint users_pkey; Key (id)=(2) already exists.\n' + + 'Two rows of users share a value of its column(s) id, which Postgres holds unique and the SQLite file did ' + + 'not (it may lack that index). The detail above names the value.\nChange or remove one of those rows in the ' + + 'SQLite file (through TREK', + ); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + }); + + it('MOVE-029: a key over an expression lists exactly the dropped values of the columns its expression reads', () => { + const spec = { + source: 'users', + target: '"users"', + label: 'users', + columns: [], + deferred: [], + primaryKey: ['id'], + exact: false, + uniqueKeys: [ + { name: 'users_pkey', key: 'id', columns: ['id'], expression: false }, + { name: 'idx_users_email_lower', key: 'lower(email)', columns: ['email'], expression: true }, + ], + }; + const dropped = [ + { where: 'users row id=1, column username', column: 'username', text: 'ab@x.test' }, + { where: 'users row id=2, column email', column: 'email', text: 'Ab@x.test' }, + { where: 'users row id=3, column bio', column: 'bio', text: 'ab@x.test' }, + ]; + const err = Object.assign(new Error('duplicate key value violates unique constraint "idx_users_email_lower"'), { + code: '23505', + constraint: 'idx_users_email_lower', + detail: 'Key (lower(email))=(ab@x.test) already exists.', + }); + expect(refusedRowsMessage(spec, err, dropped)).toBe( + 'Postgres refused rows of users: duplicate key value violates unique constraint "idx_users_email_lower"; ' + + 'constraint idx_users_email_lower; Key (lower(email))=(ab@x.test) already exists.\n' + + 'Two rows of users share a value of its column(s) email (read by its unique key lower(email)), which ' + + "Postgres holds unique and the SQLite file did not (it may lack that index, or compare differently: SQLite's " + + 'lower() changes only A to Z). The detail above names the value.\n' + + '--drop-nul removed NUL characters (U+0000) from 1 value(s) of its column(s) email (read by its unique key ' + + 'lower(email)) that may hold it; a value that differed from another only by a NUL equals it once the NUL is ' + + 'gone, so check these first:\n' + + ' - users row id=2, column email\n' + + 'Change or remove one of those rows ' + + 'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped), then run the move again.', + ); + // Without a dropped value in that column, it is a duplicate the SQLite file held apart. + expect(refusedRowsMessage(spec, err, dropped.slice(0, 1))).toContain( + 'Two rows of users share a value of its column(s) email (read by its unique key lower(email)), which Postgres ' + + "holds unique and the SQLite file did not (it may lack that index, or compare differently: SQLite's lower() " + + 'changes only A to Z). The detail above names the value.\nChange or remove one of those rows', + ); + }); + + it('MOVE-033: a dropped NUL in a column the key reads is not blamed when the duplicate value does not hold it', () => { + const spec = { + source: 'users', + target: '"users"', + label: 'users', + columns: [], + deferred: [], + primaryKey: ['id'], + exact: false, + uniqueKeys: [{ name: 'idx_users_email_lower', key: 'lower(email)', columns: ['email'], expression: true }], + }; + // Two accounts differ only in case (the pending index); an unrelated address lost a NUL. + const unrelated = { where: 'users row id=3, column email', column: 'email', text: 'other@x.test' }; + const err = Object.assign(new Error('duplicate key'), { + code: '23505', + constraint: 'idx_users_email_lower', + detail: 'Key (lower(email))=(dup@x.test) already exists.', + }); + const message = refusedRowsMessage(spec, err, [unrelated]); + expect(message).not.toContain('--drop-nul'); + expect(message).not.toContain('row id=3'); + expect(message).toContain("compare differently: SQLite's lower() changes only A to Z). The detail above names"); + // A value whose NUL-free text the duplicate holds is listed, whatever its case. + const related = { where: 'users row id=4, column email', column: 'email', text: 'DUP@x.test' }; + expect(refusedRowsMessage(spec, err, [unrelated, related])).toMatch( + /from 1 value\(s\) of its column\(s\) email .*check these first:\n {2}- users row id=4, column email\nChange/, + ); + // JSON, which jsonb reformats, and a detail of another shape cannot rule a value out: it is listed. + const json = { where: 'users row id=5, column email', column: 'email', text: null }; + expect(refusedRowsMessage(spec, err, [json])).toContain(' - users row id=5, column email\n'); + const localized = Object.assign(new Error('doppelter Schlüsselwert'), { + code: '23505', + constraint: 'idx_users_email_lower', + detail: 'Schlüssel »(lower(email))=(dup@x.test)« existiert bereits.', + }); + expect(refusedRowsMessage(spec, localized, [unrelated])).toContain(' - users row id=3, column email\n'); + }); + + it('MOVE-034: a plain key over a quoted column is named by its column, without the hint about lower()', () => { + const spec = { + source: 'p', + target: '"trek_plugin_p"."p"', + label: 'plugin p: p', + columns: [], + deferred: [], + primaryKey: [], + exact: true, + uniqueKeys: [{ name: 'p_user', key: '"userId"', columns: ['userId'], expression: false }], + }; + const err = Object.assign(new Error('duplicate key'), { + code: '23505', + constraint: 'p_user', + detail: 'Key ("userId")=(7) already exists.', + }); + expect(refusedRowsMessage(spec, err, [])).toBe( + 'Postgres refused rows of plugin p: p: duplicate key; constraint p_user; Key ("userId")=(7) already exists.\n' + + 'Two rows of plugin p: p share a value of its column(s) userId, which Postgres holds unique and the SQLite ' + + 'file did not (it may lack that index). The detail above names the value.\n' + + "Change or remove one of those rows in the plugin's plugin.db (through the plugin, or with the sqlite3 tool " + + 'while TREK is stopped), then run the move again.', + ); + }); + + it('MOVE-035: a value Postgres refuses at insert is a failure reported as a gap in the move, not a refusal', async () => { + pg.failOn = /^insert into "trips"/; + pg.failWith = { code: '22P02' }; + const failure = moveSqliteToPostgres(options()); + await expect(failure).rejects.not.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + /^Postgres refused rows of trips: refused by the fake\nThe move let through a value of trips that Postgres refuses\. It should have refused that value by table, row and column before writing anything, so this is a bug in TREK: please report it with this message\.$/, + ); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + }); + + it('MOVE-030: a row breaking a constraint is a refusal that says what to do; anything else stays a failure', async () => { + pg.failOn = /^insert into "trips"/; + pg.failWith = { + code: '23514', + constraint: 'trips_score_check', + detail: 'Failing row contains (2, 2, Porto, null, null, null).', + }; + const check = moveSqliteToPostgres(options()); + await expect(check).rejects.toBeInstanceOf(MoveRefusal); + await expect(check).rejects.toThrow( + 'Postgres refused rows of trips: refused by the fake; constraint trips_score_check; Failing row contains ' + + '(2, 2, Porto, null, null, null).\nA row of trips breaks a rule of its Postgres table that the SQLite file ' + + 'did not hold it to; the detail above names the failing row or the value it refers to. Correct or remove ' + + 'that row ' + + 'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped), then run the move again.', + ); + // A connection or a statement of the move's own is not the file's: no remedy for the file, no refusal. + pg.failWith = { code: '08006', detail: 'connection lost' }; + const lost = moveSqliteToPostgres(options()); + await expect(lost).rejects.toThrow(/^Postgres refused rows of trips: refused by the fake; connection lost$/); + await expect(lost).rejects.not.toBeInstanceOf(MoveRefusal); + }); + + it("MOVE-031: a raw NUL in TREK's JSON is refused as a NUL with --drop-nul named, and with it dropped and listed", async () => { + sqlite(path.join(dir, 'travel.db'), `update trips set meta = '{"k":"a' || char(0) || 'b"}' where id = 1;`); + const failure = moveSqliteToPostgres(options()); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + '1 value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n' + + ' - trips row id=1, column meta: the text "{\\"k\\":\\"a\\u0000b\\"}" holds a NUL character (U+0000), ' + + 'which Postgres text cannot hold\n' + + '1 of them hold a NUL character (U+0000). TREK no longer stores this character and drops it on its next ' + + 'write. Run the move again with --drop-nul to drop it now; the summary then lists every value it changed.', + ); + const summary = await moveSqliteToPostgres(options({ dropNul: true })); + expect(pg.rowsOf('"trips"')[0][3]).toBe('{"k":"ab"}'); + expect(summary.nulDropped).toEqual(['trips row id=1, column meta']); + }); + + it('MOVE-032: a refusal of other values says how to correct them, after the NUL remedy when both occur', async () => { + sqlite(path.join(dir, 'travel.db'), "update trips set score = 'high' where id = 2;"); + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + /column score: the text "high" is not a number\nCorrect them in the SQLite file \(through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is stopped\), then run the move again\.$/, + ); + sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 1;"); + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + /lists every value it changed\.\nCorrect the others in the SQLite file \(through TREK/, ); }); @@ -437,13 +620,19 @@ describe('nothing is changed silently', () => { exact: false, uniqueKeys: [], }; - const dropped = [{ where: 't row 1, column a', column: 'a' }]; + const dropped = [{ where: 't row 1, column a', column: 'a', text: 'x' }]; const err = Object.assign(new Error('duplicate key'), { code: '23505', constraint: 'gone' }); expect(refusedRowsMessage(spec, err, dropped)).toMatch( - /from 1 value\(s\) of t, and a value .*\n {2}- t row 1, column a\n/, + /from 1 value\(s\) of t that may hold it; a value .*\n {2}- t row 1, column a\n/, ); expect(refusedRowsMessage(spec, new Error('other'), dropped)).toBe('Postgres refused rows of t: other'); - expect(refusedRowsMessage(spec, err, [])).toBe('Postgres refused rows of t: duplicate key; constraint gone'); + expect(refusedRowsMessage(spec, err, [])).toBe( + 'Postgres refused rows of t: duplicate key; constraint gone\nTwo rows of t share a unique key, which Postgres ' + + 'holds unique and the SQLite file did not (it may lack that index). The detail above names the value.\n' + + 'Change or remove one of those rows ' + + 'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' + + 'stopped), then run the move again.', + ); }); }); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts index f55727f70..c063b736d 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts @@ -76,7 +76,7 @@ describe('planTrekTables', () => { { name: 'reply_to', dataType: 'bigint', nullable: true }, ], primaryKey: ['id'], - uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }], + uniqueKeys: [{ name: 'messages_pkey', key: 'id', columns: ['id'], expression: false }], }, { name: 'users', @@ -108,7 +108,7 @@ describe('planTrekTables', () => { primaryKey: ['id'], label: 'messages', exact: false, - uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }], + uniqueKeys: [{ name: 'messages_pkey', key: 'id', columns: ['id'], expression: false }], }); expect(plan.specs[1].columns).toEqual([ { name: 'id', kind: 'integer' }, @@ -168,7 +168,10 @@ describe('readTargetSchema', () => { primaryKey: ['id'], foreignKeys: [{ columns: ['user_id'], references: 'users' }], serial: ['id'], - unique: [{ name: 'trips_user_id_unique', columns: ['user_id'] }], + unique: [ + { name: 'trips_user_id_unique', columns: ['user_id'] }, + { name: 'trips_user_id_abs', columns: ['user_id'], keys: ['abs(user_id)'], expression: true }, + ], }, { name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] }, ]); @@ -182,15 +185,16 @@ describe('readTargetSchema', () => { { name: 'user_id', dataType: 'bigint', nullable: true }, ], uniqueKeys: [ - { name: 'trips_pkey', columns: ['id'] }, - { name: 'trips_user_id_unique', columns: ['user_id'] }, + { name: 'trips_pkey', key: 'id', columns: ['id'], expression: false }, + { name: 'trips_user_id_unique', key: 'user_id', columns: ['user_id'], expression: false }, + { name: 'trips_user_id_abs', key: 'abs(user_id)', columns: ['user_id'], expression: true }, ], }, { name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }], - uniqueKeys: [{ name: 'users_pkey', columns: ['id'] }], + uniqueKeys: [{ name: 'users_pkey', key: 'id', columns: ['id'], expression: false }], }, ]); expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]);