mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 03:26:07 +00:00
fix(db): trace a broken key in the move without blaming the wrong row
The move read unique indexes by joining indkey to pg_attribute. An expression key such as lower(email) has attnum 0 there, so idx_users_email_lower was missing from the plan. The catalog query now reads each index's key as Postgres prints it, whether any part of it is an expression, and the columns it reads: the plain key columns and the Var nodes of indexprs, without INCLUDE columns or the predicate. A broken unique key no longer names --drop-nul as its cause. The refusal names the key and the shared value from Postgres's detail. It explains that the SQLite file held the two rows apart, and the hint about SQLite's lower() appears only for an expression key. Values --drop-nul changed in the key's columns are listed as rows to check first, and only when the duplicate value holds their text. An unrelated value that lost a NUL is no longer named. A plain key over a quoted column is named by its column, not as an expression. A row that breaks a constraint (SQLSTATE class 23) is now a MoveRefusal that says what to correct, in the SQLite file or in a plugin's plugin.db. Before, it was a bare error with a stack. A value Postgres refuses by its type (class 22) stays a failure: it names no row, and values.ts should have refused it first, so the message asks for a bug report. MoveRefusal and the remedy text move to refusal.ts so copy.ts can use them without an import cycle. The usage text of --drop-nul and the wiki's move section describe NUL in JSON, the key listing and these refusals.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
|
||||
import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx, type UniqueKey } from './copy';
|
||||
import { planCopyOrder, type ForeignKey } from './order';
|
||||
import { columnKind } from './values';
|
||||
|
||||
@@ -23,8 +23,8 @@ interface TargetTable {
|
||||
name: string;
|
||||
columns: TargetColumn[];
|
||||
primaryKey: string[];
|
||||
/** Every unique index (the primary key's among them) over plain columns, by name. */
|
||||
uniqueKeys: Array<{ name: string; columns: string[] }>;
|
||||
/** Every unique index (the primary key's among them), by name, expression indexes included. */
|
||||
uniqueKeys: UniqueKey[];
|
||||
}
|
||||
|
||||
export interface TargetSchema {
|
||||
@@ -59,14 +59,30 @@ export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
|
||||
"and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name",
|
||||
);
|
||||
|
||||
// A unique constraint is backed by an index of its name, so the index names what a refusal reports.
|
||||
const uniqueIndexes = await run<{ table_name: string; index_name: string; columns: string[] }>(
|
||||
// A unique constraint is backed by an index of its name, so the index names what a refusal reports. Its key
|
||||
// is read as Postgres spells it. `indkey` holds 0 for a key that is an expression (`lower(email)`), which
|
||||
// marks the index as an expression index; the columns an expression reads are the Var nodes of `indexprs`,
|
||||
// listed after the plain key columns. The partial index's predicate and the INCLUDE columns are left out,
|
||||
// since neither decides whether two rows collide.
|
||||
const uniqueIndexes = await run<{
|
||||
table_name: string;
|
||||
index_name: string;
|
||||
keys: string[];
|
||||
columns: string[];
|
||||
expression: boolean;
|
||||
}>(
|
||||
trx,
|
||||
'select t.relname as table_name, i.relname as index_name, array_agg(a.attname::text order by k.ord) as columns ' +
|
||||
'select t.relname as table_name, i.relname as index_name, ' +
|
||||
'0 = any ((x.indkey::int2[])[0:x.indnkeyatts - 1]) as expression, ' +
|
||||
'array(select pg_get_indexdef(x.indexrelid, k, true) from generate_series(1, x.indnkeyatts::int) as k ' +
|
||||
'order by k) as keys, ' +
|
||||
'array(select a.attname::text from pg_attribute a where a.attrelid = x.indrelid and a.attnum > 0 and (' +
|
||||
'a.attnum = any ((x.indkey::int2[])[0:x.indnkeyatts - 1]) or a.attnum::text in (select m[1] from ' +
|
||||
"regexp_matches(coalesce(x.indexprs::text, ''), ':varattno ([0-9]+)', 'g') as m)) " +
|
||||
'order by coalesce(array_position((x.indkey::int2[])[0:x.indnkeyatts - 1], a.attnum), ' +
|
||||
'x.indnkeyatts + a.attnum)) as columns ' +
|
||||
'from pg_index x join pg_class i on i.oid = x.indexrelid join pg_class t on t.oid = x.indrelid ' +
|
||||
'cross join lateral unnest(x.indkey) with ordinality as k(attnum, ord) ' +
|
||||
'join pg_attribute a on a.attrelid = x.indrelid and a.attnum = k.attnum ' +
|
||||
'where x.indisunique and t.relnamespace = current_schema()::regnamespace group by t.relname, i.relname',
|
||||
'where x.indisunique and t.relnamespace = current_schema()::regnamespace order by t.relname, i.relname',
|
||||
);
|
||||
|
||||
const tables = new Map<string, TargetTable>();
|
||||
@@ -85,7 +101,12 @@ export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
|
||||
tables.set(column.table_name, table);
|
||||
}
|
||||
for (const index of uniqueIndexes) {
|
||||
tables.get(index.table_name)?.uniqueKeys.push({ name: index.index_name, columns: index.columns });
|
||||
tables.get(index.table_name)?.uniqueKeys.push({
|
||||
name: index.index_name,
|
||||
key: index.keys.join(', '),
|
||||
columns: index.columns,
|
||||
expression: index.expression,
|
||||
});
|
||||
}
|
||||
const foreignKeys: ForeignKey[] = [];
|
||||
for (const constraint of constraints) {
|
||||
|
||||
@@ -5,7 +5,8 @@ import type { AnyOrm } from '../../../orm';
|
||||
import { buildOrmConfig, resolveEngine } from '../../index';
|
||||
import { redactUrl } from '../../redact-url';
|
||||
import { readPostgresBaseline } from '../baseline';
|
||||
import { MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions } from './move';
|
||||
import { moveSqliteToPostgres, renderSummary, type MoveOptions } from './move';
|
||||
import { MoveRefusal } from './refusal';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
|
||||
import path from 'node:path';
|
||||
@@ -23,8 +24,9 @@ export const USAGE =
|
||||
'Usage: move-to-postgres [--from=<SQLite file>] [--drop-nul]\n' +
|
||||
'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.\n' +
|
||||
' --from=<file> the SQLite file to copy\n' +
|
||||
" --drop-nul drop NUL characters (U+0000) from TREK's own text, which Postgres cannot hold and TREK drops on\n" +
|
||||
' its next write anyway, instead of refusing the move; the summary lists every value changed.\n' +
|
||||
" --drop-nul drop NUL characters (U+0000) from TREK's own text and JSON (raw or escaped as \\u0000), which\n" +
|
||||
' Postgres cannot hold and TREK drops on its next write anyway, instead of refusing the move;\n' +
|
||||
' the summary lists every value changed.\n' +
|
||||
" A plugin's own data is never changed: a NUL there refuses the move either way.";
|
||||
|
||||
export interface MoveCommandIo {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { SqliteSource } from '../../sqlite/sqlite-source';
|
||||
import { FIX_IN_PLUGIN_DB, FIX_IN_SQLITE_FILE, MoveRefusal } from './refusal';
|
||||
import {
|
||||
beyondSafeInteger,
|
||||
canonicalFromPostgres,
|
||||
@@ -35,6 +36,20 @@ export const MAX_LISTED_PROBLEMS = 20;
|
||||
/** Postgres's SQLSTATE for a row that would break a unique key. */
|
||||
const UNIQUE_VIOLATION = '23505';
|
||||
|
||||
/**
|
||||
* A unique index of a Postgres table. `key` is its key as Postgres spells it
|
||||
* (`username`, or `lower(email)` for an expression), `columns` every column
|
||||
* the key reads, so a broken key can be traced to the values that changed.
|
||||
* `expression` is whether any part of the key is an expression rather than a
|
||||
* plain column.
|
||||
*/
|
||||
export interface UniqueKey {
|
||||
name: string;
|
||||
key: string;
|
||||
columns: string[];
|
||||
expression: boolean;
|
||||
}
|
||||
|
||||
/** One column of a table the move copies. */
|
||||
interface CopyColumn {
|
||||
name: string;
|
||||
@@ -57,7 +72,7 @@ export interface TableSpec {
|
||||
/** A plugin's own table, whose values must arrive exactly as stored (`PLUGIN_RULES` in `values.ts`). */
|
||||
exact: boolean;
|
||||
/** The unique keys of the Postgres table (the primary key among them), by constraint or index name. */
|
||||
uniqueKeys: Array<{ name: string; columns: string[] }>;
|
||||
uniqueKeys: UniqueKey[];
|
||||
}
|
||||
|
||||
/** A value from which `--drop-nul` removed NUL characters, for the summary. */
|
||||
@@ -65,6 +80,12 @@ interface DroppedNul {
|
||||
/** The table, row and column, named as a refusal names a value. */
|
||||
where: string;
|
||||
column: string;
|
||||
/**
|
||||
* The text without its NUL characters, as Postgres spells it in a broken
|
||||
* key's detail, or null where it spells it otherwise (JSON, which jsonb
|
||||
* reformats).
|
||||
*/
|
||||
text: string | null;
|
||||
}
|
||||
|
||||
/** A row's deferred values, kept until every table is in. */
|
||||
@@ -147,29 +168,106 @@ export function listLines(lines: readonly string[], total = lines.length): strin
|
||||
return shown.join('\n');
|
||||
}
|
||||
|
||||
/** The SQLSTATE of a Postgres error, when it carries one. */
|
||||
function sqlState(err: unknown): string {
|
||||
const { code } = (err ?? {}) as { code?: unknown };
|
||||
return typeof code === 'string' ? code : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Why Postgres refused rows of a table. A unique key broken in a table whose
|
||||
* values lost NUL characters to `--drop-nul` is named as that: two values
|
||||
* that differed only by a NUL are equal once it is gone, which the SQLite
|
||||
* file's own key never saw.
|
||||
* Whether Postgres refused rows over a constraint (SQLSTATE class 23: a
|
||||
* unique, check, not-null or foreign key violation). Its detail names the
|
||||
* key or the failing row, which the operator corrects in the SQLite file.
|
||||
*/
|
||||
function brokeConstraint(err: unknown): boolean {
|
||||
return sqlState(err).startsWith('23');
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether Postgres refused a value its column cannot hold (SQLSTATE class
|
||||
* 22). `values.ts` refuses every such value by table, row and column before
|
||||
* anything is written, so one that reaches Postgres is a gap in the move, not
|
||||
* something the operator can find in the file: Postgres does not say which
|
||||
* row of a batch it was.
|
||||
*/
|
||||
function valueRefused(err: unknown): boolean {
|
||||
return sqlState(err).startsWith('22');
|
||||
}
|
||||
|
||||
/**
|
||||
* The key value of a unique violation's detail (`Key (lower(email))=(ab@x.test)
|
||||
* already exists.`), lower-cased, or null when the detail has another shape
|
||||
* (none, or the server's messages are in another language).
|
||||
*/
|
||||
function duplicateKeyValue(err: unknown): string | null {
|
||||
const { detail } = (err ?? {}) as { detail?: unknown };
|
||||
if (typeof detail !== 'string') return null;
|
||||
const match = /^Key \(.*?\)=\((.*)\) already exists\.$/s.exec(detail);
|
||||
return match ? match[1].toLowerCase() : null;
|
||||
}
|
||||
|
||||
/** A unique key as a refusal names it: its columns, and for an expression also the key as Postgres spells it. */
|
||||
function describeKey(key: UniqueKey): string {
|
||||
const columns = `column(s) ${key.columns.join(', ')}`;
|
||||
return key.expression ? `${columns} (read by its unique key ${key.key})` : columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why Postgres refused rows of a table, and what to do. A broken unique key
|
||||
* is two rows the SQLite file held apart: it may lack that index, compare
|
||||
* differently, or have held two values that differed only by a NUL until
|
||||
* `--drop-nul` removed it. Only Postgres's detail tells which, so the refusal
|
||||
* never asserts a cause: it names the key and the value, and lists the values
|
||||
* `--drop-nul` changed in the columns the key reads (for an expression index,
|
||||
* the columns its expression reads) whose text the duplicate value holds.
|
||||
* That test ignores case, so a key over lower() still finds them; it may keep
|
||||
* a value that is not one of the two rows, but never leaves out one that is.
|
||||
* Any other broken constraint is a row Postgres names in its detail. A value
|
||||
* its column refuses is a gap in the move, reported as one.
|
||||
*/
|
||||
export function refusedRowsMessage(spec: TableSpec, err: unknown, dropped: readonly DroppedNul[]): string {
|
||||
const message = `Postgres refused rows of ${spec.label}: ${describePgError(err)}`;
|
||||
const { code, constraint } = (err ?? {}) as { code?: unknown; constraint?: unknown };
|
||||
if (code !== UNIQUE_VIOLATION || dropped.length === 0) return message;
|
||||
if (valueRefused(err)) {
|
||||
return (
|
||||
`${message}\nThe move let through a value of ${spec.label} that Postgres refuses. It should have refused ` +
|
||||
'that value by table, row and column before writing anything, so this is a bug in TREK: please report it ' +
|
||||
'with this message.'
|
||||
);
|
||||
}
|
||||
if (!brokeConstraint(err)) return message;
|
||||
const fix = spec.exact ? FIX_IN_PLUGIN_DB : FIX_IN_SQLITE_FILE;
|
||||
if (sqlState(err) !== UNIQUE_VIOLATION) {
|
||||
return (
|
||||
`${message}\nA row of ${spec.label} breaks a rule of its Postgres table that the SQLite file did not hold ` +
|
||||
`it to; the detail above names the failing row or the value it refers to. Correct or remove that row ${fix}`
|
||||
);
|
||||
}
|
||||
const { constraint } = (err ?? {}) as { constraint?: unknown };
|
||||
const key = spec.uniqueKeys.find((candidate) => candidate.name === constraint);
|
||||
const suspects = key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped;
|
||||
if (suspects.length === 0) return message;
|
||||
const where = key ? `its column(s) ${key.columns.join(', ')}` : spec.label;
|
||||
return (
|
||||
`${message}\nThe key is broken because --drop-nul removed NUL characters (U+0000) from ${suspects.length} ` +
|
||||
`value(s) of ${where}, and a value that differed from another only by a NUL equals it once the NUL is gone:\n` +
|
||||
`${listLines(
|
||||
suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where),
|
||||
suspects.length,
|
||||
)}\n` +
|
||||
'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.'
|
||||
const keyValue = duplicateKeyValue(err);
|
||||
const suspects = (key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped).filter(
|
||||
(value) => keyValue === null || value.text === null || keyValue.includes(value.text.toLowerCase()),
|
||||
);
|
||||
const differently = key?.expression ? ", or compare differently: SQLite's lower() changes only A to Z" : '';
|
||||
const lines = [
|
||||
message,
|
||||
`Two rows of ${spec.label} share ${key ? `a value of its ${describeKey(key)}` : 'a unique key'}, which ` +
|
||||
`Postgres holds unique and the SQLite file did not (it may lack that index${differently}). The detail ` +
|
||||
'above names the value.',
|
||||
];
|
||||
if (suspects.length > 0) {
|
||||
lines.push(
|
||||
`--drop-nul removed NUL characters (U+0000) from ${suspects.length} value(s) of ` +
|
||||
`${key ? `its ${describeKey(key)}` : spec.label} that may hold it; a value that differed from another ` +
|
||||
'only by a NUL equals it once the NUL is gone, so check these first:',
|
||||
listLines(
|
||||
suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where),
|
||||
suspects.length,
|
||||
),
|
||||
);
|
||||
}
|
||||
lines.push(`Change or remove one of those rows ${fix}`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function insertBatch(
|
||||
@@ -182,7 +280,8 @@ async function insertBatch(
|
||||
try {
|
||||
await run(trx, insertSql(spec, batch.length), batch.flat());
|
||||
} catch (err) {
|
||||
throw new Error(refusedRowsMessage(spec, err, dropped), { cause: err });
|
||||
const message = refusedRowsMessage(spec, err, dropped);
|
||||
throw brokeConstraint(err) ? new MoveRefusal(message, { cause: err }) : new Error(message, { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,7 +337,10 @@ export async function copyTable(
|
||||
if (!('reason' in converted)) {
|
||||
values.push(converted.value);
|
||||
if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++;
|
||||
if (converted.droppedNul) result.nulDropped.push({ where, column: column.name });
|
||||
if (converted.droppedNul) {
|
||||
const text = column.kind === 'text' ? String(converted.value) : null;
|
||||
result.nulDropped.push({ where, column: column.name, text });
|
||||
}
|
||||
return;
|
||||
}
|
||||
values.push(null);
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
resolveIdentitySequences,
|
||||
type PluginPlan,
|
||||
} from './plugin-data';
|
||||
import { FIX_IN_SQLITE_FILE, MoveRefusal } from './refusal';
|
||||
import type { AbstractSqlConnection } from '@mikro-orm/sql';
|
||||
|
||||
import type { Kysely } from 'kysely';
|
||||
@@ -63,9 +64,6 @@ import type { Kysely } from 'kysely';
|
||||
* the database's: they stay where the storage driver keeps them.
|
||||
*/
|
||||
|
||||
/** A refusal the operator can act on: printed without a stack. */
|
||||
export class MoveRefusal extends Error {}
|
||||
|
||||
export interface MoveOptions {
|
||||
/** The SQLite file to move: `data/travel.db` by default. */
|
||||
sqliteFile: string;
|
||||
@@ -247,10 +245,11 @@ async function copyAll(
|
||||
if (refused > 0) {
|
||||
const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS);
|
||||
const nul = nulRefused > 0 ? `${nulRefused} of them hold a NUL character (U+0000). ${TREK_NUL_REMEDY}\n` : '';
|
||||
const others =
|
||||
refused > nulRefused ? `${nulRefused > 0 ? 'Correct the others' : 'Correct them'} ${FIX_IN_SQLITE_FILE}` : '';
|
||||
throw new MoveRefusal(
|
||||
`${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` +
|
||||
`${listLines(problems, refused)}\n${nul}` +
|
||||
'Correct those rows (or ask for help with them), then run the move again.',
|
||||
`${listLines(problems, refused)}\n${nul}${others}`.trimEnd(),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* A refusal the operator can act on: the move stops because of what the
|
||||
* SQLite file holds, and the message says what to change before running it
|
||||
* again. The command prints it without a stack, where any other error is a
|
||||
* failure of the move itself.
|
||||
*/
|
||||
export class MoveRefusal extends Error {}
|
||||
|
||||
/** Where and how the operator corrects a row of TREK's own data, after "Correct them" or "Change or remove ...". */
|
||||
export const FIX_IN_SQLITE_FILE =
|
||||
'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped), then run the move again.';
|
||||
|
||||
/** The same for a row of a plugin's own data, which lives in the plugin's own SQLite file. */
|
||||
export const FIX_IN_PLUGIN_DB =
|
||||
"in the plugin's plugin.db (through the plugin, or with the sqlite3 tool while TREK is stopped), then run the " +
|
||||
'move again.';
|
||||
@@ -25,8 +25,12 @@ export interface FakeTableDef {
|
||||
foreignKeys?: Array<{ columns: string[]; references: string }>;
|
||||
/** Columns a sequence fills. */
|
||||
serial?: string[];
|
||||
/** Unique indexes besides the primary key's (`<table>_pkey`). */
|
||||
unique?: Array<{ name: string; columns: string[] }>;
|
||||
/**
|
||||
* Unique indexes besides the primary key's (`<table>_pkey`): the columns the
|
||||
* key reads, and for an expression index its key as Postgres spells it and
|
||||
* `expression`.
|
||||
*/
|
||||
unique?: Array<{ name: string; columns: string[]; keys?: string[]; expression?: boolean }>;
|
||||
}
|
||||
|
||||
interface FakeTable {
|
||||
@@ -130,9 +134,23 @@ export class FakePostgres {
|
||||
if (sql.includes('from pg_index')) {
|
||||
return this.schema.flatMap((table) => [
|
||||
...(table.primaryKey.length > 0
|
||||
? [{ table_name: table.name, index_name: `${table.name}_pkey`, columns: table.primaryKey }]
|
||||
? [
|
||||
{
|
||||
table_name: table.name,
|
||||
index_name: `${table.name}_pkey`,
|
||||
keys: table.primaryKey,
|
||||
columns: table.primaryKey,
|
||||
expression: false,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(table.unique ?? []).map((key) => ({ table_name: table.name, index_name: key.name, columns: key.columns })),
|
||||
...(table.unique ?? []).map((key) => ({
|
||||
table_name: table.name,
|
||||
index_name: key.name,
|
||||
keys: key.keys ?? key.columns,
|
||||
columns: key.columns,
|
||||
expression: key.expression ?? false,
|
||||
})),
|
||||
]);
|
||||
}
|
||||
if (sql.includes("pg_get_serial_sequence(format('%I'")) {
|
||||
|
||||
@@ -1,16 +1,27 @@
|
||||
/**
|
||||
* PGMOVENUL-001..005: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
||||
* PGMOVENUL-001..010: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
||||
* changes nothing silently, on the lane's server. A plugin value holding a NUL
|
||||
* or bytes that are not UTF-8 refuses the move before anything is written; a
|
||||
* NUL in TREK's own text refuses it unless `dropNul` (`--drop-nul`) is given;
|
||||
* a unique key that dropping a NUL breaks is named as that, with the
|
||||
* a unique key that dropping a NUL may have broken lists that value, with the
|
||||
* constraint Postgres reports; a move with `dropNul` lists every value it
|
||||
* changed and stores the value TREK would have written, JSON with nothing
|
||||
* else changed; and TREK's own text that is not UTF-8 is refused either way.
|
||||
* A key over an expression (`lower(email)`) that a dropped NUL breaks lists
|
||||
* only the dropped values of the columns its expression reads, and a raw NUL
|
||||
* in JSON text is refused as a NUL and dropped like an escaped one. Two rows
|
||||
* a Postgres key holds equal while the SQLite file held them apart are a
|
||||
* refusal that names the key and says what to change, and a value that lost
|
||||
* a NUL in a column of that key is listed only when the duplicate value holds
|
||||
* it. The catalog names a key over an expression as one, and a plain key over
|
||||
* a quoted column by that column.
|
||||
*/
|
||||
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
||||
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
||||
import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
||||
import { readTargetSchema } from '../../src/db/engine/postgres/sqlite-move/catalog';
|
||||
import { inOrder, run } from '../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import { moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
||||
import { MoveRefusal } from '../../src/db/engine/postgres/sqlite-move/refusal';
|
||||
import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine';
|
||||
import { knownMigrationNames } from '../../src/db/known-migrations';
|
||||
import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm';
|
||||
@@ -18,6 +29,7 @@ import { createBoundSqliteDriver } from '../../src/db/orm-driver';
|
||||
import { makeUser } from '../helpers/factories/users';
|
||||
import { withAdmin } from './databases';
|
||||
import { MikroORM, type Options } from '@mikro-orm/core';
|
||||
import type { AbstractSqlConnection } from '@mikro-orm/sql';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
@@ -140,12 +152,12 @@ describe('the move changes nothing silently', () => {
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks is named, with the constraint', async () => {
|
||||
it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks lists that value, with the constraint', async () => {
|
||||
await expect(move(true)).rejects.toThrow(
|
||||
new RegExp(
|
||||
'Postgres refused rows of users: .*constraint users_username_unique[\\s\\S]*' +
|
||||
'The key is broken because --drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its ' +
|
||||
`column\\(s\\) username,[\\s\\S]* - users row id=${movedId}, column username\\n`,
|
||||
'--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) username ' +
|
||||
`that may hold it;[\\s\\S]* - users row id=${movedId}, column username\\n`,
|
||||
),
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
@@ -181,5 +193,132 @@ describe('the move changes nothing silently', () => {
|
||||
'app_settings row key=move-bytes, column value: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
handle.exec("delete from app_settings where key = 'move-bytes'");
|
||||
});
|
||||
|
||||
it('PGMOVENUL-006: a key over an expression that a dropped NUL breaks lists only the values its expression reads', async () => {
|
||||
// The row loses a NUL in username and in email; only email is read by lower(email), the key that breaks.
|
||||
handle.exec(
|
||||
"update users set email = 'Same@x.test' where username = 'movera';" +
|
||||
`update users set username = 'mover' || char(0) || 'c', email = 'same' || char(0) || '@x.test' where id = ${movedId}`,
|
||||
);
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
new RegExp(
|
||||
'Postgres refused rows of users: .*constraint idx_users_email_lower; ' +
|
||||
'Key \\(lower\\(email\\)\\)=\\(same@x\\.test\\) already exists\\.\\n' +
|
||||
'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' +
|
||||
'[^\\n]*\\n' +
|
||||
'--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) email ' +
|
||||
'\\(read by its unique key lower\\(email\\)\\) that may hold it; [^\\n]*\\n' +
|
||||
` - users row id=${movedId}, column email\\n` +
|
||||
'Change or remove one of those rows',
|
||||
),
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it("PGMOVENUL-007: a raw NUL in TREK's JSON is refused as a NUL, and --drop-nul drops it like an escaped one", async () => {
|
||||
handle.exec(
|
||||
`update users set email = 'mover@x.test' where id = ${movedId};` +
|
||||
`update addons set config = '{"note":"a' || char(0) || 'b",' || char(0) || '"ratio":1.0}' where id = 'collab'`,
|
||||
);
|
||||
const failure = move();
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
'addons row id=collab, column config: the text "{\\"note\\":\\"a\\u0000b\\",\\u0000\\"ratio\\":1.0}" holds a ' +
|
||||
'NUL character (U+0000), which Postgres text cannot hold',
|
||||
);
|
||||
await expect(failure).rejects.toThrow(/Run the move again with --drop-nul/);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
|
||||
const summary = await move(true);
|
||||
expect(summary.nulDropped).toContain('addons row id=collab, column config');
|
||||
const config = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
||||
run<{ config: string }>("select config::text as config from addons where id = 'collab'"),
|
||||
);
|
||||
expect(config).toEqual([{ config: '{"note": "ab", "ratio": 1.0}' }]);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-008: two rows the SQLite file held apart but a Postgres key does not are a refusal that says what to do', async () => {
|
||||
await emptyTarget();
|
||||
// An install whose case-insensitive email index is still deferred: two accounts differ only in case.
|
||||
handle.exec(
|
||||
'drop index idx_users_email_lower;' +
|
||||
"update users set email = 'Dup@x.test' where username = 'movera';" +
|
||||
`update users set email = 'dup@x.test' where id = ${movedId}`,
|
||||
);
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
new RegExp(
|
||||
'constraint idx_users_email_lower; Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' +
|
||||
'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' +
|
||||
'which Postgres holds unique and the SQLite file did not [\\s\\S]*Change or remove one of those rows in the ' +
|
||||
'SQLite file \\(through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped\\), then run the move again\\.$',
|
||||
),
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-009: a value that lost a NUL is not blamed for a duplicate that does not hold it', async () => {
|
||||
await emptyTarget();
|
||||
// The two accounts of PGMOVENUL-008 still differ only in case; a third address loses a NUL and collides with nothing.
|
||||
const { user } = await makeUser(sqliteOrm, { username: 'mover-three' });
|
||||
handle.exec(`update users set email = 'other' || char(0) || '@x.test' where id = ${user.id}`);
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
new RegExp(
|
||||
'Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' +
|
||||
"Two rows of users share [^\\n]*SQLite's lower\\(\\) changes only A to Z\\)\\. The detail above names the " +
|
||||
'value\\.\\nChange or remove one of those rows',
|
||||
),
|
||||
);
|
||||
await expect(failure).rejects.not.toThrow(/--drop-nul/);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-010: the catalog names a key over an expression as one, and a plain key over a quoted column by it', async () => {
|
||||
const orm = await MikroORM.init({
|
||||
...buildOrmConfig(resolveEngine()),
|
||||
entities: [],
|
||||
entitiesTs: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
});
|
||||
const db = (orm.em.getConnection() as AbstractSqlConnection).getClient<object>();
|
||||
const rollback = new Error('roll back');
|
||||
let keys: unknown;
|
||||
try {
|
||||
await db.transaction().execute(async (trx) => {
|
||||
// Rolled back below, so the schema never outlives the test.
|
||||
await inOrder(
|
||||
[
|
||||
'create schema move_catalog_probe',
|
||||
'set local search_path to move_catalog_probe',
|
||||
'create table p (id bigint primary key, "userId" text, email text, note text)',
|
||||
'create unique index p_user on p ("userId")',
|
||||
'create unique index p_email on p (lower(email), "userId") include (note) where note is not null',
|
||||
],
|
||||
async (statement) => {
|
||||
await run(trx, statement);
|
||||
},
|
||||
);
|
||||
const schema = await readTargetSchema(trx);
|
||||
keys = schema.tables.find((table) => table.name === 'p')?.uniqueKeys;
|
||||
throw rollback;
|
||||
});
|
||||
} catch (err) {
|
||||
if (err !== rollback) throw err;
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
expect(keys).toEqual([
|
||||
{ name: 'p_email', key: 'lower(email), "userId"', columns: ['userId', 'email'], expression: true },
|
||||
{ name: 'p_pkey', key: 'id', columns: ['id'], expression: false },
|
||||
{ name: 'p_user', key: '"userId"', columns: ['userId'], expression: false },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,7 +15,8 @@ import { buildApp } from '../../src/bootstrap';
|
||||
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
||||
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
||||
import { PluginDataSlots, poolClients, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store';
|
||||
import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
||||
import { moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
||||
import { MoveRefusal } from '../../src/db/engine/postgres/sqlite-move/refusal';
|
||||
import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine';
|
||||
import { AppSettings } from '../../src/db/entities/AppSettings.entity';
|
||||
import { BudgetItems } from '../../src/db/entities/BudgetItems.entity';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVE-001..026: the move of a SQLite install into Postgres
|
||||
* MOVE-001..032: the move of a SQLite install into Postgres
|
||||
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
|
||||
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
|
||||
* the refusals before anything is written, the copy in foreign key order with
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import {
|
||||
migrationLevelProblem,
|
||||
MoveRefusal,
|
||||
moveSqliteToPostgres,
|
||||
renderSummary,
|
||||
type MoveOptions,
|
||||
@@ -35,6 +34,7 @@ import {
|
||||
pluginColumnKind,
|
||||
pluginDefault,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data';
|
||||
import { MoveRefusal } from '../../../../../../src/db/engine/postgres/sqlite-move/refusal';
|
||||
import {
|
||||
InvalidUtf8Text,
|
||||
sqliteInstallHoldsAccounts,
|
||||
@@ -404,7 +404,7 @@ describe('nothing is changed silently', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-024: a unique key that dropping a NUL broke is named as that, never as a bare duplicate', async () => {
|
||||
it('MOVE-024: a unique key a dropped NUL may have broken lists that value, never asserting the cause', async () => {
|
||||
sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 2;");
|
||||
pg.failOn = /^insert into "users"/;
|
||||
pg.failWith = {
|
||||
@@ -412,17 +412,200 @@ describe('nothing is changed silently', () => {
|
||||
constraint: 'users_name_unique',
|
||||
detail: 'Key (name)=(Ada) already exists.',
|
||||
};
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
const broken = moveSqliteToPostgres(options({ dropNul: true }));
|
||||
await expect(broken).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(broken).rejects.toThrow(
|
||||
'Postgres refused rows of users: refused by the fake; constraint users_name_unique; Key (name)=(Ada) already ' +
|
||||
'exists.\nThe key is broken because --drop-nul removed NUL characters (U+0000) from 1 value(s) of its ' +
|
||||
'column(s) name, and a value that differed from another only by a NUL equals it once the NUL is gone:\n' +
|
||||
'exists.\nTwo rows of users share a value of its column(s) name, which Postgres holds unique and the SQLite ' +
|
||||
'file did not (it may lack that index). The detail above names the value.\n' +
|
||||
'--drop-nul removed NUL characters (U+0000) from 1 value(s) of its column(s) name that may hold it; a value ' +
|
||||
'that differed from another only by a NUL equals it once the NUL is gone, so check these first:\n' +
|
||||
' - users row id=2, column name\n' +
|
||||
'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.',
|
||||
'Change or remove one of those rows ' +
|
||||
'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped), then run the move again.',
|
||||
);
|
||||
// A broken key whose columns lost no NUL is reported as Postgres reported it.
|
||||
// A broken key whose columns lost no NUL is a duplicate the SQLite file held apart: named, with what to do.
|
||||
pg.failWith = { code: '23505', constraint: 'users_pkey', detail: 'Key (id)=(2) already exists.' };
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
/^Postgres refused rows of users: refused by the fake; constraint users_pkey; Key \(id\)=\(2\) already exists\.$/,
|
||||
const duplicate = moveSqliteToPostgres(options({ dropNul: true }));
|
||||
await expect(duplicate).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(duplicate).rejects.toThrow(
|
||||
'Postgres refused rows of users: refused by the fake; constraint users_pkey; Key (id)=(2) already exists.\n' +
|
||||
'Two rows of users share a value of its column(s) id, which Postgres holds unique and the SQLite file did ' +
|
||||
'not (it may lack that index). The detail above names the value.\nChange or remove one of those rows in the ' +
|
||||
'SQLite file (through TREK',
|
||||
);
|
||||
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
|
||||
});
|
||||
|
||||
it('MOVE-029: a key over an expression lists exactly the dropped values of the columns its expression reads', () => {
|
||||
const spec = {
|
||||
source: 'users',
|
||||
target: '"users"',
|
||||
label: 'users',
|
||||
columns: [],
|
||||
deferred: [],
|
||||
primaryKey: ['id'],
|
||||
exact: false,
|
||||
uniqueKeys: [
|
||||
{ name: 'users_pkey', key: 'id', columns: ['id'], expression: false },
|
||||
{ name: 'idx_users_email_lower', key: 'lower(email)', columns: ['email'], expression: true },
|
||||
],
|
||||
};
|
||||
const dropped = [
|
||||
{ where: 'users row id=1, column username', column: 'username', text: 'ab@x.test' },
|
||||
{ where: 'users row id=2, column email', column: 'email', text: 'Ab@x.test' },
|
||||
{ where: 'users row id=3, column bio', column: 'bio', text: 'ab@x.test' },
|
||||
];
|
||||
const err = Object.assign(new Error('duplicate key value violates unique constraint "idx_users_email_lower"'), {
|
||||
code: '23505',
|
||||
constraint: 'idx_users_email_lower',
|
||||
detail: 'Key (lower(email))=(ab@x.test) already exists.',
|
||||
});
|
||||
expect(refusedRowsMessage(spec, err, dropped)).toBe(
|
||||
'Postgres refused rows of users: duplicate key value violates unique constraint "idx_users_email_lower"; ' +
|
||||
'constraint idx_users_email_lower; Key (lower(email))=(ab@x.test) already exists.\n' +
|
||||
'Two rows of users share a value of its column(s) email (read by its unique key lower(email)), which ' +
|
||||
"Postgres holds unique and the SQLite file did not (it may lack that index, or compare differently: SQLite's " +
|
||||
'lower() changes only A to Z). The detail above names the value.\n' +
|
||||
'--drop-nul removed NUL characters (U+0000) from 1 value(s) of its column(s) email (read by its unique key ' +
|
||||
'lower(email)) that may hold it; a value that differed from another only by a NUL equals it once the NUL is ' +
|
||||
'gone, so check these first:\n' +
|
||||
' - users row id=2, column email\n' +
|
||||
'Change or remove one of those rows ' +
|
||||
'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped), then run the move again.',
|
||||
);
|
||||
// Without a dropped value in that column, it is a duplicate the SQLite file held apart.
|
||||
expect(refusedRowsMessage(spec, err, dropped.slice(0, 1))).toContain(
|
||||
'Two rows of users share a value of its column(s) email (read by its unique key lower(email)), which Postgres ' +
|
||||
"holds unique and the SQLite file did not (it may lack that index, or compare differently: SQLite's lower() " +
|
||||
'changes only A to Z). The detail above names the value.\nChange or remove one of those rows',
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-033: a dropped NUL in a column the key reads is not blamed when the duplicate value does not hold it', () => {
|
||||
const spec = {
|
||||
source: 'users',
|
||||
target: '"users"',
|
||||
label: 'users',
|
||||
columns: [],
|
||||
deferred: [],
|
||||
primaryKey: ['id'],
|
||||
exact: false,
|
||||
uniqueKeys: [{ name: 'idx_users_email_lower', key: 'lower(email)', columns: ['email'], expression: true }],
|
||||
};
|
||||
// Two accounts differ only in case (the pending index); an unrelated address lost a NUL.
|
||||
const unrelated = { where: 'users row id=3, column email', column: 'email', text: 'other@x.test' };
|
||||
const err = Object.assign(new Error('duplicate key'), {
|
||||
code: '23505',
|
||||
constraint: 'idx_users_email_lower',
|
||||
detail: 'Key (lower(email))=(dup@x.test) already exists.',
|
||||
});
|
||||
const message = refusedRowsMessage(spec, err, [unrelated]);
|
||||
expect(message).not.toContain('--drop-nul');
|
||||
expect(message).not.toContain('row id=3');
|
||||
expect(message).toContain("compare differently: SQLite's lower() changes only A to Z). The detail above names");
|
||||
// A value whose NUL-free text the duplicate holds is listed, whatever its case.
|
||||
const related = { where: 'users row id=4, column email', column: 'email', text: 'DUP@x.test' };
|
||||
expect(refusedRowsMessage(spec, err, [unrelated, related])).toMatch(
|
||||
/from 1 value\(s\) of its column\(s\) email .*check these first:\n {2}- users row id=4, column email\nChange/,
|
||||
);
|
||||
// JSON, which jsonb reformats, and a detail of another shape cannot rule a value out: it is listed.
|
||||
const json = { where: 'users row id=5, column email', column: 'email', text: null };
|
||||
expect(refusedRowsMessage(spec, err, [json])).toContain(' - users row id=5, column email\n');
|
||||
const localized = Object.assign(new Error('doppelter Schlüsselwert'), {
|
||||
code: '23505',
|
||||
constraint: 'idx_users_email_lower',
|
||||
detail: 'Schlüssel »(lower(email))=(dup@x.test)« existiert bereits.',
|
||||
});
|
||||
expect(refusedRowsMessage(spec, localized, [unrelated])).toContain(' - users row id=3, column email\n');
|
||||
});
|
||||
|
||||
it('MOVE-034: a plain key over a quoted column is named by its column, without the hint about lower()', () => {
|
||||
const spec = {
|
||||
source: 'p',
|
||||
target: '"trek_plugin_p"."p"',
|
||||
label: 'plugin p: p',
|
||||
columns: [],
|
||||
deferred: [],
|
||||
primaryKey: [],
|
||||
exact: true,
|
||||
uniqueKeys: [{ name: 'p_user', key: '"userId"', columns: ['userId'], expression: false }],
|
||||
};
|
||||
const err = Object.assign(new Error('duplicate key'), {
|
||||
code: '23505',
|
||||
constraint: 'p_user',
|
||||
detail: 'Key ("userId")=(7) already exists.',
|
||||
});
|
||||
expect(refusedRowsMessage(spec, err, [])).toBe(
|
||||
'Postgres refused rows of plugin p: p: duplicate key; constraint p_user; Key ("userId")=(7) already exists.\n' +
|
||||
'Two rows of plugin p: p share a value of its column(s) userId, which Postgres holds unique and the SQLite ' +
|
||||
'file did not (it may lack that index). The detail above names the value.\n' +
|
||||
"Change or remove one of those rows in the plugin's plugin.db (through the plugin, or with the sqlite3 tool " +
|
||||
'while TREK is stopped), then run the move again.',
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-035: a value Postgres refuses at insert is a failure reported as a gap in the move, not a refusal', async () => {
|
||||
pg.failOn = /^insert into "trips"/;
|
||||
pg.failWith = { code: '22P02' };
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.not.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
/^Postgres refused rows of trips: refused by the fake\nThe move let through a value of trips that Postgres refuses\. It should have refused that value by table, row and column before writing anything, so this is a bug in TREK: please report it with this message\.$/,
|
||||
);
|
||||
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
|
||||
});
|
||||
|
||||
it('MOVE-030: a row breaking a constraint is a refusal that says what to do; anything else stays a failure', async () => {
|
||||
pg.failOn = /^insert into "trips"/;
|
||||
pg.failWith = {
|
||||
code: '23514',
|
||||
constraint: 'trips_score_check',
|
||||
detail: 'Failing row contains (2, 2, Porto, null, null, null).',
|
||||
};
|
||||
const check = moveSqliteToPostgres(options());
|
||||
await expect(check).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(check).rejects.toThrow(
|
||||
'Postgres refused rows of trips: refused by the fake; constraint trips_score_check; Failing row contains ' +
|
||||
'(2, 2, Porto, null, null, null).\nA row of trips breaks a rule of its Postgres table that the SQLite file ' +
|
||||
'did not hold it to; the detail above names the failing row or the value it refers to. Correct or remove ' +
|
||||
'that row ' +
|
||||
'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped), then run the move again.',
|
||||
);
|
||||
// A connection or a statement of the move's own is not the file's: no remedy for the file, no refusal.
|
||||
pg.failWith = { code: '08006', detail: 'connection lost' };
|
||||
const lost = moveSqliteToPostgres(options());
|
||||
await expect(lost).rejects.toThrow(/^Postgres refused rows of trips: refused by the fake; connection lost$/);
|
||||
await expect(lost).rejects.not.toBeInstanceOf(MoveRefusal);
|
||||
});
|
||||
|
||||
it("MOVE-031: a raw NUL in TREK's JSON is refused as a NUL with --drop-nul named, and with it dropped and listed", async () => {
|
||||
sqlite(path.join(dir, 'travel.db'), `update trips set meta = '{"k":"a' || char(0) || 'b"}' where id = 1;`);
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
'1 value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n' +
|
||||
' - trips row id=1, column meta: the text "{\\"k\\":\\"a\\u0000b\\"}" holds a NUL character (U+0000), ' +
|
||||
'which Postgres text cannot hold\n' +
|
||||
'1 of them hold a NUL character (U+0000). TREK no longer stores this character and drops it on its next ' +
|
||||
'write. Run the move again with --drop-nul to drop it now; the summary then lists every value it changed.',
|
||||
);
|
||||
const summary = await moveSqliteToPostgres(options({ dropNul: true }));
|
||||
expect(pg.rowsOf('"trips"')[0][3]).toBe('{"k":"ab"}');
|
||||
expect(summary.nulDropped).toEqual(['trips row id=1, column meta']);
|
||||
});
|
||||
|
||||
it('MOVE-032: a refusal of other values says how to correct them, after the NUL remedy when both occur', async () => {
|
||||
sqlite(path.join(dir, 'travel.db'), "update trips set score = 'high' where id = 2;");
|
||||
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
|
||||
/column score: the text "high" is not a number\nCorrect them in the SQLite file \(through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is stopped\), then run the move again\.$/,
|
||||
);
|
||||
sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 1;");
|
||||
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
|
||||
/lists every value it changed\.\nCorrect the others in the SQLite file \(through TREK/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -437,13 +620,19 @@ describe('nothing is changed silently', () => {
|
||||
exact: false,
|
||||
uniqueKeys: [],
|
||||
};
|
||||
const dropped = [{ where: 't row 1, column a', column: 'a' }];
|
||||
const dropped = [{ where: 't row 1, column a', column: 'a', text: 'x' }];
|
||||
const err = Object.assign(new Error('duplicate key'), { code: '23505', constraint: 'gone' });
|
||||
expect(refusedRowsMessage(spec, err, dropped)).toMatch(
|
||||
/from 1 value\(s\) of t, and a value .*\n {2}- t row 1, column a\n/,
|
||||
/from 1 value\(s\) of t that may hold it; a value .*\n {2}- t row 1, column a\n/,
|
||||
);
|
||||
expect(refusedRowsMessage(spec, new Error('other'), dropped)).toBe('Postgres refused rows of t: other');
|
||||
expect(refusedRowsMessage(spec, err, [])).toBe('Postgres refused rows of t: duplicate key; constraint gone');
|
||||
expect(refusedRowsMessage(spec, err, [])).toBe(
|
||||
'Postgres refused rows of t: duplicate key; constraint gone\nTwo rows of t share a unique key, which Postgres ' +
|
||||
'holds unique and the SQLite file did not (it may lack that index). The detail above names the value.\n' +
|
||||
'Change or remove one of those rows ' +
|
||||
'in the SQLite file (through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
||||
'stopped), then run the move again.',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ describe('planTrekTables', () => {
|
||||
{ name: 'reply_to', dataType: 'bigint', nullable: true },
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }],
|
||||
uniqueKeys: [{ name: 'messages_pkey', key: 'id', columns: ['id'], expression: false }],
|
||||
},
|
||||
{
|
||||
name: 'users',
|
||||
@@ -108,7 +108,7 @@ describe('planTrekTables', () => {
|
||||
primaryKey: ['id'],
|
||||
label: 'messages',
|
||||
exact: false,
|
||||
uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }],
|
||||
uniqueKeys: [{ name: 'messages_pkey', key: 'id', columns: ['id'], expression: false }],
|
||||
});
|
||||
expect(plan.specs[1].columns).toEqual([
|
||||
{ name: 'id', kind: 'integer' },
|
||||
@@ -168,7 +168,10 @@ describe('readTargetSchema', () => {
|
||||
primaryKey: ['id'],
|
||||
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
|
||||
serial: ['id'],
|
||||
unique: [{ name: 'trips_user_id_unique', columns: ['user_id'] }],
|
||||
unique: [
|
||||
{ name: 'trips_user_id_unique', columns: ['user_id'] },
|
||||
{ name: 'trips_user_id_abs', columns: ['user_id'], keys: ['abs(user_id)'], expression: true },
|
||||
],
|
||||
},
|
||||
{ name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] },
|
||||
]);
|
||||
@@ -182,15 +185,16 @@ describe('readTargetSchema', () => {
|
||||
{ name: 'user_id', dataType: 'bigint', nullable: true },
|
||||
],
|
||||
uniqueKeys: [
|
||||
{ name: 'trips_pkey', columns: ['id'] },
|
||||
{ name: 'trips_user_id_unique', columns: ['user_id'] },
|
||||
{ name: 'trips_pkey', key: 'id', columns: ['id'], expression: false },
|
||||
{ name: 'trips_user_id_unique', key: 'user_id', columns: ['user_id'], expression: false },
|
||||
{ name: 'trips_user_id_abs', key: 'abs(user_id)', columns: ['user_id'], expression: true },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'users',
|
||||
primaryKey: ['id'],
|
||||
columns: [{ name: 'id', dataType: 'bigint', nullable: false }],
|
||||
uniqueKeys: [{ name: 'users_pkey', columns: ['id'] }],
|
||||
uniqueKeys: [{ name: 'users_pkey', key: 'id', columns: ['id'], expression: false }],
|
||||
},
|
||||
]);
|
||||
expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]);
|
||||
|
||||
Reference in New Issue
Block a user