mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
fix(db): let the move to Postgres change nothing silently
The move stripped NUL from a plugin's own text, merging keys that were distinct, and turned a plugin's values into another type (text in a BLOB column into bytea, numbers in an untyped column into text). Its checksum was built from the converted values, so both passed as verified. A plugin's data now moves exactly as the Postgres plugin store would hand it back, or not at all: a NUL, a value of another storage class than its column, or an integer past 2^53 refuses the move before anything is written, listing plugin, table, row and column. A column left open by SQLite's affinity takes the kind its values need, so text in a BLOB column stays text. TREK's own text holding a NUL from an older install is refused by default too. --drop-nul drops it, lists every changed value in the summary, and names a unique key that dropping a NUL broke. Every table's checksum is now built from the values read from SQLite, so a conversion that changes a value fails the verification.
This commit is contained in:
@@ -23,6 +23,8 @@ interface TargetTable {
|
||||
name: string;
|
||||
columns: TargetColumn[];
|
||||
primaryKey: string[];
|
||||
/** Every unique index (the primary key's among them) over plain columns, by name. */
|
||||
uniqueKeys: Array<{ name: string; columns: string[] }>;
|
||||
}
|
||||
|
||||
export interface TargetSchema {
|
||||
@@ -57,9 +59,24 @@ export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
|
||||
"and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name",
|
||||
);
|
||||
|
||||
// A unique constraint is backed by an index of its name, so the index names what a refusal reports.
|
||||
const uniqueIndexes = await run<{ table_name: string; index_name: string; columns: string[] }>(
|
||||
trx,
|
||||
'select t.relname as table_name, i.relname as index_name, array_agg(a.attname::text order by k.ord) as columns ' +
|
||||
'from pg_index x join pg_class i on i.oid = x.indexrelid join pg_class t on t.oid = x.indrelid ' +
|
||||
'cross join lateral unnest(x.indkey) with ordinality as k(attnum, ord) ' +
|
||||
'join pg_attribute a on a.attrelid = x.indrelid and a.attnum = k.attnum ' +
|
||||
'where x.indisunique and t.relnamespace = current_schema()::regnamespace group by t.relname, i.relname',
|
||||
);
|
||||
|
||||
const tables = new Map<string, TargetTable>();
|
||||
for (const column of columns) {
|
||||
const table = tables.get(column.table_name) ?? { name: column.table_name, columns: [], primaryKey: [] };
|
||||
const table = tables.get(column.table_name) ?? {
|
||||
name: column.table_name,
|
||||
columns: [],
|
||||
primaryKey: [],
|
||||
uniqueKeys: [],
|
||||
};
|
||||
table.columns.push({
|
||||
name: column.column_name,
|
||||
dataType: column.data_type,
|
||||
@@ -67,6 +84,9 @@ export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
|
||||
});
|
||||
tables.set(column.table_name, table);
|
||||
}
|
||||
for (const index of uniqueIndexes) {
|
||||
tables.get(index.table_name)?.uniqueKeys.push({ name: index.index_name, columns: index.columns });
|
||||
}
|
||||
const foreignKeys: ForeignKey[] = [];
|
||||
for (const constraint of constraints) {
|
||||
if (constraint.kind === 'p') {
|
||||
@@ -154,6 +174,8 @@ export function planTrekTables(target: TargetSchema, source: SourceTables): Trek
|
||||
columns,
|
||||
deferred,
|
||||
primaryKey: table.primaryKey,
|
||||
exact: false,
|
||||
uniqueKeys: table.uniqueKeys,
|
||||
});
|
||||
}
|
||||
const sequences = target.sequences
|
||||
|
||||
@@ -14,12 +14,18 @@ import path from 'node:path';
|
||||
* The command around the move (`src/move-to-postgres.ts` runs it): reads
|
||||
* TREK_DB_URL as the server does, takes the SQLite file from the data
|
||||
* directory or `--from=<file>`, the plugin data from the plugins' data
|
||||
* directory, runs the move and prints its summary. Answers the exit code.
|
||||
* directory, runs the move and prints its summary. `--drop-nul` lets the move
|
||||
* drop NUL characters from TREK's own text instead of refusing them. Answers
|
||||
* the exit code.
|
||||
*/
|
||||
|
||||
export const USAGE =
|
||||
'Usage: move-to-postgres [--from=<SQLite file>]\n' +
|
||||
'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.';
|
||||
'Usage: move-to-postgres [--from=<SQLite file>] [--drop-nul]\n' +
|
||||
'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.\n' +
|
||||
' --from=<file> the SQLite file to copy\n' +
|
||||
" --drop-nul drop NUL characters (U+0000) from TREK's own text, which Postgres cannot hold and TREK drops on\n" +
|
||||
' its next write anyway, instead of refusing the move; the summary lists every value changed.\n' +
|
||||
" A plugin's own data is never changed: a NUL there refuses the move either way.";
|
||||
|
||||
export interface MoveCommandIo {
|
||||
env: () => AppEnv;
|
||||
@@ -51,13 +57,14 @@ const defaultIo: MoveCommandIo = {
|
||||
};
|
||||
|
||||
const FROM = '--from=';
|
||||
const DROP_NUL = '--drop-nul';
|
||||
|
||||
async function run(argv: readonly string[], io: MoveCommandIo): Promise<number> {
|
||||
if (argv.includes('--help') || argv.includes('-h')) {
|
||||
io.out(USAGE);
|
||||
return 0;
|
||||
}
|
||||
const unknown = argv.find((arg) => !arg.startsWith(FROM));
|
||||
const unknown = argv.find((arg) => !arg.startsWith(FROM) && arg !== DROP_NUL);
|
||||
if (unknown !== undefined) throw new MoveRefusal(`Unknown argument ${unknown}.\n${USAGE}`);
|
||||
const env = io.env();
|
||||
if (env.db.engine !== 'postgres')
|
||||
@@ -77,6 +84,7 @@ async function run(argv: readonly string[], io: MoveCommandIo): Promise<number>
|
||||
known: knownMigrationNames(),
|
||||
baseline: readPostgresBaseline,
|
||||
log: (line) => io.out(` ${line}`),
|
||||
dropNul: argv.includes(DROP_NUL),
|
||||
});
|
||||
io.out('');
|
||||
io.out(renderSummary(summary, target));
|
||||
|
||||
@@ -2,10 +2,13 @@ import type { SqliteSource } from '../../sqlite/sqlite-source';
|
||||
import {
|
||||
beyondSafeInteger,
|
||||
canonicalFromPostgres,
|
||||
canonicalFromSent,
|
||||
canonicalFromSource,
|
||||
convertValue,
|
||||
NUL_REASON,
|
||||
PLUGIN_RULES,
|
||||
TableChecksum,
|
||||
type ColumnKind,
|
||||
type ConvertRules,
|
||||
type PostgresValue,
|
||||
type SqliteValue,
|
||||
} from './values';
|
||||
@@ -17,7 +20,7 @@ import { CompiledQuery, type Kysely, type Transaction } from 'kysely';
|
||||
* inside the move's transaction. The same steps serve TREK's own tables and a
|
||||
* plugin's: read the rows one at a time, convert each value to its column's
|
||||
* type, insert in batches, then read the table back and compare its row count
|
||||
* and checksum with what was sent.
|
||||
* and checksum with the rows read from the SQLite file.
|
||||
*/
|
||||
|
||||
export type Trx = Transaction<object>;
|
||||
@@ -29,6 +32,8 @@ const MAX_BATCH_PARAMS = 30_000;
|
||||
const FETCH_ROWS = 2_000;
|
||||
/** The most refused values a refusal lists one by one. */
|
||||
export const MAX_LISTED_PROBLEMS = 20;
|
||||
/** Postgres's SQLSTATE for a row that would break a unique key. */
|
||||
const UNIQUE_VIOLATION = '23505';
|
||||
|
||||
/** One column of a table the move copies. */
|
||||
interface CopyColumn {
|
||||
@@ -49,6 +54,17 @@ export interface TableSpec {
|
||||
deferred: string[];
|
||||
/** The primary key's columns, which the deferred update finds a row by. */
|
||||
primaryKey: string[];
|
||||
/** A plugin's own table, whose values must arrive exactly as stored (`PLUGIN_RULES` in `values.ts`). */
|
||||
exact: boolean;
|
||||
/** The unique keys of the Postgres table (the primary key among them), by constraint or index name. */
|
||||
uniqueKeys: Array<{ name: string; columns: string[] }>;
|
||||
}
|
||||
|
||||
/** A value from which `--drop-nul` removed NUL characters, for the summary. */
|
||||
interface DroppedNul {
|
||||
/** The table, row and column, named as a refusal names a value. */
|
||||
where: string;
|
||||
column: string;
|
||||
}
|
||||
|
||||
/** A row's deferred values, kept until every table is in. */
|
||||
@@ -65,6 +81,10 @@ export interface CopiedTable {
|
||||
/** Values the Postgres column cannot hold exactly, listed up to `MAX_LISTED_PROBLEMS`; the move refuses on any. */
|
||||
problems: string[];
|
||||
problemCount: number;
|
||||
/** How many of the refused values hold a NUL character. */
|
||||
nulRefused: number;
|
||||
/** Every value `--drop-nul` changed. */
|
||||
nulDropped: DroppedNul[];
|
||||
/** Integer values beyond 2^53, which TREK's Postgres driver refuses to read back. */
|
||||
unsafeIntegers: number;
|
||||
deferredRows: DeferredRow[];
|
||||
@@ -120,26 +140,64 @@ export function insertSql(spec: Pick<TableSpec, 'target' | 'columns'>, rows: num
|
||||
return `insert into ${spec.target} (${names}) values ${tuples.join(', ')}`;
|
||||
}
|
||||
|
||||
async function insertBatch(trx: Trx, spec: TableSpec, batch: PostgresValue[][]): Promise<void> {
|
||||
/** Lines as a refusal lists them, and how many more there are when `total` is above what is listed. */
|
||||
export function listLines(lines: readonly string[], total = lines.length): string {
|
||||
const shown = lines.map((line) => ` - ${line}`);
|
||||
if (total > lines.length) shown.push(` - and ${total - lines.length} more`);
|
||||
return shown.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Why Postgres refused rows of a table. A unique key broken in a table whose
|
||||
* values lost NUL characters to `--drop-nul` is named as that: two values
|
||||
* that differed only by a NUL are equal once it is gone, which the SQLite
|
||||
* file's own key never saw.
|
||||
*/
|
||||
export function refusedRowsMessage(spec: TableSpec, err: unknown, dropped: readonly DroppedNul[]): string {
|
||||
const message = `Postgres refused rows of ${spec.label}: ${describePgError(err)}`;
|
||||
const { code, constraint } = (err ?? {}) as { code?: unknown; constraint?: unknown };
|
||||
if (code !== UNIQUE_VIOLATION || dropped.length === 0) return message;
|
||||
const key = spec.uniqueKeys.find((candidate) => candidate.name === constraint);
|
||||
const suspects = key ? dropped.filter((value) => key.columns.includes(value.column)) : dropped;
|
||||
if (suspects.length === 0) return message;
|
||||
const where = key ? `its column(s) ${key.columns.join(', ')}` : spec.label;
|
||||
return (
|
||||
`${message}\nThe key is broken because --drop-nul removed NUL characters (U+0000) from ${suspects.length} ` +
|
||||
`value(s) of ${where}, and a value that differed from another only by a NUL equals it once the NUL is gone:\n` +
|
||||
`${listLines(
|
||||
suspects.slice(0, MAX_LISTED_PROBLEMS).map((value) => value.where),
|
||||
suspects.length,
|
||||
)}\n` +
|
||||
'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.'
|
||||
);
|
||||
}
|
||||
|
||||
async function insertBatch(
|
||||
trx: Trx,
|
||||
spec: TableSpec,
|
||||
batch: PostgresValue[][],
|
||||
dropped: readonly DroppedNul[],
|
||||
): Promise<void> {
|
||||
if (batch.length === 0) return;
|
||||
try {
|
||||
await run(trx, insertSql(spec, batch.length), batch.flat());
|
||||
} catch (err) {
|
||||
throw new Error(`Postgres refused rows of ${spec.label}: ${describePgError(err)}`, { cause: err });
|
||||
throw new Error(refusedRowsMessage(spec, err, dropped), { cause: err });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies the rows of one table. Every value is converted first; once one
|
||||
* cannot be, nothing more is inserted, but the rest of the table is still read
|
||||
* so the refusal can list every problem at once. With `insert: false` it only
|
||||
* reads and converts, once an earlier table was refused.
|
||||
* so the refusal can list every problem at once. Without a transaction, or
|
||||
* with `insert: false` (once an earlier table was refused), it only reads and
|
||||
* converts, which is how the checks before anything is written run it.
|
||||
*/
|
||||
export async function copyTable(
|
||||
trx: Trx,
|
||||
trx: Trx | null,
|
||||
source: SqliteSource,
|
||||
spec: TableSpec,
|
||||
options: { insert: boolean } = { insert: true },
|
||||
options: { insert?: boolean; dropNul?: boolean } = {},
|
||||
): Promise<CopiedTable> {
|
||||
const result: CopiedTable = {
|
||||
spec,
|
||||
@@ -147,18 +205,26 @@ export async function copyTable(
|
||||
checksum: '',
|
||||
problems: [],
|
||||
problemCount: 0,
|
||||
nulRefused: 0,
|
||||
nulDropped: [],
|
||||
unsafeIntegers: 0,
|
||||
deferredRows: [],
|
||||
};
|
||||
const rules: ConvertRules = spec.exact ? PLUGIN_RULES : { exact: false, dropNul: options.dropNul === true };
|
||||
const checksum = new TableChecksum();
|
||||
const deferredAt = spec.deferred.map((name) => spec.columns.findIndex((column) => column.name === name));
|
||||
const keyAt = spec.primaryKey.map((name) => spec.columns.findIndex((column) => column.name === name));
|
||||
const size = batchRows(spec.columns.length);
|
||||
|
||||
/** How a refusal names a row: by its primary key, or by its place in the table without one. */
|
||||
/**
|
||||
* How a refusal names a row: by its primary key, or by its place in the
|
||||
* table without one. A NUL in a key is spelled `\u0000`, so the line can be
|
||||
* read and copied.
|
||||
*/
|
||||
const rowName = (raw: unknown[]): string => {
|
||||
if (keyAt.length === 0 || keyAt.includes(-1)) return `row ${result.rows}`;
|
||||
return `row ${spec.primaryKey.map((name, index) => `${name}=${String(raw[keyAt[index]])}`).join(', ')}`;
|
||||
const key = (index: number) => String(raw[keyAt[index]]).replaceAll('\u0000', '\\u0000');
|
||||
return `row ${spec.primaryKey.map((name, index) => `${name}=${key(index)}`).join(', ')}`;
|
||||
};
|
||||
|
||||
/** One SQLite row as the values sent to Postgres, or null once a value was refused. */
|
||||
@@ -166,21 +232,25 @@ export async function copyTable(
|
||||
result.rows++;
|
||||
const values: PostgresValue[] = [];
|
||||
spec.columns.forEach((column, index) => {
|
||||
const converted = convertValue(raw[index] as SqliteValue, column.kind);
|
||||
const converted = convertValue(raw[index] as SqliteValue, column.kind, rules);
|
||||
const where = `${spec.label} ${rowName(raw)}, column ${column.name}`;
|
||||
// Narrowed by `in`: the server compiles without strictNullChecks, where `ok` does not narrow the union.
|
||||
if (!('reason' in converted)) {
|
||||
values.push(converted.value);
|
||||
if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++;
|
||||
if (converted.droppedNul) result.nulDropped.push({ where, column: column.name });
|
||||
return;
|
||||
}
|
||||
values.push(null);
|
||||
result.problemCount++;
|
||||
if (result.problems.length < MAX_LISTED_PROBLEMS) {
|
||||
result.problems.push(`${spec.label} ${rowName(raw)}, column ${column.name}: ${converted.reason}`);
|
||||
}
|
||||
if (converted.reason.includes(NUL_REASON)) result.nulRefused++;
|
||||
if (result.problems.length < MAX_LISTED_PROBLEMS) result.problems.push(`${where}: ${converted.reason}`);
|
||||
});
|
||||
if (result.problemCount > 0) return null;
|
||||
checksum.add(values.map((value, index) => canonicalFromSent(value, spec.columns[index].kind)));
|
||||
// From the SQLite values, not the converted ones, so a conversion that changed a value cannot verify itself.
|
||||
checksum.add(
|
||||
spec.columns.map((column, index) => canonicalFromSource(raw[index] as SqliteValue, column.kind, rules)),
|
||||
);
|
||||
if (deferredAt.length > 0) {
|
||||
const later = deferredAt.map((index) => values[index]);
|
||||
if (later.some((value) => value !== null)) {
|
||||
@@ -209,7 +279,9 @@ export async function copyTable(
|
||||
const pump = async (): Promise<void> => {
|
||||
const { batch, done } = nextBatch();
|
||||
// Always awaited, so the next batch starts on a fresh stack even when nothing is inserted.
|
||||
await (options.insert && result.problemCount === 0 ? insertBatch(trx, spec, batch) : Promise.resolve());
|
||||
await (trx && options.insert !== false && result.problemCount === 0
|
||||
? insertBatch(trx, spec, batch, result.nulDropped)
|
||||
: Promise.resolve());
|
||||
if (!done) await pump();
|
||||
};
|
||||
try {
|
||||
@@ -243,7 +315,7 @@ export async function applyDeferred(trx: Trx, copied: CopiedTable): Promise<void
|
||||
|
||||
/**
|
||||
* Reads a table back through a cursor, every column as text, and answers its
|
||||
* row count and checksum, which must equal what `copyTable` sent.
|
||||
* row count and checksum, which must equal what `copyTable` read from SQLite.
|
||||
*/
|
||||
export async function readBack(trx: Trx, spec: TableSpec): Promise<{ rows: number; checksum: string }> {
|
||||
const checksum = new TableChecksum();
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
applyDeferred,
|
||||
copyTable,
|
||||
inOrder,
|
||||
listLines,
|
||||
MAX_LISTED_PROBLEMS,
|
||||
readBack,
|
||||
run,
|
||||
@@ -36,18 +37,27 @@ import type { Kysely } from 'kysely';
|
||||
* 1. The SQLite file must be at this build's migration level: an older one is
|
||||
* migrated by starting this TREK on it once, a newer one belongs to the
|
||||
* TREK that wrote it. Nothing else is checked by guess.
|
||||
* 2. The Postgres database must hold no table and no plugin schema.
|
||||
* 3. The schema is created exactly as a first boot creates it (the engine's
|
||||
* 2. Every plugin's data must move unchanged (`PLUGIN_RULES` in `values.ts`):
|
||||
* a value Postgres cannot hand back to the plugin as it is (a NUL, a
|
||||
* column whose values mix types) is listed, before anything is written.
|
||||
* 3. The Postgres database must hold no table and no plugin schema.
|
||||
* 4. The schema is created exactly as a first boot creates it (the engine's
|
||||
* bootstrap: the baseline and the migrations after its cut), without the
|
||||
* seeders, since every row comes from the file.
|
||||
* 4. In one transaction: the rows the bootstrap seeded are cleared, every
|
||||
* 5. In one transaction: the rows the bootstrap seeded are cleared, every
|
||||
* table is filled in foreign key order, each plugin's `plugin.db` goes into
|
||||
* its schema, the sequences are moved past the copied ids, and every table
|
||||
* is read back and compared with what was sent (row count and a checksum
|
||||
* over all rows). Only a copy that matches is committed.
|
||||
* 5. Anything that fails leaves the database as it was found: the
|
||||
* is read back and compared with the SQLite file (row count and a checksum
|
||||
* over all rows, built from the values read from SQLite). Only a copy that
|
||||
* matches is committed.
|
||||
* 6. Anything that fails leaves the database as it was found: the
|
||||
* transaction rolls back and the schema the bootstrap created is dropped.
|
||||
*
|
||||
* Nothing is changed silently. A NUL character in TREK's own text, which TREK
|
||||
* no longer stores and drops on its next write, refuses the move unless the
|
||||
* operator passes `dropNul` (`--drop-nul`); then each value that lost one is
|
||||
* listed in the summary.
|
||||
*
|
||||
* The SQLite files are opened read-only and never written. Uploads are not
|
||||
* the database's: they stay where the storage driver keeps them.
|
||||
*/
|
||||
@@ -67,6 +77,8 @@ export interface MoveOptions {
|
||||
baseline: () => PostgresBaseline;
|
||||
/** Where progress lines go. */
|
||||
log: (line: string) => void;
|
||||
/** Drop NUL characters from TREK's own text and JSON instead of refusing them (`--drop-nul`). */
|
||||
dropNul?: boolean;
|
||||
}
|
||||
|
||||
interface MovedTable {
|
||||
@@ -80,10 +92,24 @@ export interface MoveSummary {
|
||||
tables: MovedTable[];
|
||||
plugins: Array<{ pluginId: string; schema: string; tables: MovedTable[]; notCarried: string[] }>;
|
||||
sequences: number;
|
||||
/** Every value of TREK's own tables from which `--drop-nul` removed NUL characters: table, row and column. */
|
||||
nulDropped: string[];
|
||||
warnings: string[];
|
||||
seconds: number;
|
||||
}
|
||||
|
||||
/** What the operator can do about a NUL in TREK's own data, for a refusal that lists one. */
|
||||
const TREK_NUL_REMEDY =
|
||||
'TREK no longer stores this character and drops it on its next write. Run the move again with --drop-nul ' +
|
||||
'to drop it now; the summary then lists every value it changed.';
|
||||
|
||||
/** What the operator can do about plugin data that cannot move unchanged. */
|
||||
const PLUGIN_DATA_REMEDY =
|
||||
"A plugin's own data moves exactly as the plugin stored it, or not at all: Postgres text cannot hold a NUL " +
|
||||
'character (U+0000), and each column becomes one Postgres type, which hands every value back to the plugin as ' +
|
||||
"that type. Change those values in the plugin's plugin.db (through the plugin, or with the sqlite3 tool while " +
|
||||
'TREK is stopped), or uninstall the plugin and remove its data directory, then run the move again.';
|
||||
|
||||
/**
|
||||
* Why the SQLite file is not at this build's migration level, or null when it
|
||||
* is: it must have recorded exactly the migrations this build ships.
|
||||
@@ -109,12 +135,6 @@ export function migrationLevelProblem(recorded: readonly string[] | null, known:
|
||||
return null;
|
||||
}
|
||||
|
||||
function listed(problems: readonly string[], total: number): string {
|
||||
const lines = problems.map((problem) => ` - ${problem}`);
|
||||
if (total > problems.length) lines.push(` - and ${total - problems.length} more`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/** Refuses a database that holds a table or a plugin schema: the move fills an empty database only. */
|
||||
async function refuseNonEmptyTarget(db: Kysely<object>): Promise<void> {
|
||||
const tables = await run<{ table_name: string }>(
|
||||
@@ -147,6 +167,27 @@ function planPlugins(root: string, warnings: string[]): Array<{ source: SqliteSo
|
||||
return plugins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads every plugin table and checks that each value moves unchanged, before
|
||||
* anything is written: a refusal lists plugin, table, row and column of each
|
||||
* value that would not.
|
||||
*/
|
||||
async function checkPluginData(plugins: ReadonlyArray<{ source: SqliteSource; plan: PluginPlan }>): Promise<void> {
|
||||
const checked: CopiedTable[] = [];
|
||||
await inOrder(plugins, (plugin) =>
|
||||
inOrder(plugin.plan.specs, async (spec) => {
|
||||
checked.push(await copyTable(null, plugin.source, spec));
|
||||
}),
|
||||
);
|
||||
const total = checked.reduce((sum, table) => sum + table.problemCount, 0);
|
||||
if (total === 0) return;
|
||||
const problems = checked.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS);
|
||||
throw new MoveRefusal(
|
||||
`${total} value(s) in plugin data cannot move to Postgres unchanged; nothing was written:\n` +
|
||||
`${listLines(problems, total)}\n${PLUGIN_DATA_REMEDY}`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Compares a copied table with what Postgres holds now: the SQLite count, the rows sent and the checksum. */
|
||||
async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promise<string | null> {
|
||||
const back = await readBack(trx, copied.spec);
|
||||
@@ -154,7 +195,7 @@ async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promi
|
||||
return `${copied.spec.label}: the SQLite file holds ${sqliteCount} rows, ${copied.rows} were sent, Postgres holds ${back.rows}`;
|
||||
}
|
||||
if (back.checksum !== copied.checksum)
|
||||
return `${copied.spec.label}: the rows Postgres holds differ from the rows sent`;
|
||||
return `${copied.spec.label}: the rows Postgres holds differ from the rows of the SQLite file`;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -173,7 +214,7 @@ async function copyAll(
|
||||
const plan = planTrekTables(await readTargetSchema(trx), sourceTables);
|
||||
if (plan.problems.length > 0) {
|
||||
throw new MoveRefusal(
|
||||
`The SQLite file and this TREK's Postgres schema differ:\n${listed(plan.problems, plan.problems.length)}`,
|
||||
`The SQLite file and this TREK's Postgres schema differ:\n${listLines(plan.problems, plan.problems.length)}`,
|
||||
);
|
||||
}
|
||||
// The bootstrap's own rows (default settings, addons, providers) give way to the file's.
|
||||
@@ -181,10 +222,12 @@ async function copyAll(
|
||||
|
||||
const copied: CopiedTable[] = [];
|
||||
let refused = 0;
|
||||
let nulRefused = 0;
|
||||
const copyOne = (from: SqliteSource) => async (spec: CopiedTable['spec']) => {
|
||||
options.log(`Copying ${spec.label}`);
|
||||
const result = await copyTable(trx, from, spec, { insert: refused === 0 });
|
||||
const result = await copyTable(trx, from, spec, { insert: refused === 0, dropNul: options.dropNul });
|
||||
refused += result.problemCount;
|
||||
nulRefused += result.nulRefused;
|
||||
copied.push(result);
|
||||
};
|
||||
await inOrder(plan.specs, copyOne(source));
|
||||
@@ -202,9 +245,11 @@ async function copyAll(
|
||||
});
|
||||
if (refused > 0) {
|
||||
const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS);
|
||||
const nul = nulRefused > 0 ? `${nulRefused} of them hold a NUL character (U+0000). ${TREK_NUL_REMEDY}\n` : '';
|
||||
throw new MoveRefusal(
|
||||
`${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` +
|
||||
`${listed(problems, refused)}\nCorrect those rows (or ask for help with them), then run the move again.`,
|
||||
`${listLines(problems, refused)}\n${nul}` +
|
||||
'Correct those rows (or ask for help with them), then run the move again.',
|
||||
);
|
||||
}
|
||||
|
||||
@@ -228,7 +273,7 @@ async function copyAll(
|
||||
});
|
||||
if (mismatches.length > 0) {
|
||||
throw new Error(
|
||||
`The copy does not match the SQLite file; nothing was written:\n${listed(mismatches, mismatches.length)}`,
|
||||
`The copy does not match the SQLite file; nothing was written:\n${listLines(mismatches, mismatches.length)}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -245,6 +290,7 @@ async function copyAll(
|
||||
tables: (pluginTables.get(pluginPlan.pluginId) ?? []).map(moved),
|
||||
notCarried: pluginPlan.notCarried,
|
||||
}));
|
||||
summary.nulDropped = copied.flatMap((table) => table.nulDropped.map((value) => value.where));
|
||||
const unsafe = copied.filter((table) => table.unsafeIntegers > 0);
|
||||
for (const table of unsafe) {
|
||||
summary.warnings.push(
|
||||
@@ -279,6 +325,7 @@ export async function moveSqliteToPostgres(options: MoveOptions): Promise<MoveSu
|
||||
tables: [],
|
||||
plugins: [],
|
||||
sequences: 0,
|
||||
nulDropped: [],
|
||||
warnings: [],
|
||||
seconds: 0,
|
||||
};
|
||||
@@ -300,13 +347,14 @@ export async function moveSqliteToPostgres(options: MoveOptions): Promise<MoveSu
|
||||
if (orphans.length > 0) {
|
||||
throw new MoveRefusal(
|
||||
`${options.sqliteFile} holds rows whose reference points at nothing, which Postgres refuses:\n` +
|
||||
listed(
|
||||
listLines(
|
||||
orphans.map((row) => `${row.table} row ${String(row.rowid)} refers to a missing row of ${row.parent}`),
|
||||
orphans.length,
|
||||
),
|
||||
);
|
||||
}
|
||||
plugins.push(...planPlugins(options.pluginsDataDir, summary.warnings));
|
||||
await checkPluginData(plugins);
|
||||
await refuseNonEmptyTarget(db);
|
||||
|
||||
try {
|
||||
@@ -350,6 +398,10 @@ export function renderSummary(summary: MoveSummary, target: string): string {
|
||||
);
|
||||
for (const item of plugin.notCarried) lines.push(` not carried over: ${item}`);
|
||||
}
|
||||
if (summary.nulDropped.length > 0) {
|
||||
lines.push(` --drop-nul removed NUL characters (U+0000) from ${summary.nulDropped.length} value(s):`);
|
||||
for (const value of summary.nulDropped) lines.push(` ${value}`);
|
||||
}
|
||||
for (const warning of summary.warnings) lines.push(` Warning: ${warning}`);
|
||||
lines.push(
|
||||
'',
|
||||
|
||||
@@ -22,6 +22,11 @@ import path from 'node:path';
|
||||
* here (foreign keys between its tables, views, triggers, partial or
|
||||
* expression indexes, other defaults) is not created and is listed in the
|
||||
* summary. The file itself stays where it is, untouched.
|
||||
*
|
||||
* The rows are the plugin's, so none is changed on the way: every value must
|
||||
* come back from the Postgres store with the type and bytes it has in the
|
||||
* file, or the move refuses before it writes anything and lists the value
|
||||
* (`PLUGIN_RULES` in `values.ts`, checked by `checkPluginData` in `move.ts`).
|
||||
*/
|
||||
|
||||
/** The plugin store's own record of the migrations it ran, created exactly as the store creates it. */
|
||||
@@ -40,21 +45,27 @@ const PG_TYPE: Record<ColumnKind, string> = {
|
||||
/**
|
||||
* The kind of a plugin's column, by SQLite's affinity rules (the declared
|
||||
* type's text decides, in this order: INT, then CHAR/CLOB/TEXT, then BLOB or
|
||||
* none, then REAL/FLOA/DOUB, else NUMERIC). Where the affinity leaves the
|
||||
* storage open (none, or NUMERIC), the values the column holds decide, and
|
||||
* text is the answer that holds every one of them.
|
||||
* none, then REAL/FLOA/DOUB, else NUMERIC). An INTEGER, TEXT or REAL affinity
|
||||
* names the type the plugin asked for. Where the affinity leaves the storage
|
||||
* open (BLOB or none, or NUMERIC), the values the column holds decide: the
|
||||
* kind that hands every one of them back to the plugin as it reads it from
|
||||
* SQLite (an INTEGER or a REAL as a number, TEXT as a string, a BLOB as a
|
||||
* Buffer). A column whose values no single kind hands back like that still
|
||||
* gets one, and the move refuses each value it would change
|
||||
* (`PLUGIN_RULES` in `values.ts`); nothing is converted to fit.
|
||||
*/
|
||||
export function pluginColumnKind(declaredType: string, storage: ReadonlySet<string>): ColumnKind {
|
||||
const declared = declaredType.toUpperCase();
|
||||
const only = (...classes: string[]) => storage.size > 0 && [...storage].every((value) => classes.includes(value));
|
||||
if (declared.includes('INT')) return 'integer';
|
||||
if (/CHAR|CLOB|TEXT/.test(declared)) return 'text';
|
||||
if (declared.includes('BLOB')) return only('blob', 'text') ? 'bytes' : 'text';
|
||||
if (/REAL|FLOA|DOUB/.test(declared)) return only('integer', 'real') || storage.size === 0 ? 'float' : 'text';
|
||||
const blob = declared.includes('BLOB') || declared.trim() === '';
|
||||
if (!blob && /REAL|FLOA|DOUB/.test(declared)) return 'float';
|
||||
if (only('integer')) return 'integer';
|
||||
if (only('integer', 'real')) return 'float';
|
||||
if (only('blob')) return 'bytes';
|
||||
return 'text';
|
||||
// Empty, or text, or a mix: a column declared BLOB with no values keeps bytes for what the plugin writes later.
|
||||
return storage.size === 0 && declared.includes('BLOB') ? 'bytes' : 'text';
|
||||
}
|
||||
|
||||
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
|
||||
@@ -143,6 +154,8 @@ export function planPluginData(pluginId: string, source: SqliteSource): PluginPl
|
||||
],
|
||||
deferred: [],
|
||||
primaryKey: ['id'],
|
||||
exact: true,
|
||||
uniqueKeys: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@@ -168,6 +181,9 @@ export function planPluginData(pluginId: string, source: SqliteSource): PluginPl
|
||||
columns: columns.map((column, index) => ({ name: column.name, kind: kinds[index] })),
|
||||
deferred: [],
|
||||
primaryKey: keys.map((key) => key.name),
|
||||
// A plugin's values are never changed (`PLUGIN_RULES`), so no key of its tables can break on the way.
|
||||
exact: true,
|
||||
uniqueKeys: [],
|
||||
});
|
||||
// The identity's sequence is named by Postgres; resolveIdentitySequences reads it once the table exists.
|
||||
if (rowid) plan.sequences.push({ table, target, column: rowid, sequence: '' });
|
||||
|
||||
@@ -9,9 +9,23 @@ import { createHash } from 'node:crypto';
|
||||
* SQLite stores a value by what it is, not by the column's declared type, so
|
||||
* a column may hold text where the schema says integer. The move never
|
||||
* guesses: a value the Postgres column cannot hold exactly is refused with
|
||||
* the reason, and the move stops before it commits anything. The one
|
||||
* exception is a NUL character in text or JSON, which TREK stores on neither
|
||||
* engine (`db/nul-free.ts`): the move drops it, as every write does.
|
||||
* the reason, and the move stops before it commits anything.
|
||||
*
|
||||
* Two sets of rules decide what "exactly" means ({@link ConvertRules}):
|
||||
* - TREK's own tables follow TREK's storage model, so a whole number stored
|
||||
* as text reaches an integer column as that number. A NUL character in text
|
||||
* or JSON, which TREK has stopped storing on either engine
|
||||
* (`db/nul-free.ts`), is refused unless the operator asked for it to be
|
||||
* dropped (`--drop-nul`); each value that loses one is reported.
|
||||
* - A plugin's own data is the plugin's, so a value must come back from the
|
||||
* Postgres plugin store as the plugin's SQLite file hands it back today:
|
||||
* an INTEGER or a REAL as a number, TEXT as a string, a BLOB as a Buffer,
|
||||
* with the same bytes. Nothing changes its storage class, and a NUL, which
|
||||
* the Postgres store cannot hold, is refused.
|
||||
*
|
||||
* The compared text of a row is built from the value read from SQLite
|
||||
* ({@link canonicalFromSource}), never from the value sent, so a conversion
|
||||
* that changed a value fails the verification instead of passing it.
|
||||
*
|
||||
* The SQLite side is read with better-sqlite3's safe integers, so an integer
|
||||
* arrives as a `bigint` and keeps all 64 bits.
|
||||
@@ -26,13 +40,31 @@ export type SqliteValue = null | bigint | number | string | Uint8Array;
|
||||
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
|
||||
export type PostgresValue = null | string | number | Buffer;
|
||||
|
||||
type Converted = { ok: true; value: PostgresValue } | { ok: false; reason: string };
|
||||
/** A converted value, and whether `--drop-nul` removed NUL characters from it. */
|
||||
type Converted = { ok: true; value: PostgresValue; droppedNul?: true } | { ok: false; reason: string };
|
||||
|
||||
/** How strictly a value must arrive; see the header. */
|
||||
export interface ConvertRules {
|
||||
/** A plugin's own data: the value keeps its storage class and its bytes, and a NUL is refused. */
|
||||
exact: boolean;
|
||||
/** TREK's own data: a NUL in text or JSON is dropped (and reported) instead of refused. */
|
||||
dropNul: boolean;
|
||||
}
|
||||
|
||||
/** TREK's own tables, without `--drop-nul`. */
|
||||
const TREK_RULES: ConvertRules = { exact: false, dropNul: false };
|
||||
/** A plugin's own tables. */
|
||||
export const PLUGIN_RULES: ConvertRules = { exact: true, dropNul: false };
|
||||
|
||||
const INT64_MIN = -(2n ** 63n);
|
||||
const INT64_MAX = 2n ** 63n - 1n;
|
||||
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
|
||||
const DECIMAL_INTEGER = /^[+-]?\d+$/;
|
||||
const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
|
||||
const NUL = '\u0000';
|
||||
|
||||
/** Why a value holding a NUL character is refused. */
|
||||
export const NUL_REASON = 'holds a NUL character (U+0000), which Postgres text cannot hold';
|
||||
|
||||
/**
|
||||
* The kind of a Postgres column, from `information_schema.columns.data_type`.
|
||||
@@ -102,13 +134,27 @@ function toFloat(value: Exclude<SqliteValue, null>): Converted {
|
||||
return refuse(`${describe(value)} is not a number`);
|
||||
}
|
||||
|
||||
function toText(value: Exclude<SqliteValue, null>): Converted {
|
||||
if (typeof value === 'string') return { ok: true, value: stripNul(value) };
|
||||
function toText(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted {
|
||||
if (typeof value === 'string') {
|
||||
if (!value.includes(NUL)) return { ok: true, value };
|
||||
if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON}`);
|
||||
return { ok: true, value: stripNul(value), droppedNul: true };
|
||||
}
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
return refuse(`${describe(value)} is not text`);
|
||||
}
|
||||
|
||||
function toJson(value: Exclude<SqliteValue, null>): Converted {
|
||||
/** Whether a parsed JSON value holds a NUL in any string, object keys included. */
|
||||
function jsonHoldsNul(value: unknown): boolean {
|
||||
if (typeof value === 'string') return value.includes(NUL);
|
||||
if (Array.isArray(value)) return value.some(jsonHoldsNul);
|
||||
if (value !== null && typeof value === 'object') {
|
||||
return Object.entries(value).some(([key, item]) => key.includes(NUL) || jsonHoldsNul(item));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function toJson(value: Exclude<SqliteValue, null>, dropNul: boolean): Converted {
|
||||
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
|
||||
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
|
||||
let parsed: unknown;
|
||||
@@ -117,8 +163,10 @@ function toJson(value: Exclude<SqliteValue, null>): Converted {
|
||||
} catch {
|
||||
return refuse(`${describe(value)} is not valid JSON`);
|
||||
}
|
||||
// Valid JSON holds a NUL only escaped, which jsonb refuses as well: the value is written again without it.
|
||||
return { ok: true, value: value.includes('\\u0000') ? JSON.stringify(nulFreeJson(parsed)) : value };
|
||||
// Valid JSON holds a NUL only escaped, which jsonb refuses as well.
|
||||
if (!jsonHoldsNul(parsed)) return { ok: true, value };
|
||||
if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`);
|
||||
return { ok: true, value: JSON.stringify(nulFreeJson(parsed)), droppedNul: true };
|
||||
}
|
||||
|
||||
function toBytes(value: Exclude<SqliteValue, null>): Converted {
|
||||
@@ -128,18 +176,71 @@ function toBytes(value: Exclude<SqliteValue, null>): Converted {
|
||||
return refuse(`${describe(value)} is not a BLOB`);
|
||||
}
|
||||
|
||||
/** `value` as the Postgres column of `kind` takes it, or the reason it cannot hold it exactly. */
|
||||
export function convertValue(value: SqliteValue, kind: ColumnKind): Converted {
|
||||
/** What a column of `kind` holds on Postgres, for a refusal. */
|
||||
const KIND_HOLDS: Record<ColumnKind, string> = {
|
||||
integer: 'integers (bigint)',
|
||||
float: 'numbers (double precision)',
|
||||
text: 'text',
|
||||
json: 'JSON (jsonb)',
|
||||
bytes: 'bytes (bytea)',
|
||||
};
|
||||
|
||||
/** The SQLite storage class of a value, for a refusal. */
|
||||
function storageClass(value: Exclude<SqliteValue, null>): string {
|
||||
if (value instanceof Uint8Array) return 'a BLOB';
|
||||
if (typeof value === 'string') return 'TEXT';
|
||||
return typeof value === 'bigint' ? 'an INTEGER' : 'a REAL';
|
||||
}
|
||||
|
||||
/**
|
||||
* A plugin's value as its Postgres column takes it, only where the Postgres
|
||||
* plugin store hands it back to the plugin as the SQLite file did: a `bigint`
|
||||
* column reads as a number (and one past 2^53 is refused, `parseInt8` in
|
||||
* `postgres-driver.ts`), `double precision` as a number, `text` as a string
|
||||
* and `bytea` as a Buffer.
|
||||
*/
|
||||
function exactly(value: Exclude<SqliteValue, null>, kind: ColumnKind): Converted {
|
||||
const changed = () =>
|
||||
refuse(
|
||||
`${describe(value)} is stored as ${storageClass(value)}, but the column holds ${KIND_HOLDS[kind]} on Postgres, ` +
|
||||
'so the plugin would read it back as another type',
|
||||
);
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
if (typeof value !== 'bigint') return changed();
|
||||
if (value > SAFE_MAX || value < -SAFE_MAX) {
|
||||
return refuse(`${describe(value)} is beyond 2^53, which the Postgres plugin store refuses to read`);
|
||||
}
|
||||
return { ok: true, value: value.toString() };
|
||||
case 'float':
|
||||
if (typeof value === 'number') return { ok: true, value };
|
||||
if (typeof value !== 'bigint') return changed();
|
||||
return value > SAFE_MAX || value < -SAFE_MAX
|
||||
? refuse(`${describe(value)} has more digits than a double holds`)
|
||||
: { ok: true, value: Number(value) };
|
||||
case 'text':
|
||||
if (typeof value !== 'string') return changed();
|
||||
return value.includes(NUL) ? refuse(`${describe(value)} ${NUL_REASON}`) : { ok: true, value };
|
||||
case 'bytes':
|
||||
return value instanceof Uint8Array ? { ok: true, value: Buffer.from(value) } : changed();
|
||||
case 'json':
|
||||
return toJson(value, false);
|
||||
}
|
||||
}
|
||||
|
||||
/** `value` as the Postgres column of `kind` takes it under `rules`, or the reason it cannot hold it exactly. */
|
||||
export function convertValue(value: SqliteValue, kind: ColumnKind, rules: ConvertRules = TREK_RULES): Converted {
|
||||
if (value === null) return { ok: true, value: null };
|
||||
if (rules.exact) return exactly(value, kind);
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
return toInteger(value);
|
||||
case 'float':
|
||||
return toFloat(value);
|
||||
case 'text':
|
||||
return toText(value);
|
||||
return toText(value, rules.dropNul);
|
||||
case 'json':
|
||||
return toJson(value);
|
||||
return toJson(value, rules.dropNul);
|
||||
case 'bytes':
|
||||
return toBytes(value);
|
||||
}
|
||||
@@ -168,23 +269,41 @@ export function stableJson(text: string): string {
|
||||
return JSON.stringify(sort(JSON.parse(text)));
|
||||
}
|
||||
|
||||
/** Bytes as the compared text: their hex digits. */
|
||||
function hex(bytes: Uint8Array): string {
|
||||
return Buffer.from(bytes).toString('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* The text a value is compared in, from the value the move sent to Postgres.
|
||||
* `canonicalFromPostgres` must answer the same for what Postgres stored.
|
||||
* The text a value is compared in, from the value read from the SQLite file
|
||||
* (not from what the move sent), by the rules it was converted under.
|
||||
* `canonicalFromPostgres` must answer the same for what Postgres stored, so a
|
||||
* conversion that changed a value, or a value that changed on the way, fails
|
||||
* the verification. The one change the rules allow is `--drop-nul` on TREK's
|
||||
* own text and JSON, which the operator asked for and the summary lists. A
|
||||
* value of a storage class its kind does not take is refused before it is
|
||||
* sent; here it answers a marked text that matches nothing Postgres prints.
|
||||
*/
|
||||
export function canonicalFromSent(value: PostgresValue, kind: ColumnKind): string | null {
|
||||
export function canonicalFromSource(
|
||||
value: SqliteValue,
|
||||
kind: ColumnKind,
|
||||
rules: ConvertRules = TREK_RULES,
|
||||
): string | null {
|
||||
if (value === null) return null;
|
||||
if (value instanceof Uint8Array) return kind === 'bytes' ? hex(value) : `blob:${hex(value)}`;
|
||||
const dropNul = rules.dropNul && !rules.exact;
|
||||
switch (kind) {
|
||||
case 'integer':
|
||||
return BigInt(String(value)).toString();
|
||||
return typeof value === 'bigint' ? value.toString() : BigInt(String(value).trim()).toString();
|
||||
case 'float':
|
||||
return String(Number(value));
|
||||
return typeof value === 'bigint' ? value.toString() : String(Number(String(value).trim()));
|
||||
case 'text':
|
||||
return String(value);
|
||||
return typeof value === 'string' && dropNul ? stripNul(value) : String(value);
|
||||
case 'json':
|
||||
return stableJson(String(value));
|
||||
if (typeof value !== 'string') return stableJson(String(value));
|
||||
return stableJson(dropNul ? JSON.stringify(nulFreeJson(JSON.parse(value))) : value);
|
||||
case 'bytes':
|
||||
return Buffer.isBuffer(value) ? value.toString('hex') : Buffer.from(String(value), 'utf8').toString('hex');
|
||||
return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface FakeTableDef {
|
||||
foreignKeys?: Array<{ columns: string[]; references: string }>;
|
||||
/** Columns a sequence fills. */
|
||||
serial?: string[];
|
||||
/** Unique indexes besides the primary key's (`<table>_pkey`). */
|
||||
unique?: Array<{ name: string; columns: string[] }>;
|
||||
}
|
||||
|
||||
interface FakeTable {
|
||||
@@ -55,6 +57,8 @@ export class FakePostgres {
|
||||
pluginSchemas: string[] = [];
|
||||
/** A statement matching this fails, as Postgres would refuse it. */
|
||||
failOn: RegExp | null = null;
|
||||
/** What the refusal of `failOn` carries besides its message: by default a broken key without a SQLSTATE. */
|
||||
failWith: Record<string, unknown> = { detail: 'Key (id)=(1)', constraint: 'fake_key' };
|
||||
/** Rows the read-back answers in place of the stored ones, by table: a copy that came out wrong. */
|
||||
tamper = new Map<string, (rows: unknown[][]) => unknown[][]>();
|
||||
private cursor: Array<Record<string, string | null>> = [];
|
||||
@@ -81,7 +85,7 @@ export class FakePostgres {
|
||||
const params = [...query.parameters];
|
||||
this.statements.push(sql);
|
||||
if (this.failOn?.test(sql)) {
|
||||
throw Object.assign(new Error('refused by the fake'), { detail: 'Key (id)=(1)', constraint: 'fake_key' });
|
||||
throw Object.assign(new Error('refused by the fake'), this.failWith);
|
||||
}
|
||||
return { rows: this.answer(sql, params) as R[] };
|
||||
}
|
||||
@@ -123,6 +127,14 @@ export class FakePostgres {
|
||||
})),
|
||||
]);
|
||||
}
|
||||
if (sql.includes('from pg_index')) {
|
||||
return this.schema.flatMap((table) => [
|
||||
...(table.primaryKey.length > 0
|
||||
? [{ table_name: table.name, index_name: `${table.name}_pkey`, columns: table.primaryKey }]
|
||||
: []),
|
||||
...(table.unique ?? []).map((key) => ({ table_name: table.name, index_name: key.name, columns: key.columns })),
|
||||
]);
|
||||
}
|
||||
if (sql.includes("pg_get_serial_sequence(format('%I'")) {
|
||||
return this.schema.flatMap((table) =>
|
||||
(table.serial ?? []).map((column) => ({
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVE-001..016: the move of a SQLite install into Postgres
|
||||
* MOVE-001..025: the move of a SQLite install into Postgres
|
||||
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
|
||||
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
|
||||
* the refusals before anything is written, the copy in foreign key order with
|
||||
@@ -15,7 +15,12 @@ import {
|
||||
USAGE,
|
||||
type MoveCommandIo,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/command';
|
||||
import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import {
|
||||
batchRows,
|
||||
describePgError,
|
||||
insertSql,
|
||||
refusedRowsMessage,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import {
|
||||
migrationLevelProblem,
|
||||
MoveRefusal,
|
||||
@@ -82,6 +87,7 @@ const SCHEMA: FakeTableDef[] = [
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
serial: ['id'],
|
||||
unique: [{ name: 'users_name_unique', columns: ['name'] }],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -308,6 +314,134 @@ describe('moveSqliteToPostgres', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('nothing is changed silently', () => {
|
||||
/** A plugin's `plugin.db` under the plugins' data directory. */
|
||||
function pluginDb(pluginId: string, sql: string): void {
|
||||
fs.mkdirSync(path.join(dir, 'plugins-data', pluginId), { recursive: true });
|
||||
sqlite(path.join(dir, 'plugins-data', pluginId, 'plugin.db'), sql);
|
||||
}
|
||||
|
||||
it("MOVE-021: a plugin's NUL or a value that would change type refuses the move before anything is written", async () => {
|
||||
pluginDb(
|
||||
'nul-notes',
|
||||
`create table notes (id integer primary key, tag text, body text);
|
||||
insert into notes values (1, 'team' || char(0) || 'a', 'secret' || char(0) || 'payload'), (2, 'teama', 'other');
|
||||
create table kv (k text primary key, v);
|
||||
insert into kv values ('a', 5), ('b', 1.5), ('c', 'x');`,
|
||||
);
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
const message = await failure.catch((err: Error) => err.message);
|
||||
expect(message).toContain(
|
||||
'4 value(s) in plugin data cannot move to Postgres unchanged; nothing was written:\n' +
|
||||
' - plugin nul-notes: kv row k=a, column v: the value 5 is stored as an INTEGER, but the column holds text ' +
|
||||
'on Postgres, so the plugin would read it back as another type\n' +
|
||||
' - plugin nul-notes: kv row k=b, column v: the value 1.5 is stored as a REAL',
|
||||
);
|
||||
expect(message).toContain(
|
||||
' - plugin nul-notes: notes row id=1, column tag: the text "team\\u0000a" holds a NUL character (U+0000), ' +
|
||||
'which Postgres text cannot hold\n' +
|
||||
' - plugin nul-notes: notes row id=1, column body: the text "secret\\u0000payload" holds a NUL character',
|
||||
);
|
||||
expect(message).toMatch(/A plugin's own data moves exactly as the plugin stored it, or not at all/);
|
||||
// Refused before the target was even looked at: nothing created, nothing inserted.
|
||||
expect(pg.statements).toEqual([]);
|
||||
});
|
||||
|
||||
it('MOVE-022: a BLOB column holding text stays text, and a column holding only numbers stays numbers', async () => {
|
||||
pluginDb(
|
||||
'typed-notes',
|
||||
`create table docs (id integer primary key, payload blob, raw blob, data);
|
||||
insert into docs values (1, '{"a":1}', x'00ff', 5), (2, 'plain', null, 1.5);`,
|
||||
);
|
||||
const summary = await moveSqliteToPostgres(options());
|
||||
expect(pg.statements).toContain(
|
||||
'create table "trek_plugin_typed_notes"."docs" ("id" bigint generated by default as identity, ' +
|
||||
'"payload" text, "raw" bytea, "data" double precision, primary key ("id"))',
|
||||
);
|
||||
expect(pg.rowsOf('"trek_plugin_typed_notes"."docs"')).toEqual([
|
||||
['1', '{"a":1}', Buffer.from([0, 255]), 5],
|
||||
['2', 'plain', null, 1.5],
|
||||
]);
|
||||
expect(summary.plugins[0].tables).toEqual([{ label: 'docs', rows: 2, checksum: expect.any(String) }]);
|
||||
});
|
||||
|
||||
it("MOVE-023: a NUL in TREK's own text is refused by default, and with dropNul dropped and listed", async () => {
|
||||
sqlite(
|
||||
path.join(dir, 'travel.db'),
|
||||
`update users set name = 'Ad' || char(0) || 'a' where id = 1;
|
||||
update trips set meta = '{"k\\u0000":1}' where id = 1;`,
|
||||
);
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
const message = await failure.catch((err: Error) => err.message);
|
||||
expect(message).toContain(
|
||||
'2 value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n' +
|
||||
' - users row id=1, column name: the text "Ad\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold\n' +
|
||||
' - trips row id=1, column meta: the text "{\\"k\\\\u0000\\":1}" holds a NUL character (U+0000), which ' +
|
||||
'Postgres text cannot hold (escaped as \\u0000)\n' +
|
||||
'2 of them hold a NUL character (U+0000). TREK no longer stores this character and drops it on its next write. ' +
|
||||
'Run the move again with --drop-nul',
|
||||
);
|
||||
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
|
||||
|
||||
const summary = await moveSqliteToPostgres(options({ dropNul: true }));
|
||||
expect(pg.rowsOf('"users"')).toEqual([
|
||||
['1', 'Ada'],
|
||||
['2', 'Grace'],
|
||||
]);
|
||||
expect(pg.rowsOf('"trips"')[0][3]).toBe('{"k":1}');
|
||||
expect(summary.nulDropped).toEqual(['users row id=1, column name', 'trips row id=1, column meta']);
|
||||
expect(renderSummary(summary, 'postgres://db/trek')).toContain(
|
||||
' --drop-nul removed NUL characters (U+0000) from 2 value(s):\n' +
|
||||
' users row id=1, column name\n' +
|
||||
' trips row id=1, column meta',
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-024: a unique key that dropping a NUL broke is named as that, never as a bare duplicate', async () => {
|
||||
sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 2;");
|
||||
pg.failOn = /^insert into "users"/;
|
||||
pg.failWith = {
|
||||
code: '23505',
|
||||
constraint: 'users_name_unique',
|
||||
detail: 'Key (name)=(Ada) already exists.',
|
||||
};
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
'Postgres refused rows of users: refused by the fake; constraint users_name_unique; Key (name)=(Ada) already ' +
|
||||
'exists.\nThe key is broken because --drop-nul removed NUL characters (U+0000) from 1 value(s) of its ' +
|
||||
'column(s) name, and a value that differed from another only by a NUL equals it once the NUL is gone:\n' +
|
||||
' - users row id=2, column name\n' +
|
||||
'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.',
|
||||
);
|
||||
// A broken key whose columns lost no NUL is reported as Postgres reported it.
|
||||
pg.failWith = { code: '23505', constraint: 'users_pkey', detail: 'Key (id)=(2) already exists.' };
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
/^Postgres refused rows of users: refused by the fake; constraint users_pkey; Key \(id\)=\(2\) already exists\.$/,
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-025: a refused key is explained by every dropped NUL of the table when the key is not known', () => {
|
||||
const spec = {
|
||||
source: 't',
|
||||
target: '"t"',
|
||||
label: 't',
|
||||
columns: [],
|
||||
deferred: [],
|
||||
primaryKey: [],
|
||||
exact: false,
|
||||
uniqueKeys: [],
|
||||
};
|
||||
const dropped = [{ where: 't row 1, column a', column: 'a' }];
|
||||
const err = Object.assign(new Error('duplicate key'), { code: '23505', constraint: 'gone' });
|
||||
expect(refusedRowsMessage(spec, err, dropped)).toMatch(
|
||||
/from 1 value\(s\) of t, and a value .*\n {2}- t row 1, column a\n/,
|
||||
);
|
||||
expect(refusedRowsMessage(spec, new Error('other'), dropped)).toBe('Postgres refused rows of t: other');
|
||||
expect(refusedRowsMessage(spec, err, [])).toBe('Postgres refused rows of t: duplicate key; constraint gone');
|
||||
});
|
||||
});
|
||||
|
||||
describe('migrationLevelProblem', () => {
|
||||
it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => {
|
||||
const known = new Set([M1, M2]);
|
||||
@@ -327,12 +461,19 @@ describe('the pieces', () => {
|
||||
expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer');
|
||||
expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes');
|
||||
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('integer');
|
||||
// Text in a column declared BLOB stays text: the plugin reads a string back, not a Buffer.
|
||||
expect(pluginColumnKind('BLOB', classes('text'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('blob', 'text'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes())).toBe('bytes');
|
||||
expect(pluginColumnKind('DOUBLE', classes())).toBe('float');
|
||||
expect(pluginColumnKind('REAL', classes('text'))).toBe('text');
|
||||
// A REAL column keeps its type; text in it is refused by the copy, not turned into text with the numbers.
|
||||
expect(pluginColumnKind('REAL', classes('text'))).toBe('float');
|
||||
expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer');
|
||||
expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float');
|
||||
expect(pluginColumnKind('', classes('blob'))).toBe('bytes');
|
||||
expect(pluginColumnKind('', classes())).toBe('text');
|
||||
expect(pluginColumnKind('', classes('integer', 'text'))).toBe('text');
|
||||
expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text');
|
||||
|
||||
expect(pluginDefault(null, 'text')).toBeNull();
|
||||
@@ -394,6 +535,7 @@ describe('runMoveCommand', () => {
|
||||
tables: [{ label: 'users', rows: 2, checksum: 'x' }],
|
||||
plugins: [],
|
||||
sequences: 1,
|
||||
nulDropped: [],
|
||||
warnings: ['careful'],
|
||||
seconds: 0.4,
|
||||
};
|
||||
@@ -421,6 +563,7 @@ describe('runMoveCommand', () => {
|
||||
const help = io();
|
||||
expect(await runMoveCommand(['--help'], help.io)).toBe(0);
|
||||
expect(help.out).toEqual([USAGE]);
|
||||
expect(USAGE).toMatch(/\[--drop-nul\][\s\S]*--drop-nul +drop NUL characters \(U\+0000\) from TREK's own text/);
|
||||
const unknown = io();
|
||||
expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1);
|
||||
expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/);
|
||||
@@ -436,8 +579,12 @@ describe('runMoveCommand', () => {
|
||||
expect.objectContaining({
|
||||
sqliteFile: path.join(dir, 'other.db'),
|
||||
pluginsDataDir: expect.stringMatching(/plugins-data$/),
|
||||
dropNul: false,
|
||||
}),
|
||||
);
|
||||
const dropping = io();
|
||||
expect(await runMoveCommand(['--drop-nul'], dropping.io)).toBe(0);
|
||||
expect(dropping.io.move).toHaveBeenCalledWith(expect.objectContaining({ dropNul: true }));
|
||||
const printed = run.out.join('\n');
|
||||
expect(printed).toContain('postgres://trek:***@db:5432/trek');
|
||||
expect(printed).not.toContain('secret');
|
||||
|
||||
@@ -63,7 +63,12 @@ describe('planCopyOrder', () => {
|
||||
describe('planTrekTables', () => {
|
||||
const target: TargetSchema = {
|
||||
tables: [
|
||||
{ name: 'mikro_orm_migrations', columns: [{ name: 'name', dataType: 'text', nullable: false }], primaryKey: [] },
|
||||
{
|
||||
name: 'mikro_orm_migrations',
|
||||
columns: [{ name: 'name', dataType: 'text', nullable: false }],
|
||||
primaryKey: [],
|
||||
uniqueKeys: [],
|
||||
},
|
||||
{
|
||||
name: 'messages',
|
||||
columns: [
|
||||
@@ -71,6 +76,7 @@ describe('planTrekTables', () => {
|
||||
{ name: 'reply_to', dataType: 'bigint', nullable: true },
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }],
|
||||
},
|
||||
{
|
||||
name: 'users',
|
||||
@@ -79,6 +85,7 @@ describe('planTrekTables', () => {
|
||||
{ name: 'avatar', dataType: 'bytea', nullable: true },
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
uniqueKeys: [],
|
||||
},
|
||||
],
|
||||
foreignKeys: [{ table: 'messages', columns: ['reply_to'], references: 'messages' }],
|
||||
@@ -96,7 +103,13 @@ describe('planTrekTables', () => {
|
||||
);
|
||||
expect(plan.problems).toEqual([]);
|
||||
expect(plan.specs.map((spec) => spec.target)).toEqual(['"messages"', '"users"']);
|
||||
expect(plan.specs[0]).toMatchObject({ deferred: ['reply_to'], primaryKey: ['id'], label: 'messages' });
|
||||
expect(plan.specs[0]).toMatchObject({
|
||||
deferred: ['reply_to'],
|
||||
primaryKey: ['id'],
|
||||
label: 'messages',
|
||||
exact: false,
|
||||
uniqueKeys: [{ name: 'messages_pkey', columns: ['id'] }],
|
||||
});
|
||||
expect(plan.specs[1].columns).toEqual([
|
||||
{ name: 'id', kind: 'integer' },
|
||||
{ name: 'avatar', kind: 'bytes' },
|
||||
@@ -119,8 +132,9 @@ describe('planTrekTables', () => {
|
||||
{ name: 'sent_at', dataType: 'timestamp with time zone', nullable: true },
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
uniqueKeys: [],
|
||||
},
|
||||
{ name: 'extra', columns: [], primaryKey: [] },
|
||||
{ name: 'extra', columns: [], primaryKey: [], uniqueKeys: [] },
|
||||
],
|
||||
};
|
||||
const plan = planTrekTables(
|
||||
@@ -143,7 +157,7 @@ describe('planTrekTables', () => {
|
||||
});
|
||||
|
||||
describe('readTargetSchema', () => {
|
||||
it('MOVEORD-007: reads the columns, keys and sequences of the current schema', async () => {
|
||||
it('MOVEORD-007: reads the columns, keys, unique keys and sequences of the current schema', async () => {
|
||||
const pg = new FakePostgres([
|
||||
{
|
||||
name: 'trips',
|
||||
@@ -154,6 +168,7 @@ describe('readTargetSchema', () => {
|
||||
primaryKey: ['id'],
|
||||
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
|
||||
serial: ['id'],
|
||||
unique: [{ name: 'trips_user_id_unique', columns: ['user_id'] }],
|
||||
},
|
||||
{ name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] },
|
||||
]);
|
||||
@@ -166,8 +181,17 @@ describe('readTargetSchema', () => {
|
||||
{ name: 'id', dataType: 'bigint', nullable: false },
|
||||
{ name: 'user_id', dataType: 'bigint', nullable: true },
|
||||
],
|
||||
uniqueKeys: [
|
||||
{ name: 'trips_pkey', columns: ['id'] },
|
||||
{ name: 'trips_user_id_unique', columns: ['user_id'] },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'users',
|
||||
primaryKey: ['id'],
|
||||
columns: [{ name: 'id', dataType: 'bigint', nullable: false }],
|
||||
uniqueKeys: [{ name: 'users_pkey', columns: ['id'] }],
|
||||
},
|
||||
{ name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }] },
|
||||
]);
|
||||
expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]);
|
||||
expect(schema.sequences).toEqual([{ table: 'trips', column: 'id', sequence: 'public.trips_id_seq' }]);
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
/**
|
||||
* MOVEVAL-001..010: how a SQLite value becomes the value a Postgres column of
|
||||
* the move's target takes, the text both sides are compared in, and the
|
||||
* order-free table checksum (src/db/engine/postgres/sqlite-move/values.ts).
|
||||
* MOVEVAL-001..014: how a SQLite value becomes the value a Postgres column of
|
||||
* the move's target takes, under TREK's rules and under a plugin's, the text
|
||||
* both sides are compared in, and the order-free table checksum
|
||||
* (src/db/engine/postgres/sqlite-move/values.ts).
|
||||
*/
|
||||
import {
|
||||
beyondSafeInteger,
|
||||
canonicalFromPostgres,
|
||||
canonicalFromSent,
|
||||
canonicalFromSource,
|
||||
columnKind,
|
||||
convertValue,
|
||||
PLUGIN_RULES,
|
||||
stableJson,
|
||||
TableChecksum,
|
||||
type ColumnKind,
|
||||
@@ -19,6 +21,7 @@ import path from 'node:path';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const ok = (value: unknown) => ({ ok: true, value });
|
||||
const DROP_NUL = { exact: false, dropNul: true };
|
||||
|
||||
describe('columnKind', () => {
|
||||
it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => {
|
||||
@@ -80,11 +83,15 @@ describe('convertValue', () => {
|
||||
expect(convertValue('', 'float')).toMatchObject({ ok: false });
|
||||
});
|
||||
|
||||
it('MOVEVAL-005: text takes strings and numbers, drops a NUL as every write does, and refuses a BLOB', () => {
|
||||
it('MOVEVAL-005: text takes strings and numbers, refuses a NUL unless told to drop it, and refuses a BLOB', () => {
|
||||
expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa'));
|
||||
expect(convertValue(7n, 'text')).toEqual(ok('7'));
|
||||
expect(convertValue(2.5, 'text')).toEqual(ok('2.5'));
|
||||
expect(convertValue('a\u0000b\u0000', 'text')).toEqual(ok('ab'));
|
||||
expect(convertValue('a\u0000b\u0000', 'text')).toEqual({
|
||||
ok: false,
|
||||
reason: 'the text "a\\u0000b\\u0000" holds a NUL character (U+0000), which Postgres text cannot hold',
|
||||
});
|
||||
expect(convertValue('a\u0000b\u0000', 'text', DROP_NUL)).toEqual({ ok: true, value: 'ab', droppedNul: true });
|
||||
// The six characters of an escape are text like any other.
|
||||
expect(convertValue('a\\u0000b', 'text')).toEqual(ok('a\\u0000b'));
|
||||
expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({
|
||||
@@ -97,15 +104,25 @@ describe('convertValue', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes', () => {
|
||||
it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes only when told to', () => {
|
||||
expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}'));
|
||||
expect(convertValue(5n, 'json')).toEqual(ok('5'));
|
||||
expect(convertValue('{broken', 'json')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringMatching(/is not valid JSON/),
|
||||
});
|
||||
expect(convertValue('"\\u0000"', 'json')).toEqual(ok('""'));
|
||||
expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json')).toEqual(ok('{"a":["xy",1]}'));
|
||||
expect(convertValue('"\\u0000"', 'json')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringMatching(/holds a NUL character \(U\+0000\), .* \(escaped as \\u0000\)$/),
|
||||
});
|
||||
expect(convertValue('"\\u0000"', 'json', DROP_NUL)).toEqual({ ok: true, value: '""', droppedNul: true });
|
||||
expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json', DROP_NUL)).toEqual({
|
||||
ok: true,
|
||||
value: '{"a":["xy",1]}',
|
||||
droppedNul: true,
|
||||
});
|
||||
// An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is.
|
||||
expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"'));
|
||||
// A raw NUL is not valid JSON in the first place.
|
||||
expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) });
|
||||
expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({
|
||||
@@ -130,24 +147,33 @@ describe('convertValue', () => {
|
||||
});
|
||||
|
||||
describe('the compared text', () => {
|
||||
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSent>[0], string]> = [
|
||||
['integer', '-12', '-12'],
|
||||
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSource>[0], string]> = [
|
||||
['integer', ' -12 ', '-12'],
|
||||
['integer', 9007199254740993n, '9007199254740993'],
|
||||
['integer', 42, '42'],
|
||||
['float', 0.30000000000000004, '0.30000000000000004'],
|
||||
['float', 1e21, '1e+21'],
|
||||
['float', 12n, '12'],
|
||||
['float', '3.5e2', '350'],
|
||||
['text', 'São Paulo', 'São Paulo'],
|
||||
['text', 7n, '7'],
|
||||
['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'],
|
||||
['bytes', Buffer.from([0, 171]), '\\x00ab'],
|
||||
['json', 5n, '5'],
|
||||
['bytes', new Uint8Array([0, 171]), '\\x00ab'],
|
||||
];
|
||||
|
||||
it.each(cases)('MOVEVAL-009: %s values compare equal as sent and as Postgres prints them', (kind, sent, printed) => {
|
||||
expect(canonicalFromSent(sent, kind)).toBe(canonicalFromPostgres(printed, kind));
|
||||
expect(canonicalFromSent(null, kind)).toBeNull();
|
||||
expect(canonicalFromPostgres(null, kind)).toBeNull();
|
||||
});
|
||||
it.each(cases)(
|
||||
'MOVEVAL-009: %s values compare equal as read from SQLite and as Postgres prints them',
|
||||
(kind, read, printed) => {
|
||||
expect(canonicalFromSource(read, kind)).toBe(canonicalFromPostgres(printed, kind));
|
||||
expect(canonicalFromSource(null, kind)).toBeNull();
|
||||
expect(canonicalFromPostgres(null, kind)).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it('MOVEVAL-009b: text sent for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
|
||||
expect(canonicalFromSent('ab', 'bytes')).toBe('6162');
|
||||
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSent(1e20, 'float'));
|
||||
it('MOVEVAL-009b: text read for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
|
||||
expect(canonicalFromSource('ab', 'bytes')).toBe('6162');
|
||||
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSource(1e20, 'float'));
|
||||
expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain');
|
||||
expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}');
|
||||
});
|
||||
@@ -176,3 +202,63 @@ describe('the compared text', () => {
|
||||
expect(sum([['null']])).not.toBe(sum([[null]]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("a plugin's values (PLUGIN_RULES)", () => {
|
||||
const exact = (value: Parameters<typeof convertValue>[0], kind: ColumnKind) =>
|
||||
convertValue(value, kind, PLUGIN_RULES);
|
||||
const changed =
|
||||
/is stored as .*, but the column holds .* on Postgres, so the plugin would read it back as another type/;
|
||||
|
||||
it('MOVEVAL-011: a value keeps its storage class and its bytes, as the Postgres plugin store hands it back', () => {
|
||||
expect(exact(42n, 'integer')).toEqual(ok('42'));
|
||||
expect(exact(1.5, 'float')).toEqual(ok(1.5));
|
||||
expect(exact(3n, 'float')).toEqual(ok(3));
|
||||
expect(exact('{"a":1}', 'text')).toEqual(ok('{"a":1}'));
|
||||
expect(exact(new Uint8Array([0, 1]), 'bytes')).toEqual(ok(Buffer.from([0, 1])));
|
||||
expect(exact(null, 'text')).toEqual(ok(null));
|
||||
});
|
||||
|
||||
it('MOVEVAL-012: a value that would come back as another type is refused, not converted', () => {
|
||||
expect(exact(1.5, 'text')).toEqual({
|
||||
ok: false,
|
||||
reason:
|
||||
'the value 1.5 is stored as a REAL, but the column holds text on Postgres, ' +
|
||||
'so the plugin would read it back as another type',
|
||||
});
|
||||
expect(exact(5n, 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact('{"a":1}', 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact(' 7', 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact(2.5, 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact('2.5', 'float')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact(new Uint8Array([1]), 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
||||
expect(exact(9007199254740993n, 'integer')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringMatching(/beyond 2\^53, which the Postgres plugin store refuses to read/),
|
||||
});
|
||||
expect(exact(-9007199254740993n, 'float')).toMatchObject({
|
||||
ok: false,
|
||||
reason: expect.stringMatching(/more digits than a double holds/),
|
||||
});
|
||||
});
|
||||
|
||||
it('MOVEVAL-013: a NUL in a plugin value is refused, never dropped, whatever the operator asked for', () => {
|
||||
expect(exact('team\u0000a', 'text')).toEqual({
|
||||
ok: false,
|
||||
reason: 'the text "team\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold',
|
||||
});
|
||||
expect(convertValue('team\u0000a', 'text', { exact: true, dropNul: true })).toMatchObject({ ok: false });
|
||||
});
|
||||
|
||||
it('MOVEVAL-014: the compared text comes from the SQLite value, so a value that changed on the way does not match', () => {
|
||||
// What Postgres would print had the NUL been dropped, or the type changed, on the way.
|
||||
expect(canonicalFromSource('team\u0000a', 'text', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('teama', 'text'));
|
||||
expect(canonicalFromSource('team\u0000a', 'text', { exact: true, dropNul: true })).not.toBe('teama');
|
||||
expect(canonicalFromSource(new Uint8Array([0x61]), 'text', PLUGIN_RULES)).not.toBe(
|
||||
canonicalFromPostgres('a', 'text'),
|
||||
);
|
||||
expect(canonicalFromSource(5n, 'bytes')).not.toBe(canonicalFromPostgres('\\x35', 'bytes'));
|
||||
// The one change allowed is the one the operator asked for on TREK's own data.
|
||||
expect(canonicalFromSource('team\u0000a', 'text', DROP_NUL)).toBe(canonicalFromPostgres('teama', 'text'));
|
||||
expect(canonicalFromSource('{"k\\u0000":1}', 'json', DROP_NUL)).toBe(canonicalFromPostgres('{"k": 1}', 'json'));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user