feat(db): move a SQLite install into an empty Postgres database

npm run db:move-to-postgres (dist/move-to-postgres.js in the image) copies
a SQLite file at this build's migration level into an empty Postgres
database. It refuses a file at another level, rows whose references point
at nothing and a target holding any table or plugin schema, creates the
schema with the engine's own bootstrap, then in one transaction copies every
table in foreign key order (setting a self-reference once its rows are in),
converts each value to its column's type and refuses one Postgres cannot
hold exactly, copies each plugin's plugin.db into its schema, moves the
sequences past the copied ids and compares every table's row count and an
order-free checksum over all rows before it commits. A failure drops the
schema again. The SQLite files and the uploads are never written.
This commit is contained in:
Maurice
2026-11-17 13:41:20 +01:00
parent 1ea08dcf0a
commit 55607ff2f9
14 changed files with 2535 additions and 0 deletions
+1
View File
@@ -11,6 +11,7 @@
"build": "node scripts/build.mjs",
"start:prod": "node --require tsconfig-paths/register dist/index.js",
"reset-admin": "node reset-admin.js",
"db:move-to-postgres": "node --import tsx src/move-to-postgres.ts",
"typecheck": "tsc --noEmit",
"typecheck:tests": "tsc -p tsconfig.tests.json --noEmit",
"typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit",
@@ -0,0 +1,163 @@
import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
import { planCopyOrder, type ForeignKey } from './order';
import { columnKind } from './values';
/**
* TREK's tables as the target database holds them after the engine's
* bootstrap, and how each one is filled from the table of the same name in
* the SQLite file. The target's catalog decides the column types; the SQLite
* file only has to hold the same tables and columns, which a file at this
* build's migration level does.
*/
/** The migration history: each side keeps its own, and the bootstrap wrote the target's. */
const HISTORY_TABLE = 'mikro_orm_migrations';
interface TargetColumn {
name: string;
dataType: string;
nullable: boolean;
}
interface TargetTable {
name: string;
columns: TargetColumn[];
primaryKey: string[];
}
export interface TargetSchema {
tables: TargetTable[];
foreignKeys: ForeignKey[];
/** Every column a sequence fills, by table. */
sequences: Array<{ table: string; column: string; sequence: string }>;
}
/** Reads the tables, keys and sequences of the current schema. */
export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
const columns = await run<{ table_name: string; column_name: string; data_type: string; is_nullable: string }>(
trx,
'select c.table_name, c.column_name, c.data_type, c.is_nullable from information_schema.columns c ' +
'join information_schema.tables t on t.table_schema = c.table_schema and t.table_name = c.table_name ' +
"where c.table_schema = current_schema() and t.table_type = 'BASE TABLE' order by c.table_name, c.ordinal_position",
);
const constraints = await run<{ kind: string; table_name: string; target_name: string | null; columns: string[] }>(
trx,
'select c.contype as kind, src.relname as table_name, dst.relname as target_name, ' +
'array_agg(a.attname::text order by k.ord) as columns from pg_constraint c ' +
'join pg_class src on src.oid = c.conrelid left join pg_class dst on dst.oid = c.confrelid ' +
'cross join lateral unnest(c.conkey) with ordinality as k(attnum, ord) ' +
'join pg_attribute a on a.attrelid = c.conrelid and a.attnum = k.attnum ' +
"where c.connamespace = current_schema()::regnamespace and c.contype in ('p', 'f') " +
'group by c.oid, c.contype, src.relname, dst.relname',
);
const sequences = await run<{ table_name: string; column_name: string; sequence: string }>(
trx,
"select table_name, column_name, pg_get_serial_sequence(format('%I', table_name), column_name) as sequence " +
'from information_schema.columns where table_schema = current_schema() ' +
"and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name",
);
const tables = new Map<string, TargetTable>();
for (const column of columns) {
const table = tables.get(column.table_name) ?? { name: column.table_name, columns: [], primaryKey: [] };
table.columns.push({
name: column.column_name,
dataType: column.data_type,
nullable: column.is_nullable === 'YES',
});
tables.set(column.table_name, table);
}
const foreignKeys: ForeignKey[] = [];
for (const constraint of constraints) {
if (constraint.kind === 'p') {
const table = tables.get(constraint.table_name);
if (table) table.primaryKey = constraint.columns;
} else if (constraint.target_name) {
foreignKeys.push({
table: constraint.table_name,
columns: constraint.columns,
references: constraint.target_name,
});
}
}
return {
tables: [...tables.values()],
foreignKeys,
sequences: sequences.map((row) => ({ table: row.table_name, column: row.column_name, sequence: row.sequence })),
};
}
/** The tables and columns of the SQLite file, as the move compares them with the target. */
export type SourceTables = ReadonlyMap<string, readonly string[]>;
interface TrekCopyPlan {
/** The tables in the order they are filled. */
specs: TableSpec[];
sequences: CopiedSequence[];
/** Why the two schemas do not match; the move refuses when there is any. */
problems: string[];
}
/**
* Pairs every table of the target with the SQLite table of the same name and
* puts them in foreign key order. Anything the two do not share, or a column
* type the move has no rule for, is a problem: copying around it would lose
* data or guess.
*/
export function planTrekTables(target: TargetSchema, source: SourceTables): TrekCopyPlan {
const problems: string[] = [];
const tables = target.tables.filter((table) => table.name !== HISTORY_TABLE);
const names = new Set(tables.map((table) => table.name));
for (const name of source.keys()) {
if (name !== HISTORY_TABLE && !names.has(name)) problems.push(`the SQLite table ${name} has no Postgres table`);
}
const plan = planCopyOrder(
tables.map((table) => table.name),
target.foreignKeys,
);
const byName = new Map(tables.map((table) => [table.name, table]));
const specs: TableSpec[] = [];
for (const name of plan.order) {
const table = byName.get(name);
const sourceColumns = source.get(name);
if (!table) continue;
if (!sourceColumns) {
problems.push(`the Postgres table ${name} has no SQLite table`);
continue;
}
for (const column of sourceColumns) {
if (!table.columns.some((candidate) => candidate.name === column)) {
problems.push(`the SQLite column ${name}.${column} has no Postgres column`);
}
}
const columns = table.columns.flatMap((column) => {
const kind = columnKind(column.dataType);
if (!sourceColumns.includes(column.name))
problems.push(`the Postgres column ${name}.${column.name} has no SQLite column`);
else if (kind === null)
problems.push(
`the Postgres column ${name}.${column.name} has the type ${column.dataType}, which the move does not know`,
);
else return [{ name: column.name, kind }];
return [];
});
const deferred = plan.deferred.get(name) ?? [];
for (const column of deferred) {
if (!table.columns.find((candidate) => candidate.name === column)?.nullable) {
problems.push(`${name}.${column} refers back to a table filled later and cannot be NULL meanwhile`);
}
}
specs.push({
source: name,
target: quoteIdentifier(name),
label: name,
columns,
deferred,
primaryKey: table.primaryKey,
});
}
const sequences = target.sequences
.filter((entry) => names.has(entry.table))
.map((entry) => ({ ...entry, target: quoteIdentifier(entry.table) }));
return { specs, sequences, problems };
}
@@ -0,0 +1,104 @@
import { readEnv, type AppEnv } from '../../../../app-config';
import { resolveDataPaths } from '../../../../app-config/data-paths';
import { knownMigrationNames } from '../../../known-migrations';
import type { AnyOrm } from '../../../orm';
import { buildOrmConfig, resolveEngine } from '../../index';
import { redactUrl } from '../../redact-url';
import { readPostgresBaseline } from '../baseline';
import { MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions } from './move';
import { MikroORM } from '@mikro-orm/core';
import path from 'node:path';
/**
* The command around the move (`src/move-to-postgres.ts` runs it): reads
* TREK_DB_URL as the server does, takes the SQLite file from the data
* directory or `--from=<file>`, the plugin data from the plugins' data
* directory, runs the move and prints its summary. Answers the exit code.
*/
export const USAGE =
'Usage: move-to-postgres [--from=<SQLite file>]\n' +
'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.';
export interface MoveCommandIo {
env: () => AppEnv;
out: (line: string) => void;
err: (line: string) => void;
/** An ORM on the target database, from the Postgres engine's own config. */
openOrm: (env: AppEnv) => Promise<AnyOrm>;
move: (options: MoveOptions) => ReturnType<typeof moveSqliteToPostgres>;
}
/** The ORM the move runs on: the engine's config, without entity discovery (the move runs SQL only). */
async function openTargetOrm(env: AppEnv): Promise<AnyOrm> {
const config = buildOrmConfig(resolveEngine(env.db));
return MikroORM.init({
...config,
entities: [],
entitiesTs: [],
discovery: { warnWhenNoEntities: false },
migrations: { ...config.migrations, silent: true },
});
}
const defaultIo: MoveCommandIo = {
env: readEnv,
out: (line) => process.stdout.write(`${line}\n`),
err: (line) => process.stderr.write(`${line}\n`),
openOrm: openTargetOrm,
move: moveSqliteToPostgres,
};
const FROM = '--from=';
async function run(argv: readonly string[], io: MoveCommandIo): Promise<number> {
if (argv.includes('--help') || argv.includes('-h')) {
io.out(USAGE);
return 0;
}
const unknown = argv.find((arg) => !arg.startsWith(FROM));
if (unknown !== undefined) throw new MoveRefusal(`Unknown argument ${unknown}.\n${USAGE}`);
const env = io.env();
if (env.db.engine !== 'postgres')
throw new MoveRefusal('Set TREK_DB_URL to the empty Postgres database to move into.');
const paths = resolveDataPaths(env);
const from = argv.find((arg) => arg.startsWith(FROM))?.slice(FROM.length);
const sqliteFile = path.resolve(from || paths.dbFile);
const target = redactUrl(env.db.url);
io.out(`Moving ${sqliteFile} into ${target}`);
const orm = await io.openOrm(env);
try {
const summary = await io.move({
sqliteFile,
pluginsDataDir: env.plugins.dataDir || path.join(paths.dataDir, 'plugins-data'),
orm,
known: knownMigrationNames(),
baseline: readPostgresBaseline,
log: (line) => io.out(` ${line}`),
});
io.out('');
io.out(renderSummary(summary, target));
return 0;
} finally {
await orm.close(true);
}
}
/** Runs the command and answers its exit code: 0 once the move committed, 1 for a refusal or a failure. */
export async function runMoveCommand(argv: readonly string[], io: Partial<MoveCommandIo> = {}): Promise<number> {
const resolved = { ...defaultIo, ...io };
try {
return await run(argv, resolved);
} catch (err) {
if (err instanceof MoveRefusal) resolved.err(`\nThe move was refused: ${err.message}`);
else {
const text = err instanceof Error ? (err.stack ?? err.message) : String(err);
resolved.err(
`\nThe move failed; the Postgres database is left as the move found it unless this says otherwise: ${text}`,
);
}
return 1;
}
}
@@ -0,0 +1,301 @@
import type { SqliteSource } from '../../sqlite/sqlite-source';
import {
beyondSafeInteger,
canonicalFromPostgres,
canonicalFromSent,
convertValue,
TableChecksum,
type ColumnKind,
type PostgresValue,
type SqliteValue,
} from './values';
import { CompiledQuery, type Kysely, type Transaction } from 'kysely';
/**
* Copying one table from the SQLite file into Postgres and proving the copy,
* inside the move's transaction. The same steps serve TREK's own tables and a
* plugin's: read the rows one at a time, convert each value to its column's
* type, insert in batches, then read the table back and compare its row count
* and checksum with what was sent.
*/
export type Trx = Transaction<object>;
/** Rows per INSERT at most; fewer for a wide table, so a batch stays far below Postgres's 65535 parameters. */
const MAX_BATCH_ROWS = 500;
const MAX_BATCH_PARAMS = 30_000;
/** Rows a verification cursor fetches at once. */
const FETCH_ROWS = 2_000;
/** The most refused values a refusal lists one by one. */
export const MAX_LISTED_PROBLEMS = 20;
/** One column of a table the move copies. */
interface CopyColumn {
name: string;
kind: ColumnKind;
}
/** What the move copies from one SQLite table into one Postgres table. */
export interface TableSpec {
/** The table in the SQLite file. */
source: string;
/** The Postgres table, quoted and qualified as SQL needs it. */
target: string;
/** A name for messages: `users`, or `plugin my-plugin: notes`. */
label: string;
columns: CopyColumn[];
/** Foreign key columns inserted as NULL first and set once every table is in (`order.ts`). */
deferred: string[];
/** The primary key's columns, which the deferred update finds a row by. */
primaryKey: string[];
}
/** A row's deferred values, kept until every table is in. */
interface DeferredRow {
key: PostgresValue[];
values: PostgresValue[];
}
/** What copying one table found. */
export interface CopiedTable {
spec: TableSpec;
rows: number;
checksum: string;
/** Values the Postgres column cannot hold exactly, listed up to `MAX_LISTED_PROBLEMS`; the move refuses on any. */
problems: string[];
problemCount: number;
/** Integer values beyond 2^53, which TREK's Postgres driver refuses to read back. */
unsafeIntegers: number;
deferredRows: DeferredRow[];
}
/** `name` as a Postgres identifier. */
export function quoteIdentifier(name: string): string {
return `"${name.replaceAll('"', '""')}"`;
}
/** Runs one statement, in the move's transaction or on the pool, and returns its rows. */
export async function run<R = Record<string, unknown>>(
trx: Kysely<object>,
sql: string,
params: unknown[] = [],
): Promise<R[]> {
const result = await trx.executeQuery<R>(CompiledQuery.raw(sql, params));
return result.rows;
}
/**
* Runs `work` for each item, one after the other, each started once the one
* before it has finished: the move's statements share one transaction, which
* runs one statement at a time.
*/
export async function inOrder<T>(items: readonly T[], work: (item: T) => Promise<void>): Promise<void> {
await items.reduce<Promise<void>>((previous, item) => previous.then(() => work(item)), Promise.resolve());
}
/** A Postgres error with its detail and the constraint it broke, which `pg` keeps apart from the message. */
export function describePgError(err: unknown): string {
if (!(err instanceof Error)) return String(err);
const extra = err as Error & { detail?: string; constraint?: string };
const parts = [err.message];
if (extra.constraint) parts.push(`constraint ${extra.constraint}`);
if (extra.detail) parts.push(extra.detail);
return parts.join('; ');
}
/** How many rows one INSERT carries for a table of `columns` columns. */
export function batchRows(columns: number): number {
return Math.max(1, Math.min(MAX_BATCH_ROWS, Math.floor(MAX_BATCH_PARAMS / Math.max(1, columns))));
}
/** The INSERT of `rows` rows into the table, with numbered parameters row by row. */
export function insertSql(spec: Pick<TableSpec, 'target' | 'columns'>, rows: number): string {
const names = spec.columns.map((column) => quoteIdentifier(column.name)).join(', ');
const width = spec.columns.length;
const tuples = Array.from(
{ length: rows },
(_, row) => `(${spec.columns.map((_column, index) => `$${row * width + index + 1}`).join(', ')})`,
);
return `insert into ${spec.target} (${names}) values ${tuples.join(', ')}`;
}
async function insertBatch(trx: Trx, spec: TableSpec, batch: PostgresValue[][]): Promise<void> {
if (batch.length === 0) return;
try {
await run(trx, insertSql(spec, batch.length), batch.flat());
} catch (err) {
throw new Error(`Postgres refused rows of ${spec.label}: ${describePgError(err)}`, { cause: err });
}
}
/**
* Copies the rows of one table. Every value is converted first; once one
* cannot be, nothing more is inserted, but the rest of the table is still read
* so the refusal can list every problem at once. With `insert: false` it only
* reads and converts, once an earlier table was refused.
*/
export async function copyTable(
trx: Trx,
source: SqliteSource,
spec: TableSpec,
options: { insert: boolean } = { insert: true },
): Promise<CopiedTable> {
const result: CopiedTable = {
spec,
rows: 0,
checksum: '',
problems: [],
problemCount: 0,
unsafeIntegers: 0,
deferredRows: [],
};
const checksum = new TableChecksum();
const deferredAt = spec.deferred.map((name) => spec.columns.findIndex((column) => column.name === name));
const keyAt = spec.primaryKey.map((name) => spec.columns.findIndex((column) => column.name === name));
const size = batchRows(spec.columns.length);
/** How a refusal names a row: by its primary key, or by its place in the table without one. */
const rowName = (raw: unknown[]): string => {
if (keyAt.length === 0 || keyAt.includes(-1)) return `row ${result.rows}`;
return `row ${spec.primaryKey.map((name, index) => `${name}=${String(raw[keyAt[index]])}`).join(', ')}`;
};
/** One SQLite row as the values sent to Postgres, or null once a value was refused. */
const convertRow = (raw: unknown[]): PostgresValue[] | null => {
result.rows++;
const values: PostgresValue[] = [];
spec.columns.forEach((column, index) => {
const converted = convertValue(raw[index] as SqliteValue, column.kind);
if (converted.ok) {
values.push(converted.value);
if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++;
return;
}
values.push(null);
result.problemCount++;
if (result.problems.length < MAX_LISTED_PROBLEMS) {
result.problems.push(`${spec.label} ${rowName(raw)}, column ${column.name}: ${converted.reason}`);
}
});
if (result.problemCount > 0) return null;
checksum.add(values.map((value, index) => canonicalFromSent(value, spec.columns[index].kind)));
if (deferredAt.length > 0) {
const later = deferredAt.map((index) => values[index]);
if (later.some((value) => value !== null)) {
result.deferredRows.push({ key: keyAt.map((index) => values[index]), values: later });
}
for (const index of deferredAt) values[index] = null;
}
return values;
};
const rows = source.rows(
spec.source,
spec.columns.map((column) => column.name),
);
/** Reads up to one batch of rows; `done` once the table has no more. */
const nextBatch = (): { batch: PostgresValue[][]; done: boolean } => {
const batch: PostgresValue[][] = [];
while (batch.length < size) {
const next = rows.next();
if (next.done) return { batch, done: true };
const values = convertRow(next.value);
if (values) batch.push(values);
}
return { batch, done: false };
};
const pump = async (): Promise<void> => {
const { batch, done } = nextBatch();
// Always awaited, so the next batch starts on a fresh stack even when nothing is inserted.
await (options.insert && result.problemCount === 0 ? insertBatch(trx, spec, batch) : Promise.resolve());
if (!done) await pump();
};
try {
await pump();
} finally {
// An iterator left open keeps the SQLite handle busy.
rows.return?.();
}
result.checksum = checksum.hex();
return result;
}
/** Sets the foreign key columns `copyTable` held back, once every table is in. */
export async function applyDeferred(trx: Trx, copied: CopiedTable): Promise<void> {
const { spec } = copied;
if (copied.deferredRows.length === 0) return;
if (spec.primaryKey.length === 0) throw new Error(`${spec.label} has no primary key to set its deferred columns by.`);
const width = spec.deferred.length;
const sets = spec.deferred.map((name, index) => `${quoteIdentifier(name)} = $${index + 1}`).join(', ');
const where = spec.primaryKey.map((name, index) => `${quoteIdentifier(name)} = $${width + index + 1}`).join(' and ');
const sql = `update ${spec.target} set ${sets} where ${where}`;
// One row at a time: only the rows of a table that points at itself, a handful in practice.
await inOrder(copied.deferredRows, async (row) => {
try {
await run(trx, sql, [...row.values, ...row.key]);
} catch (err) {
throw new Error(`Postgres refused a reference in ${spec.label}: ${describePgError(err)}`, { cause: err });
}
});
}
/**
* Reads a table back through a cursor, every column as text, and answers its
* row count and checksum, which must equal what `copyTable` sent.
*/
export async function readBack(trx: Trx, spec: TableSpec): Promise<{ rows: number; checksum: string }> {
const checksum = new TableChecksum();
const list = spec.columns.map((column, index) => `${quoteIdentifier(column.name)}::text as c${index}`).join(', ');
const fetchAll = async (): Promise<void> => {
const rows = await run<Record<string, string | null>>(trx, `fetch forward ${FETCH_ROWS} from trek_move_verify`);
for (const row of rows) {
checksum.add(spec.columns.map((column, index) => canonicalFromPostgres(row[`c${index}`] ?? null, column.kind)));
}
if (rows.length === FETCH_ROWS) await fetchAll();
};
await run(trx, `declare trek_move_verify no scroll cursor for select ${list} from ${spec.target}`);
try {
await fetchAll();
} finally {
await run(trx, 'close trek_move_verify');
}
return { rows: checksum.rows, checksum: checksum.hex() };
}
/** A sequence behind a copied column: `table` names it in the SQLite file, `target` in Postgres. */
export interface CopiedSequence {
table: string;
target: string;
column: string;
sequence: string;
}
/**
* Moves every sequence of the copied tables past the ids they now hold, and
* past the highest id an AUTOINCREMENT table ever handed out in the SQLite
* file, so a new row never takes an id SQLite would not have reused.
*/
export async function advanceSequences(
trx: Trx,
sequences: readonly CopiedSequence[],
autoincrementSeen: ReadonlyMap<string, bigint>,
): Promise<number> {
await inOrder(sequences, async (entry) => {
const [row] = await run<{ max: string | null }>(
trx,
`select max(${quoteIdentifier(entry.column)})::text as max from ${entry.target}`,
);
const seen = autoincrementSeen.get(entry.table) ?? null;
const copied = row?.max ? BigInt(row.max) : null;
const highest = seen !== null && (copied === null || seen > copied) ? seen : copied;
// Nothing copied: the next id is 1, as on a new database.
const called = highest !== null && highest >= 1n;
await run(trx, 'select setval($1::regclass, $2::bigint, $3::boolean)', [
entry.sequence,
called ? highest.toString() : '1',
called,
]);
});
return sequences.length;
}
@@ -0,0 +1,358 @@
import { NO_SAFETY_NET } from '../../../legacy-baseline';
import type { AnyOrm } from '../../../orm';
import { SqliteSource } from '../../sqlite/sqlite-source';
import type { PostgresBaseline } from '../baseline';
import { CLEAR_SCHEMA_SQL } from '../clear-schema';
import { PLUGIN_SCHEMA_PREFIX } from '../plugin-schema';
import { bootstrapPostgresSchema } from '../schema-bootstrap';
import { planTrekTables, readTargetSchema, type SourceTables } from './catalog';
import {
advanceSequences,
applyDeferred,
copyTable,
inOrder,
MAX_LISTED_PROBLEMS,
readBack,
run,
type CopiedSequence,
type CopiedTable,
type Trx,
} from './copy';
import {
createPluginSchema,
findPluginDataFiles,
planPluginData,
resolveIdentitySequences,
type PluginPlan,
} from './plugin-data';
import type { Kysely } from 'kysely';
/**
* Moves a SQLite install into an empty Postgres database (`npm run
* db:move-to-postgres`, `dist/move-to-postgres.js` in the image).
*
* 1. The SQLite file must be at this build's migration level: an older one is
* migrated by starting this TREK on it once, a newer one belongs to the
* TREK that wrote it. Nothing else is checked by guess.
* 2. The Postgres database must hold no table and no plugin schema.
* 3. The schema is created exactly as a first boot creates it (the engine's
* bootstrap: the baseline and the migrations after its cut), without the
* seeders, since every row comes from the file.
* 4. In one transaction: the rows the bootstrap seeded are cleared, every
* table is filled in foreign key order, each plugin's `plugin.db` goes into
* its schema, the sequences are moved past the copied ids, and every table
* is read back and compared with what was sent (row count and a checksum
* over all rows). Only a copy that matches is committed.
* 5. Anything that fails leaves the database as it was found: the
* transaction rolls back and the schema the bootstrap created is dropped.
*
* The SQLite files are opened read-only and never written. Uploads are not
* the database's: they stay where the storage driver keeps them.
*/
/** A refusal the operator can act on: printed without a stack. */
export class MoveRefusal extends Error {}
export interface MoveOptions {
/** The SQLite file to move: `data/travel.db` by default. */
sqliteFile: string;
/** Where the plugins keep their data directories (`plugin.db` files). */
pluginsDataDir: string;
/** An ORM on the target database, from the Postgres engine's own config. */
orm: AnyOrm;
/** The migrations this build ships. */
known: ReadonlySet<string>;
baseline: () => PostgresBaseline;
/** Where progress lines go. */
log: (line: string) => void;
}
interface MovedTable {
label: string;
rows: number;
checksum: string;
}
export interface MoveSummary {
sqliteFile: string;
tables: MovedTable[];
plugins: Array<{ pluginId: string; schema: string; tables: MovedTable[]; notCarried: string[] }>;
sequences: number;
warnings: string[];
seconds: number;
}
/**
* Why the SQLite file is not at this build's migration level, or null when it
* is: it must have recorded exactly the migrations this build ships.
*/
export function migrationLevelProblem(recorded: readonly string[] | null, known: ReadonlySet<string>): string | null {
const startOnce =
'Start this TREK version on it once, without TREK_DB_URL, so it brings the file up to date; stop it; then run the move again.';
if (recorded === null || recorded.length === 0) {
return `the SQLite file has no TREK migration history (an install from before 5.0, or not a TREK database). ${startOnce}`;
}
const unknown = recorded.filter((name) => !known.has(name));
if (unknown.length > 0) {
return (
`a newer TREK migrated the SQLite file (${unknown.length} migration(s) this version does not know, ` +
`the last ${[...unknown].sort().at(-1)}). Run the move with the TREK version that last ran on it.`
);
}
const have = new Set(recorded);
const missing = [...known].filter((name) => !have.has(name));
if (missing.length > 0) {
return `the SQLite file is ${missing.length} migration(s) behind this TREK version. ${startOnce}`;
}
return null;
}
function listed(problems: readonly string[], total: number): string {
const lines = problems.map((problem) => ` - ${problem}`);
if (total > problems.length) lines.push(` - and ${total - problems.length} more`);
return lines.join('\n');
}
/** Refuses a database that holds a table or a plugin schema: the move fills an empty database only. */
async function refuseNonEmptyTarget(db: Kysely<object>): Promise<void> {
const tables = await run<{ table_name: string }>(
db,
'select table_name from information_schema.tables where table_schema = current_schema() order by table_name limit 5',
);
const schemas = await run<{ nspname: string }>(
db,
'select nspname from pg_namespace where starts_with(nspname, $1) order by nspname limit 5',
[PLUGIN_SCHEMA_PREFIX],
);
const found = [...tables.map((row) => row.table_name), ...schemas.map((row) => `schema ${row.nspname}`)];
if (found.length === 0) return;
throw new MoveRefusal(
`The Postgres database is not empty (${found.join(', ')}). The move fills an empty database only, so it never ` +
'mixes two installs. Point TREK_DB_URL at a new, empty database. If an earlier move stopped part of the way, ' +
'drop that database and create it again.',
);
}
/** Opens every plugin's data file and plans its copy; the sources stay open for the copy. */
function planPlugins(root: string, warnings: string[]): Array<{ source: SqliteSource; plan: PluginPlan }> {
const { files, skipped } = findPluginDataFiles(root);
for (const file of skipped) warnings.push(`${file} is in a directory whose name is no plugin id; it was not moved.`);
const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = [];
for (const { pluginId, file } of files) {
const source = SqliteSource.open(file);
plugins.push({ source, plan: planPluginData(pluginId, source) });
}
return plugins;
}
/** Compares a copied table with what Postgres holds now: the SQLite count, the rows sent and the checksum. */
async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promise<string | null> {
const back = await readBack(trx, copied.spec);
if (back.rows !== sqliteCount || copied.rows !== sqliteCount) {
return `${copied.spec.label}: the SQLite file holds ${sqliteCount} rows, ${copied.rows} were sent, Postgres holds ${back.rows}`;
}
if (back.checksum !== copied.checksum)
return `${copied.spec.label}: the rows Postgres holds differ from the rows sent`;
return null;
}
/** Everything the copy transaction does, in order; throws on the first thing that does not hold. */
async function copyAll(
trx: Trx,
source: SqliteSource,
plugins: ReadonlyArray<{ source: SqliteSource; plan: PluginPlan }>,
options: MoveOptions,
summary: MoveSummary,
): Promise<void> {
await run(trx, "set local bytea_output = 'hex'");
const sourceTables: SourceTables = new Map(
source.tables().map((table) => [table, source.columns(table).map((column) => column.name)]),
);
const plan = planTrekTables(await readTargetSchema(trx), sourceTables);
if (plan.problems.length > 0) {
throw new MoveRefusal(
`The SQLite file and this TREK's Postgres schema differ:\n${listed(plan.problems, plan.problems.length)}`,
);
}
// The bootstrap's own rows (default settings, addons, providers) give way to the file's.
await run(trx, `truncate table ${plan.specs.map((spec) => spec.target).join(', ')}`);
const copied: CopiedTable[] = [];
let refused = 0;
const copyOne = (from: SqliteSource) => async (spec: CopiedTable['spec']) => {
options.log(`Copying ${spec.label}`);
const result = await copyTable(trx, from, spec, { insert: refused === 0 });
refused += result.problemCount;
copied.push(result);
};
await inOrder(plan.specs, copyOne(source));
await inOrder(copied, (table) => applyDeferred(trx, table));
const sequences: CopiedSequence[] = [...plan.sequences];
const pluginTables = new Map<string, CopiedTable[]>();
await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => {
if (refused > 0) return;
await createPluginSchema(trx, pluginPlan);
const before = copied.length;
await inOrder(pluginPlan.specs, copyOne(pluginSource));
pluginTables.set(pluginPlan.pluginId, copied.slice(before));
});
if (refused > 0) {
const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS);
throw new MoveRefusal(
`${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` +
`${listed(problems, refused)}\nCorrect those rows (or ask for help with them), then run the move again.`,
);
}
options.log('Moving the id sequences past the copied rows');
summary.sequences = await advanceSequences(trx, sequences, source.autoincrementSeen());
await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => {
summary.sequences += await advanceSequences(
trx,
await resolveIdentitySequences(trx, pluginPlan),
pluginSource.autoincrementSeen(),
);
});
options.log('Verifying every table');
const mismatches: string[] = [];
const sourceOf = (table: CopiedTable) =>
plugins.find(({ plan: pluginPlan }) => pluginPlan.specs.includes(table.spec))?.source ?? source;
await inOrder(copied, async (table) => {
const mismatch = await verify(trx, table, sourceOf(table).count(table.spec.source));
if (mismatch) mismatches.push(mismatch);
});
if (mismatches.length > 0) {
throw new Error(
`The copy does not match the SQLite file; nothing was written:\n${listed(mismatches, mismatches.length)}`,
);
}
const moved = (table: CopiedTable): MovedTable => ({
label: table.spec.source,
rows: table.rows,
checksum: table.checksum,
});
const pluginCopies = new Set([...pluginTables.values()].flat());
summary.tables = copied.filter((table) => !pluginCopies.has(table)).map(moved);
summary.plugins = plugins.map(({ plan: pluginPlan }) => ({
pluginId: pluginPlan.pluginId,
schema: pluginPlan.schema,
tables: (pluginTables.get(pluginPlan.pluginId) ?? []).map(moved),
notCarried: pluginPlan.notCarried,
}));
const unsafe = copied.filter((table) => table.unsafeIntegers > 0);
for (const table of unsafe) {
summary.warnings.push(
`${table.spec.label} holds ${table.unsafeIntegers} integer(s) beyond 2^53, which TREK refuses to read on Postgres.`,
);
}
}
/**
* Drops what the bootstrap created after a move failed (the database was
* empty before it, so everything in its schema is the move's). When even that
* fails, the error says so, since the database then needs dropping by hand.
*/
async function emptyAgain(db: Kysely<object>, cause: unknown): Promise<void> {
try {
await run(db, CLEAR_SCHEMA_SQL);
} catch (err) {
throw new Error(
`The move failed (${cause instanceof Error ? cause.message : String(cause)}), and the schema it had created ` +
`could not be dropped (${err instanceof Error ? err.message : String(err)}). Drop the Postgres database and ` +
'create it again before the next attempt.',
{ cause },
);
}
}
/** Runs the move. Throws a `MoveRefusal` for what the operator can fix, and leaves the target as it found it. */
export async function moveSqliteToPostgres(options: MoveOptions): Promise<MoveSummary> {
const started = Date.now();
const summary: MoveSummary = {
sqliteFile: options.sqliteFile,
tables: [],
plugins: [],
sequences: 0,
warnings: [],
seconds: 0,
};
const db = options.orm.em.getConnection().getClient<object>();
let source: SqliteSource;
try {
source = SqliteSource.open(options.sqliteFile);
} catch (err) {
throw new MoveRefusal(
`Cannot read the SQLite file ${options.sqliteFile}: ${err instanceof Error ? err.message : String(err)}. ` +
'Stop TREK first, and point --from at the file if it is not data/travel.db.',
);
}
const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = [];
try {
const level = migrationLevelProblem(source.migrationNames(), options.known);
if (level) throw new MoveRefusal(`Cannot move ${options.sqliteFile}: ${level}`);
const orphans = source.orphans(MAX_LISTED_PROBLEMS);
if (orphans.length > 0) {
throw new MoveRefusal(
`${options.sqliteFile} holds rows whose reference points at nothing, which Postgres refuses:\n` +
listed(
orphans.map((row) => `${row.table} row ${String(row.rowid)} refers to a missing row of ${row.parent}`),
orphans.length,
),
);
}
plugins.push(...planPlugins(options.pluginsDataDir, summary.warnings));
await refuseNonEmptyTarget(db);
try {
options.log('Creating the schema as a first boot does');
await bootstrapPostgresSchema(options.orm, {
baseline: options.baseline,
known: new Set(options.known),
safetyNet: NO_SAFETY_NET,
// This is the move itself.
sqliteLeftBehind: () => null,
});
await db.transaction().execute((trx) => copyAll(trx, source, plugins, options, summary));
} catch (err) {
// The copy rolled back; what the bootstrap created goes too, so the database is empty again.
await emptyAgain(db, err);
throw err;
}
} finally {
source.close();
for (const plugin of plugins) plugin.source.close();
}
summary.seconds = Math.round((Date.now() - started) / 100) / 10;
return summary;
}
/** The summary the command prints once the move committed. */
export function renderSummary(summary: MoveSummary, target: string): string {
const rows = summary.tables.reduce((sum, table) => sum + table.rows, 0);
const lines = [
`Moved ${summary.sqliteFile} into ${target} in ${summary.seconds} s.`,
` ${summary.tables.length} tables, ${rows} rows, each table verified by row count and checksum.`,
` ${summary.sequences} id sequence(s) moved past the copied rows.`,
];
for (const table of summary.tables.filter((entry) => entry.rows > 0)) {
lines.push(` ${table.label}: ${table.rows}`);
}
for (const plugin of summary.plugins) {
const pluginRows = plugin.tables.reduce((sum, table) => sum + table.rows, 0);
lines.push(
` Plugin ${plugin.pluginId}: ${plugin.tables.length} table(s), ${pluginRows} rows, into schema ${plugin.schema}.`,
);
for (const item of plugin.notCarried) lines.push(` not carried over: ${item}`);
}
for (const warning of summary.warnings) lines.push(` Warning: ${warning}`);
lines.push(
'',
'The SQLite file and the uploads were not changed. Next: set TREK_DB_URL for TREK, start it, sign in and check',
`your trips. To go back, unset TREK_DB_URL: TREK starts on ${summary.sqliteFile} as before.`,
);
return lines.join('\n');
}
@@ -0,0 +1,56 @@
/**
* The order the move fills the tables in, from the target's foreign keys: a
* table after every table it points at, so each row it inserts finds the row
* it references. Ties go by name, so two runs over one schema copy in the
* same order.
*
* A foreign key that cannot be satisfied by the order alone (a table pointing
* at itself, like a reply at its message, or a cycle between tables) is
* deferred: its columns are inserted as NULL and set once every row of the
* tables is in, still inside the move's one transaction. Postgres checks each
* foreign key when the row is written, so this is what lets a parent row
* arrive after its child.
*/
/** One foreign key of the target: `columns` of `table` point at `references`. */
export interface ForeignKey {
table: string;
columns: readonly string[];
references: string;
}
interface CopyPlan {
/** Every table, each after the tables it points at. */
order: string[];
/** Per table, the foreign key columns written as NULL first and set after all rows are in. */
deferred: Map<string, string[]>;
}
function defer(plan: CopyPlan, fk: ForeignKey): void {
const columns = plan.deferred.get(fk.table) ?? [];
for (const column of fk.columns) if (!columns.includes(column)) columns.push(column);
plan.deferred.set(fk.table, columns);
}
/** Plans the order for `tables`. A foreign key to a table outside the list (one the move does not copy) is ignored. */
export function planCopyOrder(tables: readonly string[], foreignKeys: readonly ForeignKey[]): CopyPlan {
const plan: CopyPlan = { order: [], deferred: new Map() };
const known = new Set(tables);
const pending = new Set([...tables].sort());
const keys = foreignKeys.filter((fk) => known.has(fk.table) && known.has(fk.references));
for (const fk of keys) if (fk.table === fk.references) defer(plan, fk);
const between = keys.filter((fk) => fk.table !== fk.references);
while (pending.size > 0) {
const waitsOn = (table: string) => between.filter((fk) => fk.table === table && pending.has(fk.references));
let next = [...pending].find((table) => waitsOn(table).length === 0);
if (next === undefined) {
// A cycle: break it at the first table by name, deferring its keys into the rest of the cycle.
next = [...pending][0];
for (const fk of waitsOn(next)) defer(plan, fk);
}
plan.order.push(next);
pending.delete(next);
}
return plan;
}
@@ -0,0 +1,226 @@
import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source';
import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema';
import { portableDefault } from '../schema-adjust';
import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
import type { ColumnKind } from './values';
import fs from 'node:fs';
import path from 'node:path';
/**
* A plugin's own database (`db:own`) moved from its `plugin.db` file into the
* schema TREK gives it on Postgres (`plugin-schema.ts`), so a plugin finds its
* rows where the Postgres store (`plugin-data-store.ts`) reads them.
*
* A plugin's tables are its own, created by its own SQL, so the move derives
* each Postgres table from the SQLite one: the column's type from SQLite's
* type affinity (and from the values it holds where the affinity leaves it
* open), NOT NULL, the primary key, a default that has a Postgres spelling,
* and the indexes over plain columns. The plugin's migration record
* (`_plugin_migrations`) is copied as it is, so the plugin does not run its
* migrations again over the copied tables. What has no Postgres counterpart
* here (foreign keys between its tables, views, triggers, partial or
* expression indexes, other defaults) is not created and is listed in the
* summary. The file itself stays where it is, untouched.
*/
/** The plugin store's own record of the migrations it ran, created exactly as the store creates it. */
const LEDGER = '_plugin_migrations';
const LEDGER_DDL = (schema: string) => `create table ${schema}.${LEDGER} (id text primary key, applied_at bigint)`;
/** The Postgres type the move gives each kind. */
const PG_TYPE: Record<ColumnKind, string> = {
integer: 'bigint',
float: 'double precision',
text: 'text',
json: 'jsonb',
bytes: 'bytea',
};
/**
* The kind of a plugin's column, by SQLite's affinity rules (the declared
* type's text decides, in this order: INT, then CHAR/CLOB/TEXT, then BLOB or
* none, then REAL/FLOA/DOUB, else NUMERIC). Where the affinity leaves the
* storage open (none, or NUMERIC), the values the column holds decide, and
* text is the answer that holds every one of them.
*/
export function pluginColumnKind(declaredType: string, storage: ReadonlySet<string>): ColumnKind {
const declared = declaredType.toUpperCase();
const only = (...classes: string[]) => storage.size > 0 && [...storage].every((value) => classes.includes(value));
if (declared.includes('INT')) return 'integer';
if (/CHAR|CLOB|TEXT/.test(declared)) return 'text';
if (declared.includes('BLOB')) return only('blob', 'text') ? 'bytes' : 'text';
if (/REAL|FLOA|DOUB/.test(declared)) return only('integer', 'real') || storage.size === 0 ? 'float' : 'text';
if (only('integer')) return 'integer';
if (only('integer', 'real')) return 'float';
if (only('blob')) return 'bytes';
return 'text';
}
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const STRING_LITERAL = /^'(?:[^']|'')*'$/s;
/** The Postgres spelling of a column default, or null when it has none the move can trust. */
export function pluginDefault(sql: string | null, kind: ColumnKind): string | null {
if (sql === null) return null;
const text = sql
.trim()
.replace(/^\((.*)\)$/s, '$1')
.trim();
const ported = portableDefault(text);
if (ported !== text) return ported;
if (/^null$/i.test(text)) return 'null';
if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text;
if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null;
if (STRING_LITERAL.test(text)) return text;
return null;
}
/** One plugin's database as the move plans to copy it. */
export interface PluginPlan {
pluginId: string;
file: string;
schema: string;
/** The DDL that creates its tables and indexes, in order. */
ddl: string[];
specs: TableSpec[];
sequences: CopiedSequence[];
/** What the move does not carry over, for the summary. */
notCarried: string[];
}
function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, notes: string[], table: string): string {
const parts = [quoteIdentifier(column.name), PG_TYPE[kind]];
if (identity) parts.push('generated by default as identity');
if (column.notNull && !identity) parts.push('not null');
if (column.defaultSql !== null && !identity) {
const value = pluginDefault(column.defaultSql, kind);
if (value === null) notes.push(`the default ${column.defaultSql} of ${table}.${column.name}`);
else parts.push(`default ${value}`);
}
return parts.join(' ');
}
function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null {
if (index.origin === 'pk') return null;
if (index.partial || index.columns === null) {
notes.push(`the index ${index.name} on ${table} (partial or over an expression)`);
return null;
}
const columns = index.columns.map(quoteIdentifier).join(', ');
return `create ${index.unique ? 'unique ' : ''}index on ${target} (${columns})`;
}
/** Plans the copy of one plugin's `plugin.db`: what to create in its schema and what to fill. */
export function planPluginData(pluginId: string, source: SqliteSource): PluginPlan {
const schemaName = pluginSchemaName(pluginId);
const schema = quotePluginSchema(schemaName);
const plan: PluginPlan = {
pluginId,
file: source.file,
schema: schemaName,
ddl: [`create schema ${schema}`],
specs: [],
sequences: [],
notCarried: [],
};
for (const item of source.viewsAndTriggers()) plan.notCarried.push(`the ${item.type} ${item.name}`);
const tables = source.tables();
if (!tables.includes(LEDGER)) plan.ddl.push(LEDGER_DDL(schema));
for (const table of tables) {
const target = `${schema}.${quoteIdentifier(table)}`;
const label = `plugin ${pluginId}: ${table}`;
const columns = source.columns(table);
if (table === LEDGER) {
plan.ddl.push(LEDGER_DDL(schema));
plan.specs.push({
source: table,
target,
label,
columns: [
{ name: 'id', kind: 'text' },
{ name: 'applied_at', kind: 'integer' },
],
deferred: [],
primaryKey: ['id'],
});
continue;
}
const keys = columns.filter((column) => column.primaryKey > 0).sort((a, b) => a.primaryKey - b.primaryKey);
const kinds = columns.map((column) =>
pluginColumnKind(column.declaredType, source.storageClasses(table, column.name)),
);
// SQLite's rowid alias: the one INTEGER PRIMARY KEY column, which hands out its own ids.
const rowid = keys.length === 1 && keys[0].declaredType.toUpperCase() === 'INTEGER' ? keys[0].name : null;
const definitions = columns.map((column, index) =>
columnDdl(column, kinds[index], column.name === rowid, plan.notCarried, table),
);
if (keys.length > 0) definitions.push(`primary key (${keys.map((key) => quoteIdentifier(key.name)).join(', ')})`);
plan.ddl.push(`create table ${target} (${definitions.join(', ')})`);
for (const index of source.indexes(table)) {
const ddl = indexDdl(index, target, plan.notCarried, table);
if (ddl) plan.ddl.push(ddl);
}
plan.specs.push({
source: table,
target,
label,
columns: columns.map((column, index) => ({ name: column.name, kind: kinds[index] })),
deferred: [],
primaryKey: keys.map((key) => key.name),
});
// The identity's sequence is named by Postgres; resolveIdentitySequences reads it once the table exists.
if (rowid) plan.sequences.push({ table, target, column: rowid, sequence: '' });
}
return plan;
}
/** A plugin's data file under the plugins' data directory, by plugin id. */
interface PluginDataFile {
pluginId: string;
file: string;
}
/**
* The `plugin.db` files under `root`, one per plugin directory. A directory
* whose name is no plugin id cannot have a schema, and is reported instead.
*/
export function findPluginDataFiles(root: string): { files: PluginDataFile[]; skipped: string[] } {
if (!fs.existsSync(root)) return { files: [], skipped: [] };
const files: PluginDataFile[] = [];
const skipped: string[] = [];
for (const entry of fs.readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
const file = path.join(root, entry.name, 'plugin.db');
if (!entry.isDirectory() || !fs.existsSync(file)) continue;
if (isPluginId(entry.name)) files.push({ pluginId: entry.name, file });
else skipped.push(file);
}
return { files, skipped };
}
/** Creates a plugin's schema, tables and indexes in the move's transaction. */
export async function createPluginSchema(trx: Trx, plan: PluginPlan): Promise<void> {
await inOrder(plan.ddl, async (statement) => {
try {
await run(trx, statement);
} catch (err) {
throw new Error(
`Postgres refused the table of plugin ${plan.pluginId} (${statement}): ${err instanceof Error ? err.message : String(err)}`,
{ cause: err },
);
}
});
}
/** The plan's identity columns with the sequence Postgres gave each, read from the catalog. */
export async function resolveIdentitySequences(trx: Trx, plan: PluginPlan): Promise<CopiedSequence[]> {
const resolved: CopiedSequence[] = [];
await inOrder(plan.sequences, async (entry) => {
const [row] = await run<{ sequence: string | null }>(trx, 'select pg_get_serial_sequence($1, $2) as sequence', [
entry.target,
entry.column,
]);
if (row?.sequence) resolved.push({ ...entry, sequence: row.sequence });
});
return resolved;
}
@@ -0,0 +1,234 @@
import { createHash } from 'node:crypto';
/**
* How a value read from a SQLite file becomes the value a Postgres column of
* the move's target takes, and the one text form both sides are compared in.
*
* SQLite stores a value by what it is, not by the column's declared type, so
* a column may hold text where the schema says integer. The move never
* guesses: a value the Postgres column cannot hold exactly is refused with
* the reason, and the move stops before it commits anything.
*
* The SQLite side is read with better-sqlite3's safe integers, so an integer
* arrives as a `bigint` and keeps all 64 bits.
*/
/** The Postgres column types the move writes, by what they hold. */
export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes';
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */
export type SqliteValue = null | bigint | number | string | Uint8Array;
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
export type PostgresValue = null | string | number | Buffer;
type Converted = { ok: true; value: PostgresValue } | { ok: false; reason: string };
const INT64_MIN = -(2n ** 63n);
const INT64_MAX = 2n ** 63n - 1n;
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
const DECIMAL_INTEGER = /^[+-]?\d+$/;
const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const NUL = '\u0000';
/**
* The kind of a Postgres column, from `information_schema.columns.data_type`.
* Null for a type the move has no rule for: the caller refuses the move then,
* so a new column type fails loudly here instead of being copied by guess.
*/
export function columnKind(dataType: string): ColumnKind | null {
switch (dataType) {
case 'bigint':
case 'integer':
case 'smallint':
return 'integer';
case 'double precision':
case 'real':
return 'float';
case 'text':
case 'character varying':
case 'character':
return 'text';
case 'json':
case 'jsonb':
return 'json';
case 'bytea':
return 'bytes';
default:
return null;
}
}
function refuse(reason: string): Converted {
return { ok: false, reason };
}
function describe(value: SqliteValue): string {
if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`;
if (typeof value === 'string')
return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`;
return `the value ${String(value)}`;
}
function toInteger(value: Exclude<SqliteValue, null>): Converted {
let whole: bigint;
if (typeof value === 'bigint') whole = value;
else if (typeof value === 'number') {
if (!Number.isInteger(value)) return refuse(`${describe(value)} is not a whole number`);
whole = BigInt(value);
} else if (typeof value === 'string' && DECIMAL_INTEGER.test(value.trim())) {
whole = BigInt(value.trim());
} else {
return refuse(`${describe(value)} is not a whole number`);
}
if (whole < INT64_MIN || whole > INT64_MAX) return refuse(`${describe(value)} does not fit in 64 bits`);
return { ok: true, value: whole.toString() };
}
function toFloat(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'number') return { ok: true, value };
if (typeof value === 'bigint') {
const number = Number(value);
if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`);
return { ok: true, value: number };
}
if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) {
const number = Number(value.trim());
if (Number.isFinite(number)) return { ok: true, value: number };
}
return refuse(`${describe(value)} is not a number`);
}
function toText(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'string') {
return value.includes(NUL)
? refuse('the text holds a NUL character, which Postgres text cannot store')
: { ok: true, value };
}
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
return refuse(`${describe(value)} is not text`);
}
function toJson(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
try {
JSON.parse(value);
} catch {
return refuse(`${describe(value)} is not valid JSON`);
}
// jsonb refuses the escaped NUL character as well as the raw one.
if (value.includes(NUL) || value.includes('\\u0000'))
return refuse('the JSON holds a NUL character, which jsonb cannot store');
return { ok: true, value };
}
function toBytes(value: Exclude<SqliteValue, null>): Converted {
if (value instanceof Uint8Array) return { ok: true, value: Buffer.from(value) };
// A text value in a BLOB column: its UTF-8 bytes, what SQLite's CAST(x AS BLOB) answers.
if (typeof value === 'string') return { ok: true, value: Buffer.from(value, 'utf8') };
return refuse(`${describe(value)} is not a BLOB`);
}
/** `value` as the Postgres column of `kind` takes it, or the reason it cannot hold it exactly. */
export function convertValue(value: SqliteValue, kind: ColumnKind): Converted {
if (value === null) return { ok: true, value: null };
switch (kind) {
case 'integer':
return toInteger(value);
case 'float':
return toFloat(value);
case 'text':
return toText(value);
case 'json':
return toJson(value);
case 'bytes':
return toBytes(value);
}
}
/** Whether an integer value read back would be beyond what TREK's Postgres driver reads (2^53). */
export function beyondSafeInteger(value: PostgresValue): boolean {
if (typeof value !== 'string' || !DECIMAL_INTEGER.test(value)) return false;
const whole = BigInt(value);
return whole > SAFE_MAX || whole < -SAFE_MAX;
}
/** JSON text with its object keys sorted, so two spellings of one value compare equal (jsonb reorders keys). */
export function stableJson(text: string): string {
const sort = (value: unknown): unknown => {
if (Array.isArray(value)) return value.map(sort);
if (value !== null && typeof value === 'object') {
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, sort((value as Record<string, unknown>)[key])]),
);
}
return value;
};
return JSON.stringify(sort(JSON.parse(text)));
}
/**
* The text a value is compared in, from the value the move sent to Postgres.
* `canonicalFromPostgres` must answer the same for what Postgres stored.
*/
export function canonicalFromSent(value: PostgresValue, kind: ColumnKind): string | null {
if (value === null) return null;
switch (kind) {
case 'integer':
return BigInt(String(value)).toString();
case 'float':
return String(Number(value));
case 'text':
return String(value);
case 'json':
return stableJson(String(value));
case 'bytes':
return Buffer.isBuffer(value) ? value.toString('hex') : Buffer.from(String(value), 'utf8').toString('hex');
}
}
/**
* The text a value is compared in, from the column read back as `::text`
* (with `bytea_output = hex`, so bytes read as `\x` and their hex digits).
*/
export function canonicalFromPostgres(text: string | null, kind: ColumnKind): string | null {
if (text === null) return null;
switch (kind) {
case 'integer':
return BigInt(text).toString();
case 'float':
return String(Number(text));
case 'text':
return text;
case 'json':
return stableJson(text);
case 'bytes':
return text.startsWith('\\x') ? text.slice(2).toLowerCase() : text;
}
}
const CHECKSUM_MODULUS = 2n ** 256n;
/**
* An order-free checksum over a table's rows: the sum of each row's SHA-256
* modulo 2^256. Postgres and SQLite need not hand the rows back in the same
* order (their collations differ), and a row that is missing, added, doubled
* or changed changes the sum.
*/
export class TableChecksum {
private sum = 0n;
rows = 0;
add(canonicalRow: ReadonlyArray<string | null>): void {
const digest = createHash('sha256').update(JSON.stringify(canonicalRow)).digest('hex');
this.sum = (this.sum + BigInt(`0x${digest}`)) % CHECKSUM_MODULUS;
this.rows++;
}
hex(): string {
return this.sum.toString(16).padStart(64, '0');
}
}
+20
View File
@@ -0,0 +1,20 @@
import 'reflect-metadata';
import 'dotenv/config';
// The same fail-fast validation as a server start: TREK_DB_URL is read as the server reads it.
import './app-config/boot-validate';
import { runMoveCommand } from './db/engine/postgres/sqlite-move/command';
/**
* Moves a SQLite install into an empty Postgres database, with TREK stopped:
*
* docker compose run --rm --user node -w /app/server app \
* node --require tsconfig-paths/register dist/move-to-postgres.js
* npm run db:move-to-postgres (in server/, outside Docker)
*
* The target is TREK_DB_URL; the source is data/travel.db, or --from=<file>.
* The SQLite files and the uploads are never changed. The wiki page
* Database-Postgres ("Moving an existing install to Postgres") has the steps,
* and src/db/engine/postgres/sqlite-move/ the move itself.
*/
void runMoveCommand(process.argv.slice(2)).then((code) => process.exit(code));
+261
View File
@@ -0,0 +1,261 @@
import type { AnyOrm } from '../../src/db/orm';
import type { CompiledQuery, Kysely } from 'kysely';
import { vi } from 'vitest';
/**
* A stand-in for the Postgres side of the SQLite move
* (`src/db/engine/postgres/sqlite-move/`), for the unit tests in the SQLite
* lane: it answers exactly the statements the move sends, keeps the rows it
* is given in memory and reads them back as Postgres's `::text` would. The
* real database is the Postgres lane's (`tests/postgres/sqlite-move.test.ts`).
*/
interface FakeColumn {
name: string;
dataType: string;
nullable?: boolean;
}
export interface FakeTableDef {
name: string;
columns: FakeColumn[];
primaryKey: string[];
/** Foreign keys: `columns` of this table point at `references`. */
foreignKeys?: Array<{ columns: string[]; references: string }>;
/** Columns a sequence fills. */
serial?: string[];
}
interface FakeTable {
columns: string[];
rows: unknown[][];
}
const QUOTED = /"((?:[^"]|"")*)"/g;
function unquote(text: string): string[] {
return [...text.matchAll(QUOTED)].map((match) => match[1].replaceAll('""', '"'));
}
/** A value as Postgres would print it with `::text` and `bytea_output = hex`. */
function asText(value: unknown): string | null {
if (value === null || value === undefined) return null;
if (Buffer.isBuffer(value)) return `\\x${value.toString('hex')}`;
return String(value);
}
export class FakePostgres {
/** Every statement, in order. */
readonly statements: string[] = [];
readonly tables = new Map<string, FakeTable>();
readonly sequences = new Map<string, { value: string; called: boolean }>();
/** Tables the database holds before anything runs (the move refuses them). */
existing: string[] = [];
pluginSchemas: string[] = [];
/** A statement matching this fails, as Postgres would refuse it. */
failOn: RegExp | null = null;
/** Rows the read-back answers in place of the stored ones, by table: a copy that came out wrong. */
tamper = new Map<string, (rows: unknown[][]) => unknown[][]>();
private cursor: Array<Record<string, string | null>> = [];
private bootstrapped = false;
constructor(readonly schema: FakeTableDef[]) {}
/** What the engine's bootstrap leaves: every table of the schema, with a seeded row or two. */
bootstrap(): void {
this.bootstrapped = true;
for (const table of this.schema) {
this.tables.set(`"${table.name}"`, { columns: table.columns.map((column) => column.name), rows: [] });
}
const first = this.schema[0];
this.tables.get(`"${first.name}"`)?.rows.push(first.columns.map(() => null));
}
rowsOf(target: string): unknown[][] {
return this.tables.get(target)?.rows ?? [];
}
async executeQuery<R>(query: CompiledQuery): Promise<{ rows: R[] }> {
const sql = query.sql;
const params = [...query.parameters];
this.statements.push(sql);
if (this.failOn?.test(sql)) {
throw Object.assign(new Error('refused by the fake'), { detail: 'Key (id)=(1)', constraint: 'fake_key' });
}
return { rows: this.answer(sql, params) as R[] };
}
/** `db.transaction().execute(fn)`: the fake has no isolation, only the order of statements. */
transaction() {
return { execute: <T>(work: (trx: Kysely<object>) => Promise<T>) => work(this.asKysely()) };
}
asKysely(): Kysely<object> {
return this as unknown as Kysely<object>;
}
private answer(sql: string, params: unknown[]): unknown[] {
if (sql.startsWith('select table_name from information_schema.tables')) {
return [...this.existing, ...(this.bootstrapped ? this.schema.map((table) => table.name) : [])].map(
(table_name) => ({ table_name }),
);
}
if (sql.startsWith('select nspname from pg_namespace')) return this.pluginSchemas.map((nspname) => ({ nspname }));
if (sql.includes('from information_schema.columns c')) {
return this.schema.flatMap((table) =>
table.columns.map((column) => ({
table_name: table.name,
column_name: column.name,
data_type: column.dataType,
is_nullable: column.nullable === false ? 'NO' : 'YES',
})),
);
}
if (sql.includes('from pg_constraint')) {
return this.schema.flatMap((table) => [
{ kind: 'p', table_name: table.name, target_name: null, columns: table.primaryKey },
...(table.foreignKeys ?? []).map((fk) => ({
kind: 'f',
table_name: table.name,
target_name: fk.references,
columns: fk.columns,
})),
]);
}
if (sql.includes("pg_get_serial_sequence(format('%I'")) {
return this.schema.flatMap((table) =>
(table.serial ?? []).map((column) => ({
table_name: table.name,
column_name: column,
sequence: `public.${table.name}_${column}_seq`,
})),
);
}
if (sql.startsWith('DO $clear$')) {
// The current schema is emptied: the database is as the move found it.
this.bootstrapped = false;
this.tables.clear();
return [];
}
if (sql.startsWith('select pg_get_serial_sequence($1, $2)')) return [{ sequence: `${String(params[0])}_seq` }];
if (sql.startsWith('truncate table ')) {
for (const target of sql.slice('truncate table '.length).split(', ')) this.tableOf(target).rows = [];
return [];
}
if (sql.startsWith('insert into ')) return this.insert(sql, params);
if (sql.startsWith('update ')) return this.update(sql, params);
if (sql.startsWith('create table ')) return this.createTable(sql);
if (sql.startsWith('declare trek_move_verify')) return this.declare(sql);
if (sql.startsWith('fetch forward ')) {
const size = Number(/fetch forward (\d+)/.exec(sql)?.[1]);
return this.cursor.splice(0, size);
}
if (sql.startsWith('select max(')) {
const [column] = unquote(sql.slice(0, sql.indexOf(')')));
const table = this.tableOf(sql.slice(sql.indexOf(' from ') + 6));
const index = table.columns.indexOf(column);
const values = table.rows
.map((row) => row[index])
.filter((value) => value !== null)
.map((value) => BigInt(String(value)));
const max = values.reduce<bigint | null>((best, value) => (best === null || value > best ? value : best), null);
return [{ max: max === null ? null : max.toString() }];
}
if (sql.startsWith('select setval(')) {
this.sequences.set(String(params[0]), { value: String(params[1]), called: Boolean(params[2]) });
return [];
}
return [];
}
private tableOf(target: string): FakeTable {
const table = this.tables.get(target.trim());
if (!table) throw new Error(`relation ${target} does not exist`);
return table;
}
private insert(sql: string, params: unknown[]): unknown[] {
const match = /^insert into (.+?) \((.+?)\) values /.exec(sql);
if (!match) throw new Error(`unparsed insert: ${sql}`);
const table = this.tableOf(match[1]);
const columns = unquote(match[2]);
for (let at = 0; at < params.length; at += columns.length) {
const values = params.slice(at, at + columns.length);
table.rows.push(table.columns.map((name) => values[columns.indexOf(name)] ?? null));
}
return [];
}
private update(sql: string, params: unknown[]): unknown[] {
const match = /^update (.+?) set (.+) where (.+)$/.exec(sql);
if (!match) throw new Error(`unparsed update: ${sql}`);
const table = this.tableOf(match[1]);
const sets = unquote(match[2]);
const keys = unquote(match[3]);
const keyValues = params.slice(sets.length);
const row = table.rows.find((candidate) =>
keys.every((key, index) => String(candidate[table.columns.indexOf(key)]) === String(keyValues[index])),
);
if (!row) throw new Error(`no row in ${match[1]} to update`);
sets.forEach((column, index) => {
row[table.columns.indexOf(column)] = params[index];
});
return [];
}
private createTable(sql: string): unknown[] {
const match = /^create table (\S+?\.\S+?|\S+) \((.*)\)$/s.exec(sql);
if (!match) throw new Error(`unparsed create table: ${sql}`);
const definitions = match[2].split(/, (?![^(]*\))/).filter((part) => !part.startsWith('primary key'));
const columns = definitions.map((part) => (part.startsWith('"') ? unquote(part)[0] : part.split(' ')[0]));
const target = match[1].replace(/\._plugin_migrations$/, '."_plugin_migrations"');
this.tables.set(target, { columns, rows: [] });
return [];
}
private declare(sql: string): unknown[] {
const target = sql.slice(sql.lastIndexOf(' from ') + 6);
const table = this.tableOf(target);
const selected = [...sql.matchAll(/"((?:[^"]|"")*)"::text/g)].map((match) => match[1]);
const tamper = this.tamper.get(target) ?? ((rows: unknown[][]) => rows);
this.cursor = tamper(table.rows).map((row) =>
Object.fromEntries(selected.map((name, index) => [`c${index}`, asText(row[table.columns.indexOf(name)])])),
);
return [];
}
}
/**
* An ORM whose connection is the fake: enough for the engine's bootstrap
* (`schema-bootstrap.ts`) to run its empty-database path, and for the move to
* reach the fake through `getClient()`.
*/
export function fakeMoveOrm(pg: FakePostgres, options: { pending?: string[]; known: string[] }): AnyOrm {
const executed: string[] = [];
const connection = {
execute: vi.fn(async (sql: string) => {
if (sql.includes('server_version_num')) return [{ schema: 'public', version: '170004', privileged: false }];
const answer = await pg.executeQuery({ sql, parameters: [], query: undefined } as unknown as CompiledQuery);
if (sql.startsWith('create table "baseline"')) pg.bootstrap();
return answer.rows;
}),
transactional: vi.fn(async (work: (ctx: unknown) => Promise<void>) => work({})),
getClient: () => pg.asKysely(),
};
const migrator = {
getStorage: () => ({
ensureTable: vi.fn(async () => undefined),
logMigration: vi.fn(async ({ name }: { name: string }) => {
executed.push(name);
}),
}),
getExecuted: vi.fn(async () => executed.map((name) => ({ name }))),
getPending: vi.fn(async () => (options.pending ?? []).map((name) => ({ name }))),
up: vi.fn(async () => undefined),
};
return {
config: { getExtension: () => migrator, get: () => ({ tableName: 'mikro_orm_migrations' }) },
em: { getConnection: () => connection },
} as unknown as AnyOrm;
}
@@ -0,0 +1,455 @@
/**
* MOVE-001..016: the move of a SQLite install into Postgres
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
* the refusals before anything is written, the copy in foreign key order with
* a table that points at itself, the sequences, the verification, a plugin's
* own database, the cleanup after a failure, and the command around it. The
* Postgres lane runs the same move against a real server
* (tests/postgres/sqlite-move.test.ts).
*/
import type { AppEnv } from '../../../../../../src/app-config';
import { CLEAR_SCHEMA_SQL } from '../../../../../../src/db/engine/postgres/clear-schema';
import {
runMoveCommand,
USAGE,
type MoveCommandIo,
} from '../../../../../../src/db/engine/postgres/sqlite-move/command';
import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
import {
migrationLevelProblem,
MoveRefusal,
moveSqliteToPostgres,
renderSummary,
type MoveOptions,
type MoveSummary,
} from '../../../../../../src/db/engine/postgres/sqlite-move/move';
import {
findPluginDataFiles,
pluginColumnKind,
pluginDefault,
} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data';
import { sqliteInstallHoldsAccounts, SqliteSource } from '../../../../../../src/db/engine/sqlite/sqlite-source';
import type { AnyOrm } from '../../../../../../src/db/orm';
import { FakePostgres, fakeMoveOrm, type FakeTableDef } from '../../../../../helpers/fake-postgres-move';
import Database from 'better-sqlite3';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const M1 = 'Migration20200101000000_baseline_schema';
const M2 = 'Migration20200101000100_second';
/** TREK's schema in miniature, as the bootstrap leaves it on Postgres. */
const SCHEMA: FakeTableDef[] = [
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'trip_id', dataType: 'bigint' },
{ name: 'reply_to', dataType: 'bigint' },
{ name: 'body', dataType: 'text' },
],
primaryKey: ['id'],
foreignKeys: [
{ columns: ['trip_id'], references: 'trips' },
{ columns: ['reply_to'], references: 'messages' },
],
serial: ['id'],
},
{
name: 'trips',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint' },
{ name: 'title', dataType: 'text' },
{ name: 'meta', dataType: 'jsonb' },
{ name: 'cover', dataType: 'bytea' },
{ name: 'score', dataType: 'double precision' },
],
primaryKey: ['id'],
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
serial: ['id'],
},
{
name: 'users',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'name', dataType: 'text', nullable: false },
],
primaryKey: ['id'],
serial: ['id'],
},
];
const SOURCE_SQL = `
create table mikro_orm_migrations (id integer primary key autoincrement, name text, executed_at text);
insert into mikro_orm_migrations (name) values ('${M1}'), ('${M2}');
create table users (id integer primary key autoincrement, name text not null);
create table trips (id integer primary key, user_id integer references users (id), title text, meta text, cover blob, score real);
create table messages (id integer primary key, trip_id integer references trips (id), reply_to integer references messages (id), body text);
insert into users (name) values ('Ada'), ('Grace'), ('Gone');
delete from users where name = 'Gone';
insert into trips values (1, 1, 'Lisbon', '{"b":1,"a":2}', x'00ff', 1.5), (2, 2, 'Porto', null, null, null);
insert into messages values (1, 1, 2, 'I do'), (2, 1, null, 'Who books the tram?');
`;
let dir: string;
let pg: FakePostgres;
let orm: AnyOrm;
const log = vi.fn();
function sqlite(file: string, sql: string): string {
const db = new Database(file);
db.exec(sql);
db.close();
return file;
}
function options(overrides: Partial<MoveOptions> = {}): MoveOptions {
return {
sqliteFile: path.join(dir, 'travel.db'),
pluginsDataDir: path.join(dir, 'plugins-data'),
orm,
known: new Set([M1, M2]),
baseline: () => ({ through: M2, migrations: [M1, M2], statements: ['create table "baseline" ("id" bigint)'] }),
log,
...overrides,
};
}
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-move-'));
sqlite(path.join(dir, 'travel.db'), SOURCE_SQL);
pg = new FakePostgres(SCHEMA);
orm = fakeMoveOrm(pg, { known: [M1, M2] });
});
afterEach(() => {
fs.rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe('moveSqliteToPostgres', () => {
it('MOVE-001: fills every table in foreign key order, sets the self-references afterwards and verifies the copy', async () => {
const summary = await moveSqliteToPostgres(options());
expect(summary.tables.map((table) => [table.label, table.rows])).toEqual([
['users', 2],
['trips', 2],
['messages', 2],
]);
expect(pg.rowsOf('"users"')).toEqual([
['1', 'Ada'],
['2', 'Grace'],
]);
expect(pg.rowsOf('"trips"')).toEqual([
['1', '1', 'Lisbon', '{"b":1,"a":2}', Buffer.from([0, 255]), 1.5],
['2', '2', 'Porto', null, null, null],
]);
// The reply was inserted without its reference, then pointed at the message it answers.
expect(pg.rowsOf('"messages"')).toEqual([
['1', '1', '2', 'I do'],
['2', '1', null, 'Who books the tram?'],
]);
const statements = pg.statements.join('\n');
expect(statements).toMatch(/truncate table "users", "trips", "messages"/);
expect(statements).toMatch(/update "messages" set "reply_to" = \$1 where "id" = \$2/);
expect(statements).not.toContain(CLEAR_SCHEMA_SQL);
expect(summary.warnings).toEqual([]);
});
it('MOVE-002: moves every sequence past the copied ids, and past ids AUTOINCREMENT handed out before', async () => {
const summary = await moveSqliteToPostgres(options());
expect(summary.sequences).toBe(3);
expect(pg.sequences.get('public.users_id_seq')).toEqual({ value: '3', called: true });
expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '2', called: true });
expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '2', called: true });
});
it('MOVE-003: an empty table starts its sequence at 1', async () => {
sqlite(path.join(dir, 'travel.db'), 'delete from messages;');
await moveSqliteToPostgres(options());
expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '1', called: false });
});
it('MOVE-004: a copy that does not read back as sent is refused and the schema dropped again', async () => {
pg.tamper.set('"trips"', (rows) => rows.map((row) => row.map((value) => (value === 'Porto' ? 'Braga' : value))));
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
/does not match the SQLite file; nothing was written:\n {2}- trips: the rows Postgres holds differ/,
);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-005: a row count that differs is named with all three counts', async () => {
pg.tamper.set('"users"', (rows) => rows.slice(1));
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
'users: the SQLite file holds 2 rows, 2 were sent, Postgres holds 1',
);
});
it('MOVE-006: values Postgres cannot hold are listed by row and column, and nothing is inserted', async () => {
sqlite(path.join(dir, 'travel.db'), "update users set id = id; update trips set score = 'high' where id = 2;");
const failure = moveSqliteToPostgres(options());
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
/1 value\(s\) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n {2}- trips row id=2, column score: the text "high" is not a number/,
);
expect(pg.statements.some((sql) => sql.startsWith('insert into "messages"'))).toBe(false);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-007: a statement Postgres refuses is named with its table, detail and constraint', async () => {
pg.failOn = /^insert into "trips"/;
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
'Postgres refused rows of trips: refused by the fake; constraint fake_key; Key (id)=(1)',
);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-008: when the schema cannot be dropped after a failure, the error says the database needs dropping', async () => {
pg.failOn = /^(insert into "trips"|DO \$clear\$)/;
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/could not be dropped .* Drop the Postgres database/);
});
it('MOVE-009: refuses a target that holds a table or a plugin schema, before it writes anything', async () => {
pg.existing = ['customers'];
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/The Postgres database is not empty \(customers\)/);
pg.existing = [];
pg.pluginSchemas = ['trek_plugin_notes'];
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/schema trek_plugin_notes/);
expect(pg.statements.some((sql) => sql.startsWith('create') || sql.startsWith('insert'))).toBe(false);
});
it('MOVE-010: refuses a SQLite file at another migration level, one with orphaned rows, and one it cannot read', async () => {
await expect(
moveSqliteToPostgres(options({ known: new Set([M1, M2, 'Migration20200101000200_third']) })),
).rejects.toThrow(/1 migration\(s\) behind this TREK version\. Start this TREK version on it once/);
sqlite(
path.join(dir, 'travel.db'),
"pragma foreign_keys = off; insert into trips (id, user_id, title) values (9, 77, 'Nowhere');",
);
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/trips row 9 refers to a missing row of users/);
await expect(moveSqliteToPostgres(options({ sqliteFile: path.join(dir, 'missing.db') }))).rejects.toThrow(
/Cannot read the SQLite file .*missing\.db: There is no SQLite file/,
);
});
it('MOVE-011: a plugin database goes into its schema with its migration record, ids and indexes', async () => {
const pluginDir = path.join(dir, 'plugins-data', 'move-notes');
fs.mkdirSync(pluginDir, { recursive: true });
fs.mkdirSync(path.join(dir, 'plugins-data', 'Not_An_Id'), { recursive: true });
sqlite(path.join(dir, 'plugins-data', 'Not_An_Id', 'plugin.db'), 'create table t (a);');
sqlite(
path.join(pluginDir, 'plugin.db'),
`create table _plugin_migrations (id text primary key, applied_at integer);
insert into _plugin_migrations values ('001', 1700000000000);
create table notes (id integer primary key autoincrement, body text not null default 'x', score real, raw blob,
added text default current_timestamp, odd text default (lower('A')));
create unique index notes_body on notes (body);
create index notes_hot on notes (score) where score > 1;
create view recent as select * from notes;
insert into notes (body, score, raw) values ('a', 1.5, x'01'), ('b', null, null), ('c', 3, null);
delete from notes where body = 'c';`,
);
const summary = await moveSqliteToPostgres(options());
expect(summary.plugins).toEqual([
{
pluginId: 'move-notes',
schema: 'trek_plugin_move_notes',
tables: [
{ label: '_plugin_migrations', rows: 1, checksum: expect.any(String) },
{ label: 'notes', rows: 2, checksum: expect.any(String) },
],
notCarried: [
'the view recent',
expect.stringMatching(/^the default \(?lower\('A'\)\)? of notes\.odd$/),
'the index notes_hot on notes (partial or over an expression)',
],
},
]);
expect(summary.warnings).toEqual([
expect.stringMatching(/Not_An_Id.*plugin\.db is in a directory whose name is no plugin id/),
]);
const statements = pg.statements.join('\n');
expect(statements).toContain('create schema "trek_plugin_move_notes"');
expect(statements).toContain(
'create table "trek_plugin_move_notes"."notes" ("id" bigint generated by default as identity, ' +
'"body" text not null default \'x\', "score" double precision, "raw" bytea, "added" text default ' +
'to_char((CURRENT_TIMESTAMP AT TIME ZONE \'UTC\'), \'YYYY-MM-DD HH24:MI:SS\'), "odd" text, primary key ("id"))',
);
expect(statements).toContain('create unique index on "trek_plugin_move_notes"."notes" ("body")');
expect(pg.rowsOf('"trek_plugin_move_notes"."_plugin_migrations"')).toEqual([['001', '1700000000000']]);
expect(pg.sequences.get('"trek_plugin_move_notes"."notes"_seq')).toEqual({ value: '3', called: true });
expect(renderSummary(summary, 'postgres://db/trek')).toContain(
'Plugin move-notes: 2 table(s), 3 rows, into schema trek_plugin_move_notes.',
);
});
it("MOVE-012: a plugin database without a migration record gets the store's empty one", async () => {
const pluginDir = path.join(dir, 'plugins-data', 'bare-notes');
fs.mkdirSync(pluginDir, { recursive: true });
sqlite(path.join(pluginDir, 'plugin.db'), 'create table kv (k text primary key, v);');
await moveSqliteToPostgres(options());
expect(pg.statements).toContain(
'create table "trek_plugin_bare_notes"._plugin_migrations (id text primary key, applied_at bigint)',
);
expect(findPluginDataFiles(path.join(dir, 'nowhere'))).toEqual({ files: [], skipped: [] });
});
it('MOVE-013: copies an integer past 2^53 exactly and warns that TREK cannot read it back on Postgres', async () => {
sqlite(path.join(dir, 'travel.db'), 'update trips set id = 9007199254740993 where id = 2;');
const summary = await moveSqliteToPostgres(options());
expect(pg.rowsOf('"trips"').map((row) => row[0])).toEqual(['1', '9007199254740993']);
expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '9007199254740993', called: true });
expect(summary.warnings).toEqual(['trips holds 1 integer(s) beyond 2^53, which TREK refuses to read on Postgres.']);
});
});
describe('migrationLevelProblem', () => {
it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => {
const known = new Set([M1, M2]);
expect(migrationLevelProblem([M1, M2], known)).toBeNull();
expect(migrationLevelProblem(null, known)).toMatch(/no TREK migration history/);
expect(migrationLevelProblem([], known)).toMatch(/no TREK migration history/);
expect(migrationLevelProblem([M1, M2, 'Migration20990101000000_future'], known)).toMatch(
/a newer TREK migrated the SQLite file \(1 migration\(s\) this version does not know, the last Migration20990101000000_future\)/,
);
expect(migrationLevelProblem([M1], known)).toMatch(/1 migration\(s\) behind/);
});
});
describe('the pieces', () => {
it('MOVE-015: the plugin column rules, defaults and batches', () => {
const classes = (...names: string[]) => new Set(names);
expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer');
expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text');
expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes');
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text');
expect(pluginColumnKind('DOUBLE', classes())).toBe('float');
expect(pluginColumnKind('REAL', classes('text'))).toBe('text');
expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer');
expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float');
expect(pluginColumnKind('', classes('blob'))).toBe('bytes');
expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text');
expect(pluginDefault(null, 'text')).toBeNull();
expect(pluginDefault('NULL', 'text')).toBe('null');
expect(pluginDefault('(42)', 'integer')).toBe('42');
expect(pluginDefault('42', 'text')).toBe("'42'");
expect(pluginDefault('TRUE', 'integer')).toBe('1');
expect(pluginDefault('false', 'integer')).toBe('0');
expect(pluginDefault('true', 'text')).toBeNull();
expect(pluginDefault("'it''s'", 'text')).toBe("'it''s'");
expect(pluginDefault("strftime('%s','now')", 'integer')).toBe(
'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)',
);
expect(batchRows(2)).toBe(500);
expect(batchRows(200)).toBe(150);
expect(
insertSql(
{
target: '"t"',
columns: [
{ name: 'a"b', kind: 'text' },
{ name: 'c', kind: 'integer' },
],
},
2,
),
).toBe('insert into "t" ("a""b", "c") values ($1, $2), ($3, $4)');
expect(describePgError('plain')).toBe('plain');
expect(describePgError(new Error('bare'))).toBe('bare');
});
it('MOVE-016: a SQLite install holds accounts only with a user row; an unreadable file counts as holding them', () => {
const empty = path.join(dir, 'empty.db');
fs.writeFileSync(empty, '');
expect(sqliteInstallHoldsAccounts(empty)).toBe(false);
expect(sqliteInstallHoldsAccounts(path.join(dir, 'none.db'))).toBe(false);
expect(sqliteInstallHoldsAccounts(path.join(dir, 'travel.db'))).toBe(true);
expect(sqliteInstallHoldsAccounts(sqlite(path.join(dir, 'other.db'), 'create table notes (a);'))).toBe(false);
const broken = path.join(dir, 'broken.db');
fs.writeFileSync(broken, 'not a database at all, but long enough to be read as one');
expect(sqliteInstallHoldsAccounts(broken)).toBe(true);
// Reading never writes the file.
const before = fs.readFileSync(path.join(dir, 'travel.db'));
const source = SqliteSource.open(path.join(dir, 'travel.db'));
expect(source.count('users')).toBe(2);
source.close();
source.close();
expect(fs.readFileSync(path.join(dir, 'travel.db')).equals(before)).toBe(true);
});
});
describe('runMoveCommand', () => {
const env = (db: Record<string, unknown>) => () =>
({ db: { trekDbFile: path.join(dir, 'travel.db'), ...db }, plugins: { dataDir: '' } }) as unknown as AppEnv;
const postgres = env({ engine: 'postgres', url: 'postgres://trek:secret@db:5432/trek' });
const summary: MoveSummary = {
sqliteFile: 'travel.db',
tables: [{ label: 'users', rows: 2, checksum: 'x' }],
plugins: [],
sequences: 1,
warnings: ['careful'],
seconds: 0.4,
};
function io(overrides: Partial<MoveCommandIo> = {}) {
const out: string[] = [];
const err: string[] = [];
const close = vi.fn(async () => undefined);
return {
out,
err,
close,
io: {
env: postgres,
out: (line: string) => out.push(line),
err: (line: string) => err.push(line),
openOrm: async () => ({ close }) as unknown as AnyOrm,
move: vi.fn(async () => summary),
...overrides,
},
};
}
it('MOVE-017: prints the usage, refuses an unknown argument and a missing TREK_DB_URL', async () => {
const help = io();
expect(await runMoveCommand(['--help'], help.io)).toBe(0);
expect(help.out).toEqual([USAGE]);
const unknown = io();
expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1);
expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/);
const onSqlite = io({ env: env({ engine: 'sqlite' }) });
expect(await runMoveCommand([], onSqlite.io)).toBe(1);
expect(onSqlite.err[0]).toMatch(/Set TREK_DB_URL/);
});
it('MOVE-018: moves the given file, prints the summary without the password and closes the ORM', async () => {
const run = io();
expect(await runMoveCommand([`--from=${path.join(dir, 'other.db')}`], run.io)).toBe(0);
expect(run.io.move).toHaveBeenCalledWith(
expect.objectContaining({
sqliteFile: path.join(dir, 'other.db'),
pluginsDataDir: expect.stringMatching(/plugins-data$/),
}),
);
const printed = run.out.join('\n');
expect(printed).toContain('postgres://trek:***@db:5432/trek');
expect(printed).not.toContain('secret');
expect(printed).toMatch(
/1 tables, 2 rows, each table verified by row count and checksum[\s\S]*users: 2[\s\S]*Warning: careful/,
);
expect(run.close).toHaveBeenCalledWith(true);
});
it('MOVE-019: a failure prints its cause and still closes the ORM', async () => {
const failing = io({ move: vi.fn(async () => Promise.reject(new Error('connection lost'))) });
expect(await runMoveCommand([], failing.io)).toBe(1);
expect(failing.err[0]).toMatch(/The move failed; .*connection lost/s);
expect(failing.close).toHaveBeenCalled();
});
});
@@ -0,0 +1,175 @@
/**
* MOVEORD-001..005: the order the move fills the tables in, and the foreign
* keys it sets only once every row is in (src/db/engine/postgres/sqlite-move/order.ts),
* plus how the target's tables are paired with the SQLite file's (catalog.ts).
*/
import {
planTrekTables,
readTargetSchema,
type TargetSchema,
} from '../../../../../../src/db/engine/postgres/sqlite-move/catalog';
import { planCopyOrder } from '../../../../../../src/db/engine/postgres/sqlite-move/order';
import { FakePostgres } from '../../../../../helpers/fake-postgres-move';
import type { Transaction } from 'kysely';
import { describe, expect, it } from 'vitest';
describe('planCopyOrder', () => {
it('MOVEORD-001: puts every table after the tables it points at, ties by name', () => {
const plan = planCopyOrder(
['places', 'trips', 'users', 'days', 'categories'],
[
{ table: 'trips', columns: ['user_id'], references: 'users' },
{ table: 'days', columns: ['trip_id'], references: 'trips' },
{ table: 'places', columns: ['trip_id'], references: 'trips' },
{ table: 'places', columns: ['category_id'], references: 'categories' },
],
);
expect(plan.order).toEqual(['categories', 'users', 'trips', 'days', 'places']);
expect(plan.deferred.size).toBe(0);
});
it('MOVEORD-002: a table pointing at itself has those columns set after its rows are in', () => {
const plan = planCopyOrder(
['collab_messages'],
[
{ table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' },
{ table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' },
],
);
expect(plan.order).toEqual(['collab_messages']);
expect(plan.deferred.get('collab_messages')).toEqual(['reply_to']);
});
it('MOVEORD-003: a cycle is broken at the first table by name, deferring its keys into the cycle', () => {
const plan = planCopyOrder(
['b', 'a', 'c'],
[
{ table: 'a', columns: ['b_id'], references: 'b' },
{ table: 'b', columns: ['a_id'], references: 'a' },
{ table: 'c', columns: ['a_id'], references: 'a' },
],
);
expect(plan.order).toEqual(['a', 'b', 'c']);
expect([...plan.deferred]).toEqual([['a', ['b_id']]]);
});
it('MOVEORD-004: a key to a table the move does not copy is ignored', () => {
const plan = planCopyOrder(['trips'], [{ table: 'trips', columns: ['user_id'], references: 'users' }]);
expect(plan.order).toEqual(['trips']);
});
});
describe('planTrekTables', () => {
const target: TargetSchema = {
tables: [
{ name: 'mikro_orm_migrations', columns: [{ name: 'name', dataType: 'text', nullable: false }], primaryKey: [] },
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'reply_to', dataType: 'bigint', nullable: true },
],
primaryKey: ['id'],
},
{
name: 'users',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'avatar', dataType: 'bytea', nullable: true },
],
primaryKey: ['id'],
},
],
foreignKeys: [{ table: 'messages', columns: ['reply_to'], references: 'messages' }],
sequences: [{ table: 'users', column: 'id', sequence: 'public.users_id_seq' }],
};
it('MOVEORD-005: pairs the tables by name, leaves the history out and quotes every name', () => {
const plan = planTrekTables(
target,
new Map([
['mikro_orm_migrations', ['name']],
['messages', ['id', 'reply_to']],
['users', ['id', 'avatar']],
]),
);
expect(plan.problems).toEqual([]);
expect(plan.specs.map((spec) => spec.target)).toEqual(['"messages"', '"users"']);
expect(plan.specs[0]).toMatchObject({ deferred: ['reply_to'], primaryKey: ['id'], label: 'messages' });
expect(plan.specs[1].columns).toEqual([
{ name: 'id', kind: 'integer' },
{ name: 'avatar', kind: 'bytes' },
]);
expect(plan.sequences).toEqual([
{ table: 'users', column: 'id', sequence: 'public.users_id_seq', target: '"users"' },
]);
});
it('MOVEORD-006: names every difference between the two schemas instead of copying around it', () => {
const odd: TargetSchema = {
...target,
tables: [
...target.tables.filter((table) => table.name !== 'messages'),
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'reply_to', dataType: 'bigint', nullable: false },
{ name: 'sent_at', dataType: 'timestamp with time zone', nullable: true },
],
primaryKey: ['id'],
},
{ name: 'extra', columns: [], primaryKey: [] },
],
};
const plan = planTrekTables(
odd,
new Map([
['messages', ['id', 'reply_to', 'sent_at', 'legacy']],
['users', ['id']],
['leftover', ['id']],
]),
);
expect(plan.problems).toEqual([
'the SQLite table leftover has no Postgres table',
'the Postgres table extra has no SQLite table',
'the SQLite column messages.legacy has no Postgres column',
'the Postgres column messages.sent_at has the type timestamp with time zone, which the move does not know',
'messages.reply_to refers back to a table filled later and cannot be NULL meanwhile',
'the Postgres column users.avatar has no SQLite column',
]);
});
});
describe('readTargetSchema', () => {
it('MOVEORD-007: reads the columns, keys and sequences of the current schema', async () => {
const pg = new FakePostgres([
{
name: 'trips',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint' },
],
primaryKey: ['id'],
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
serial: ['id'],
},
{ name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] },
]);
const schema = await readTargetSchema(pg.asKysely() as Transaction<object>);
expect(schema.tables).toEqual([
{
name: 'trips',
primaryKey: ['id'],
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint', nullable: true },
],
},
{ name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }] },
]);
expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]);
expect(schema.sequences).toEqual([{ table: 'trips', column: 'id', sequence: 'public.trips_id_seq' }]);
});
});
@@ -0,0 +1,179 @@
/**
* MOVEVAL-001..010: how a SQLite value becomes the value a Postgres column of
* the move's target takes, the text both sides are compared in, and the
* order-free table checksum (src/db/engine/postgres/sqlite-move/values.ts).
*/
import {
beyondSafeInteger,
canonicalFromPostgres,
canonicalFromSent,
columnKind,
convertValue,
stableJson,
TableChecksum,
type ColumnKind,
} from '../../../../../../src/db/engine/postgres/sqlite-move/values';
import fs from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
const ok = (value: unknown) => ({ ok: true, value });
describe('columnKind', () => {
it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => {
const sql = fs.readFileSync(path.join(__dirname, '../../../../../../assets/postgres/baseline.sql'), 'utf8');
const types = new Set(
[
...sql.matchAll(
/" (bigserial|bigint|integer|smallint|text|varchar\(\d+\)|double precision|real|jsonb|json|bytea|boolean|numeric|timestamptz|date|uuid)(?=[ ,)])/g,
),
].map((match) => match[1]),
);
const asCatalog: Record<string, string> = { bigserial: 'bigint' };
for (const type of types) {
const dataType = asCatalog[type] ?? (type.startsWith('varchar') ? 'character varying' : type);
expect(columnKind(dataType), type).not.toBeNull();
}
});
it('MOVEVAL-002: has no rule for a type it does not know, so the move refuses instead of guessing', () => {
expect(columnKind('timestamp with time zone')).toBeNull();
expect(columnKind('boolean')).toBeNull();
expect(columnKind('smallint')).toBe('integer');
expect(columnKind('real')).toBe('float');
expect(columnKind('character')).toBe('text');
expect(columnKind('json')).toBe('json');
});
});
describe('convertValue', () => {
it('MOVEVAL-003: integers keep all 64 bits and accept whole numbers in any spelling SQLite stored them in', () => {
expect(convertValue(9007199254740993n, 'integer')).toEqual(ok('9007199254740993'));
expect(convertValue(42, 'integer')).toEqual(ok('42'));
expect(convertValue(' -7 ', 'integer')).toEqual(ok('-7'));
expect(convertValue(null, 'integer')).toEqual(ok(null));
expect(convertValue(1.5, 'integer')).toMatchObject({ ok: false, reason: 'the value 1.5 is not a whole number' });
expect(convertValue('abc', 'integer')).toMatchObject({ ok: false, reason: 'the text "abc" is not a whole number' });
expect(convertValue('99999999999999999999', 'integer')).toMatchObject({
ok: false,
reason: expect.stringMatching(/does not fit in 64 bits/),
});
expect(convertValue(new Uint8Array([1]), 'integer')).toMatchObject({
ok: false,
reason: 'a 1-byte BLOB is not a whole number',
});
});
it('MOVEVAL-004: floats take numbers and numeric text, never a value a double cannot hold', () => {
expect(convertValue(1.25, 'float')).toEqual(ok(1.25));
expect(convertValue(12n, 'float')).toEqual(ok(12));
expect(convertValue('3.5e2', 'float')).toEqual(ok(350));
expect(convertValue(9007199254740993n, 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/more digits than a double holds/),
});
expect(convertValue('north', 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not a number/),
});
expect(convertValue('', 'float')).toMatchObject({ ok: false });
});
it('MOVEVAL-005: text takes strings and numbers, and refuses a NUL Postgres cannot store or a BLOB', () => {
expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa'));
expect(convertValue(7n, 'text')).toEqual(ok('7'));
expect(convertValue(2.5, 'text')).toEqual(ok('2.5'));
expect(convertValue('a\u0000b', 'text')).toMatchObject({
ok: false,
reason: expect.stringMatching(/NUL character/),
});
expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not text/),
});
const long = 'x'.repeat(60);
expect(convertValue(long, 'integer')).toMatchObject({
reason: `the text "${'x'.repeat(40)}..." is not a whole number`,
});
});
it('MOVEVAL-006: JSON must parse and hold no NUL, escaped or not', () => {
expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}'));
expect(convertValue(5n, 'json')).toEqual(ok('5'));
expect(convertValue('{broken', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not valid JSON/),
});
expect(convertValue('"\\u0000"', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/jsonb cannot store/),
});
expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not JSON text/),
});
});
it('MOVEVAL-007: bytes take a BLOB, or the UTF-8 of text, and refuse a number', () => {
expect(convertValue(new Uint8Array([0, 255]), 'bytes')).toEqual(ok(Buffer.from([0, 255])));
expect(convertValue('hé', 'bytes')).toEqual(ok(Buffer.from('hé', 'utf8')));
expect(convertValue(3n, 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not a BLOB/) });
});
it('MOVEVAL-008: flags an integer past 2^53, which the Postgres driver refuses to read', () => {
expect(beyondSafeInteger('9007199254740993')).toBe(true);
expect(beyondSafeInteger('-9007199254740993')).toBe(true);
expect(beyondSafeInteger('9007199254740991')).toBe(false);
expect(beyondSafeInteger(12)).toBe(false);
expect(beyondSafeInteger('text')).toBe(false);
});
});
describe('the compared text', () => {
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSent>[0], string]> = [
['integer', '-12', '-12'],
['float', 0.30000000000000004, '0.30000000000000004'],
['float', 1e21, '1e+21'],
['text', 'São Paulo', 'São Paulo'],
['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'],
['bytes', Buffer.from([0, 171]), '\\x00ab'],
];
it.each(cases)('MOVEVAL-009: %s values compare equal as sent and as Postgres prints them', (kind, sent, printed) => {
expect(canonicalFromSent(sent, kind)).toBe(canonicalFromPostgres(printed, kind));
expect(canonicalFromSent(null, kind)).toBeNull();
expect(canonicalFromPostgres(null, kind)).toBeNull();
});
it('MOVEVAL-009b: text sent for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
expect(canonicalFromSent('ab', 'bytes')).toBe('6162');
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSent(1e20, 'float'));
expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain');
expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}');
});
it('MOVEVAL-010: the checksum ignores row order and catches a changed, missing or doubled row', () => {
const sum = (rows: Array<Array<string | null>>) => {
const checksum = new TableChecksum();
for (const row of rows) checksum.add(row);
return checksum.hex();
};
const rows = [
['1', 'a'],
['2', null],
];
expect(sum(rows)).toBe(sum([...rows].reverse()));
expect(sum(rows)).toHaveLength(64);
expect(sum(rows)).not.toBe(
sum([
['1', 'a'],
['2', 'b'],
]),
);
expect(sum(rows)).not.toBe(sum([['1', 'a']]));
expect(sum(rows)).not.toBe(sum([...rows, ['2', null]]));
// A NULL and the text "null" are different values.
expect(sum([['null']])).not.toBe(sum([[null]]));
});
});