feat(db): move a SQLite install into an empty Postgres database

npm run db:move-to-postgres (dist/move-to-postgres.js in the image) copies
a SQLite file at this build's migration level into an empty Postgres
database. It refuses a file at another level, rows whose references point
at nothing and a target holding any table or plugin schema, creates the
schema with the engine's own bootstrap, then in one transaction copies every
table in foreign key order (setting a self-reference once its rows are in),
converts each value to its column's type and refuses one Postgres cannot
hold exactly, copies each plugin's plugin.db into its schema, moves the
sequences past the copied ids and compares every table's row count and an
order-free checksum over all rows before it commits. A failure drops the
schema again. The SQLite files and the uploads are never written.
This commit is contained in:
Maurice
2026-11-17 13:41:20 +01:00
parent 1ea08dcf0a
commit 55607ff2f9
14 changed files with 2535 additions and 0 deletions
@@ -0,0 +1,163 @@
import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
import { planCopyOrder, type ForeignKey } from './order';
import { columnKind } from './values';
/**
* TREK's tables as the target database holds them after the engine's
* bootstrap, and how each one is filled from the table of the same name in
* the SQLite file. The target's catalog decides the column types; the SQLite
* file only has to hold the same tables and columns, which a file at this
* build's migration level does.
*/
/** The migration history: each side keeps its own, and the bootstrap wrote the target's. */
const HISTORY_TABLE = 'mikro_orm_migrations';
interface TargetColumn {
name: string;
dataType: string;
nullable: boolean;
}
interface TargetTable {
name: string;
columns: TargetColumn[];
primaryKey: string[];
}
export interface TargetSchema {
tables: TargetTable[];
foreignKeys: ForeignKey[];
/** Every column a sequence fills, by table. */
sequences: Array<{ table: string; column: string; sequence: string }>;
}
/** Reads the tables, keys and sequences of the current schema. */
export async function readTargetSchema(trx: Trx): Promise<TargetSchema> {
const columns = await run<{ table_name: string; column_name: string; data_type: string; is_nullable: string }>(
trx,
'select c.table_name, c.column_name, c.data_type, c.is_nullable from information_schema.columns c ' +
'join information_schema.tables t on t.table_schema = c.table_schema and t.table_name = c.table_name ' +
"where c.table_schema = current_schema() and t.table_type = 'BASE TABLE' order by c.table_name, c.ordinal_position",
);
const constraints = await run<{ kind: string; table_name: string; target_name: string | null; columns: string[] }>(
trx,
'select c.contype as kind, src.relname as table_name, dst.relname as target_name, ' +
'array_agg(a.attname::text order by k.ord) as columns from pg_constraint c ' +
'join pg_class src on src.oid = c.conrelid left join pg_class dst on dst.oid = c.confrelid ' +
'cross join lateral unnest(c.conkey) with ordinality as k(attnum, ord) ' +
'join pg_attribute a on a.attrelid = c.conrelid and a.attnum = k.attnum ' +
"where c.connamespace = current_schema()::regnamespace and c.contype in ('p', 'f') " +
'group by c.oid, c.contype, src.relname, dst.relname',
);
const sequences = await run<{ table_name: string; column_name: string; sequence: string }>(
trx,
"select table_name, column_name, pg_get_serial_sequence(format('%I', table_name), column_name) as sequence " +
'from information_schema.columns where table_schema = current_schema() ' +
"and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name",
);
const tables = new Map<string, TargetTable>();
for (const column of columns) {
const table = tables.get(column.table_name) ?? { name: column.table_name, columns: [], primaryKey: [] };
table.columns.push({
name: column.column_name,
dataType: column.data_type,
nullable: column.is_nullable === 'YES',
});
tables.set(column.table_name, table);
}
const foreignKeys: ForeignKey[] = [];
for (const constraint of constraints) {
if (constraint.kind === 'p') {
const table = tables.get(constraint.table_name);
if (table) table.primaryKey = constraint.columns;
} else if (constraint.target_name) {
foreignKeys.push({
table: constraint.table_name,
columns: constraint.columns,
references: constraint.target_name,
});
}
}
return {
tables: [...tables.values()],
foreignKeys,
sequences: sequences.map((row) => ({ table: row.table_name, column: row.column_name, sequence: row.sequence })),
};
}
/** The tables and columns of the SQLite file, as the move compares them with the target. */
export type SourceTables = ReadonlyMap<string, readonly string[]>;
interface TrekCopyPlan {
/** The tables in the order they are filled. */
specs: TableSpec[];
sequences: CopiedSequence[];
/** Why the two schemas do not match; the move refuses when there is any. */
problems: string[];
}
/**
* Pairs every table of the target with the SQLite table of the same name and
* puts them in foreign key order. Anything the two do not share, or a column
* type the move has no rule for, is a problem: copying around it would lose
* data or guess.
*/
export function planTrekTables(target: TargetSchema, source: SourceTables): TrekCopyPlan {
const problems: string[] = [];
const tables = target.tables.filter((table) => table.name !== HISTORY_TABLE);
const names = new Set(tables.map((table) => table.name));
for (const name of source.keys()) {
if (name !== HISTORY_TABLE && !names.has(name)) problems.push(`the SQLite table ${name} has no Postgres table`);
}
const plan = planCopyOrder(
tables.map((table) => table.name),
target.foreignKeys,
);
const byName = new Map(tables.map((table) => [table.name, table]));
const specs: TableSpec[] = [];
for (const name of plan.order) {
const table = byName.get(name);
const sourceColumns = source.get(name);
if (!table) continue;
if (!sourceColumns) {
problems.push(`the Postgres table ${name} has no SQLite table`);
continue;
}
for (const column of sourceColumns) {
if (!table.columns.some((candidate) => candidate.name === column)) {
problems.push(`the SQLite column ${name}.${column} has no Postgres column`);
}
}
const columns = table.columns.flatMap((column) => {
const kind = columnKind(column.dataType);
if (!sourceColumns.includes(column.name))
problems.push(`the Postgres column ${name}.${column.name} has no SQLite column`);
else if (kind === null)
problems.push(
`the Postgres column ${name}.${column.name} has the type ${column.dataType}, which the move does not know`,
);
else return [{ name: column.name, kind }];
return [];
});
const deferred = plan.deferred.get(name) ?? [];
for (const column of deferred) {
if (!table.columns.find((candidate) => candidate.name === column)?.nullable) {
problems.push(`${name}.${column} refers back to a table filled later and cannot be NULL meanwhile`);
}
}
specs.push({
source: name,
target: quoteIdentifier(name),
label: name,
columns,
deferred,
primaryKey: table.primaryKey,
});
}
const sequences = target.sequences
.filter((entry) => names.has(entry.table))
.map((entry) => ({ ...entry, target: quoteIdentifier(entry.table) }));
return { specs, sequences, problems };
}
@@ -0,0 +1,104 @@
import { readEnv, type AppEnv } from '../../../../app-config';
import { resolveDataPaths } from '../../../../app-config/data-paths';
import { knownMigrationNames } from '../../../known-migrations';
import type { AnyOrm } from '../../../orm';
import { buildOrmConfig, resolveEngine } from '../../index';
import { redactUrl } from '../../redact-url';
import { readPostgresBaseline } from '../baseline';
import { MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions } from './move';
import { MikroORM } from '@mikro-orm/core';
import path from 'node:path';
/**
* The command around the move (`src/move-to-postgres.ts` runs it): reads
* TREK_DB_URL as the server does, takes the SQLite file from the data
* directory or `--from=<file>`, the plugin data from the plugins' data
* directory, runs the move and prints its summary. Answers the exit code.
*/
export const USAGE =
'Usage: move-to-postgres [--from=<SQLite file>]\n' +
'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.';
export interface MoveCommandIo {
env: () => AppEnv;
out: (line: string) => void;
err: (line: string) => void;
/** An ORM on the target database, from the Postgres engine's own config. */
openOrm: (env: AppEnv) => Promise<AnyOrm>;
move: (options: MoveOptions) => ReturnType<typeof moveSqliteToPostgres>;
}
/** The ORM the move runs on: the engine's config, without entity discovery (the move runs SQL only). */
async function openTargetOrm(env: AppEnv): Promise<AnyOrm> {
const config = buildOrmConfig(resolveEngine(env.db));
return MikroORM.init({
...config,
entities: [],
entitiesTs: [],
discovery: { warnWhenNoEntities: false },
migrations: { ...config.migrations, silent: true },
});
}
const defaultIo: MoveCommandIo = {
env: readEnv,
out: (line) => process.stdout.write(`${line}\n`),
err: (line) => process.stderr.write(`${line}\n`),
openOrm: openTargetOrm,
move: moveSqliteToPostgres,
};
const FROM = '--from=';
async function run(argv: readonly string[], io: MoveCommandIo): Promise<number> {
if (argv.includes('--help') || argv.includes('-h')) {
io.out(USAGE);
return 0;
}
const unknown = argv.find((arg) => !arg.startsWith(FROM));
if (unknown !== undefined) throw new MoveRefusal(`Unknown argument ${unknown}.\n${USAGE}`);
const env = io.env();
if (env.db.engine !== 'postgres')
throw new MoveRefusal('Set TREK_DB_URL to the empty Postgres database to move into.');
const paths = resolveDataPaths(env);
const from = argv.find((arg) => arg.startsWith(FROM))?.slice(FROM.length);
const sqliteFile = path.resolve(from || paths.dbFile);
const target = redactUrl(env.db.url);
io.out(`Moving ${sqliteFile} into ${target}`);
const orm = await io.openOrm(env);
try {
const summary = await io.move({
sqliteFile,
pluginsDataDir: env.plugins.dataDir || path.join(paths.dataDir, 'plugins-data'),
orm,
known: knownMigrationNames(),
baseline: readPostgresBaseline,
log: (line) => io.out(` ${line}`),
});
io.out('');
io.out(renderSummary(summary, target));
return 0;
} finally {
await orm.close(true);
}
}
/** Runs the command and answers its exit code: 0 once the move committed, 1 for a refusal or a failure. */
export async function runMoveCommand(argv: readonly string[], io: Partial<MoveCommandIo> = {}): Promise<number> {
const resolved = { ...defaultIo, ...io };
try {
return await run(argv, resolved);
} catch (err) {
if (err instanceof MoveRefusal) resolved.err(`\nThe move was refused: ${err.message}`);
else {
const text = err instanceof Error ? (err.stack ?? err.message) : String(err);
resolved.err(
`\nThe move failed; the Postgres database is left as the move found it unless this says otherwise: ${text}`,
);
}
return 1;
}
}
@@ -0,0 +1,301 @@
import type { SqliteSource } from '../../sqlite/sqlite-source';
import {
beyondSafeInteger,
canonicalFromPostgres,
canonicalFromSent,
convertValue,
TableChecksum,
type ColumnKind,
type PostgresValue,
type SqliteValue,
} from './values';
import { CompiledQuery, type Kysely, type Transaction } from 'kysely';
/**
* Copying one table from the SQLite file into Postgres and proving the copy,
* inside the move's transaction. The same steps serve TREK's own tables and a
* plugin's: read the rows one at a time, convert each value to its column's
* type, insert in batches, then read the table back and compare its row count
* and checksum with what was sent.
*/
export type Trx = Transaction<object>;
/** Rows per INSERT at most; fewer for a wide table, so a batch stays far below Postgres's 65535 parameters. */
const MAX_BATCH_ROWS = 500;
const MAX_BATCH_PARAMS = 30_000;
/** Rows a verification cursor fetches at once. */
const FETCH_ROWS = 2_000;
/** The most refused values a refusal lists one by one. */
export const MAX_LISTED_PROBLEMS = 20;
/** One column of a table the move copies. */
interface CopyColumn {
name: string;
kind: ColumnKind;
}
/** What the move copies from one SQLite table into one Postgres table. */
export interface TableSpec {
/** The table in the SQLite file. */
source: string;
/** The Postgres table, quoted and qualified as SQL needs it. */
target: string;
/** A name for messages: `users`, or `plugin my-plugin: notes`. */
label: string;
columns: CopyColumn[];
/** Foreign key columns inserted as NULL first and set once every table is in (`order.ts`). */
deferred: string[];
/** The primary key's columns, which the deferred update finds a row by. */
primaryKey: string[];
}
/** A row's deferred values, kept until every table is in. */
interface DeferredRow {
key: PostgresValue[];
values: PostgresValue[];
}
/** What copying one table found. */
export interface CopiedTable {
spec: TableSpec;
rows: number;
checksum: string;
/** Values the Postgres column cannot hold exactly, listed up to `MAX_LISTED_PROBLEMS`; the move refuses on any. */
problems: string[];
problemCount: number;
/** Integer values beyond 2^53, which TREK's Postgres driver refuses to read back. */
unsafeIntegers: number;
deferredRows: DeferredRow[];
}
/** `name` as a Postgres identifier. */
export function quoteIdentifier(name: string): string {
return `"${name.replaceAll('"', '""')}"`;
}
/** Runs one statement, in the move's transaction or on the pool, and returns its rows. */
export async function run<R = Record<string, unknown>>(
trx: Kysely<object>,
sql: string,
params: unknown[] = [],
): Promise<R[]> {
const result = await trx.executeQuery<R>(CompiledQuery.raw(sql, params));
return result.rows;
}
/**
* Runs `work` for each item, one after the other, each started once the one
* before it has finished: the move's statements share one transaction, which
* runs one statement at a time.
*/
export async function inOrder<T>(items: readonly T[], work: (item: T) => Promise<void>): Promise<void> {
await items.reduce<Promise<void>>((previous, item) => previous.then(() => work(item)), Promise.resolve());
}
/** A Postgres error with its detail and the constraint it broke, which `pg` keeps apart from the message. */
export function describePgError(err: unknown): string {
if (!(err instanceof Error)) return String(err);
const extra = err as Error & { detail?: string; constraint?: string };
const parts = [err.message];
if (extra.constraint) parts.push(`constraint ${extra.constraint}`);
if (extra.detail) parts.push(extra.detail);
return parts.join('; ');
}
/** How many rows one INSERT carries for a table of `columns` columns. */
export function batchRows(columns: number): number {
return Math.max(1, Math.min(MAX_BATCH_ROWS, Math.floor(MAX_BATCH_PARAMS / Math.max(1, columns))));
}
/** The INSERT of `rows` rows into the table, with numbered parameters row by row. */
export function insertSql(spec: Pick<TableSpec, 'target' | 'columns'>, rows: number): string {
const names = spec.columns.map((column) => quoteIdentifier(column.name)).join(', ');
const width = spec.columns.length;
const tuples = Array.from(
{ length: rows },
(_, row) => `(${spec.columns.map((_column, index) => `$${row * width + index + 1}`).join(', ')})`,
);
return `insert into ${spec.target} (${names}) values ${tuples.join(', ')}`;
}
async function insertBatch(trx: Trx, spec: TableSpec, batch: PostgresValue[][]): Promise<void> {
if (batch.length === 0) return;
try {
await run(trx, insertSql(spec, batch.length), batch.flat());
} catch (err) {
throw new Error(`Postgres refused rows of ${spec.label}: ${describePgError(err)}`, { cause: err });
}
}
/**
* Copies the rows of one table. Every value is converted first; once one
* cannot be, nothing more is inserted, but the rest of the table is still read
* so the refusal can list every problem at once. With `insert: false` it only
* reads and converts, once an earlier table was refused.
*/
export async function copyTable(
trx: Trx,
source: SqliteSource,
spec: TableSpec,
options: { insert: boolean } = { insert: true },
): Promise<CopiedTable> {
const result: CopiedTable = {
spec,
rows: 0,
checksum: '',
problems: [],
problemCount: 0,
unsafeIntegers: 0,
deferredRows: [],
};
const checksum = new TableChecksum();
const deferredAt = spec.deferred.map((name) => spec.columns.findIndex((column) => column.name === name));
const keyAt = spec.primaryKey.map((name) => spec.columns.findIndex((column) => column.name === name));
const size = batchRows(spec.columns.length);
/** How a refusal names a row: by its primary key, or by its place in the table without one. */
const rowName = (raw: unknown[]): string => {
if (keyAt.length === 0 || keyAt.includes(-1)) return `row ${result.rows}`;
return `row ${spec.primaryKey.map((name, index) => `${name}=${String(raw[keyAt[index]])}`).join(', ')}`;
};
/** One SQLite row as the values sent to Postgres, or null once a value was refused. */
const convertRow = (raw: unknown[]): PostgresValue[] | null => {
result.rows++;
const values: PostgresValue[] = [];
spec.columns.forEach((column, index) => {
const converted = convertValue(raw[index] as SqliteValue, column.kind);
if (converted.ok) {
values.push(converted.value);
if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++;
return;
}
values.push(null);
result.problemCount++;
if (result.problems.length < MAX_LISTED_PROBLEMS) {
result.problems.push(`${spec.label} ${rowName(raw)}, column ${column.name}: ${converted.reason}`);
}
});
if (result.problemCount > 0) return null;
checksum.add(values.map((value, index) => canonicalFromSent(value, spec.columns[index].kind)));
if (deferredAt.length > 0) {
const later = deferredAt.map((index) => values[index]);
if (later.some((value) => value !== null)) {
result.deferredRows.push({ key: keyAt.map((index) => values[index]), values: later });
}
for (const index of deferredAt) values[index] = null;
}
return values;
};
const rows = source.rows(
spec.source,
spec.columns.map((column) => column.name),
);
/** Reads up to one batch of rows; `done` once the table has no more. */
const nextBatch = (): { batch: PostgresValue[][]; done: boolean } => {
const batch: PostgresValue[][] = [];
while (batch.length < size) {
const next = rows.next();
if (next.done) return { batch, done: true };
const values = convertRow(next.value);
if (values) batch.push(values);
}
return { batch, done: false };
};
const pump = async (): Promise<void> => {
const { batch, done } = nextBatch();
// Always awaited, so the next batch starts on a fresh stack even when nothing is inserted.
await (options.insert && result.problemCount === 0 ? insertBatch(trx, spec, batch) : Promise.resolve());
if (!done) await pump();
};
try {
await pump();
} finally {
// An iterator left open keeps the SQLite handle busy.
rows.return?.();
}
result.checksum = checksum.hex();
return result;
}
/** Sets the foreign key columns `copyTable` held back, once every table is in. */
export async function applyDeferred(trx: Trx, copied: CopiedTable): Promise<void> {
const { spec } = copied;
if (copied.deferredRows.length === 0) return;
if (spec.primaryKey.length === 0) throw new Error(`${spec.label} has no primary key to set its deferred columns by.`);
const width = spec.deferred.length;
const sets = spec.deferred.map((name, index) => `${quoteIdentifier(name)} = $${index + 1}`).join(', ');
const where = spec.primaryKey.map((name, index) => `${quoteIdentifier(name)} = $${width + index + 1}`).join(' and ');
const sql = `update ${spec.target} set ${sets} where ${where}`;
// One row at a time: only the rows of a table that points at itself, a handful in practice.
await inOrder(copied.deferredRows, async (row) => {
try {
await run(trx, sql, [...row.values, ...row.key]);
} catch (err) {
throw new Error(`Postgres refused a reference in ${spec.label}: ${describePgError(err)}`, { cause: err });
}
});
}
/**
* Reads a table back through a cursor, every column as text, and answers its
* row count and checksum, which must equal what `copyTable` sent.
*/
export async function readBack(trx: Trx, spec: TableSpec): Promise<{ rows: number; checksum: string }> {
const checksum = new TableChecksum();
const list = spec.columns.map((column, index) => `${quoteIdentifier(column.name)}::text as c${index}`).join(', ');
const fetchAll = async (): Promise<void> => {
const rows = await run<Record<string, string | null>>(trx, `fetch forward ${FETCH_ROWS} from trek_move_verify`);
for (const row of rows) {
checksum.add(spec.columns.map((column, index) => canonicalFromPostgres(row[`c${index}`] ?? null, column.kind)));
}
if (rows.length === FETCH_ROWS) await fetchAll();
};
await run(trx, `declare trek_move_verify no scroll cursor for select ${list} from ${spec.target}`);
try {
await fetchAll();
} finally {
await run(trx, 'close trek_move_verify');
}
return { rows: checksum.rows, checksum: checksum.hex() };
}
/** A sequence behind a copied column: `table` names it in the SQLite file, `target` in Postgres. */
export interface CopiedSequence {
table: string;
target: string;
column: string;
sequence: string;
}
/**
* Moves every sequence of the copied tables past the ids they now hold, and
* past the highest id an AUTOINCREMENT table ever handed out in the SQLite
* file, so a new row never takes an id SQLite would not have reused.
*/
export async function advanceSequences(
trx: Trx,
sequences: readonly CopiedSequence[],
autoincrementSeen: ReadonlyMap<string, bigint>,
): Promise<number> {
await inOrder(sequences, async (entry) => {
const [row] = await run<{ max: string | null }>(
trx,
`select max(${quoteIdentifier(entry.column)})::text as max from ${entry.target}`,
);
const seen = autoincrementSeen.get(entry.table) ?? null;
const copied = row?.max ? BigInt(row.max) : null;
const highest = seen !== null && (copied === null || seen > copied) ? seen : copied;
// Nothing copied: the next id is 1, as on a new database.
const called = highest !== null && highest >= 1n;
await run(trx, 'select setval($1::regclass, $2::bigint, $3::boolean)', [
entry.sequence,
called ? highest.toString() : '1',
called,
]);
});
return sequences.length;
}
@@ -0,0 +1,358 @@
import { NO_SAFETY_NET } from '../../../legacy-baseline';
import type { AnyOrm } from '../../../orm';
import { SqliteSource } from '../../sqlite/sqlite-source';
import type { PostgresBaseline } from '../baseline';
import { CLEAR_SCHEMA_SQL } from '../clear-schema';
import { PLUGIN_SCHEMA_PREFIX } from '../plugin-schema';
import { bootstrapPostgresSchema } from '../schema-bootstrap';
import { planTrekTables, readTargetSchema, type SourceTables } from './catalog';
import {
advanceSequences,
applyDeferred,
copyTable,
inOrder,
MAX_LISTED_PROBLEMS,
readBack,
run,
type CopiedSequence,
type CopiedTable,
type Trx,
} from './copy';
import {
createPluginSchema,
findPluginDataFiles,
planPluginData,
resolveIdentitySequences,
type PluginPlan,
} from './plugin-data';
import type { Kysely } from 'kysely';
/**
* Moves a SQLite install into an empty Postgres database (`npm run
* db:move-to-postgres`, `dist/move-to-postgres.js` in the image).
*
* 1. The SQLite file must be at this build's migration level: an older one is
* migrated by starting this TREK on it once, a newer one belongs to the
* TREK that wrote it. Nothing else is checked by guess.
* 2. The Postgres database must hold no table and no plugin schema.
* 3. The schema is created exactly as a first boot creates it (the engine's
* bootstrap: the baseline and the migrations after its cut), without the
* seeders, since every row comes from the file.
* 4. In one transaction: the rows the bootstrap seeded are cleared, every
* table is filled in foreign key order, each plugin's `plugin.db` goes into
* its schema, the sequences are moved past the copied ids, and every table
* is read back and compared with what was sent (row count and a checksum
* over all rows). Only a copy that matches is committed.
* 5. Anything that fails leaves the database as it was found: the
* transaction rolls back and the schema the bootstrap created is dropped.
*
* The SQLite files are opened read-only and never written. Uploads are not
* the database's: they stay where the storage driver keeps them.
*/
/** A refusal the operator can act on: printed without a stack. */
export class MoveRefusal extends Error {}
export interface MoveOptions {
/** The SQLite file to move: `data/travel.db` by default. */
sqliteFile: string;
/** Where the plugins keep their data directories (`plugin.db` files). */
pluginsDataDir: string;
/** An ORM on the target database, from the Postgres engine's own config. */
orm: AnyOrm;
/** The migrations this build ships. */
known: ReadonlySet<string>;
baseline: () => PostgresBaseline;
/** Where progress lines go. */
log: (line: string) => void;
}
interface MovedTable {
label: string;
rows: number;
checksum: string;
}
export interface MoveSummary {
sqliteFile: string;
tables: MovedTable[];
plugins: Array<{ pluginId: string; schema: string; tables: MovedTable[]; notCarried: string[] }>;
sequences: number;
warnings: string[];
seconds: number;
}
/**
* Why the SQLite file is not at this build's migration level, or null when it
* is: it must have recorded exactly the migrations this build ships.
*/
export function migrationLevelProblem(recorded: readonly string[] | null, known: ReadonlySet<string>): string | null {
const startOnce =
'Start this TREK version on it once, without TREK_DB_URL, so it brings the file up to date; stop it; then run the move again.';
if (recorded === null || recorded.length === 0) {
return `the SQLite file has no TREK migration history (an install from before 5.0, or not a TREK database). ${startOnce}`;
}
const unknown = recorded.filter((name) => !known.has(name));
if (unknown.length > 0) {
return (
`a newer TREK migrated the SQLite file (${unknown.length} migration(s) this version does not know, ` +
`the last ${[...unknown].sort().at(-1)}). Run the move with the TREK version that last ran on it.`
);
}
const have = new Set(recorded);
const missing = [...known].filter((name) => !have.has(name));
if (missing.length > 0) {
return `the SQLite file is ${missing.length} migration(s) behind this TREK version. ${startOnce}`;
}
return null;
}
function listed(problems: readonly string[], total: number): string {
const lines = problems.map((problem) => ` - ${problem}`);
if (total > problems.length) lines.push(` - and ${total - problems.length} more`);
return lines.join('\n');
}
/** Refuses a database that holds a table or a plugin schema: the move fills an empty database only. */
async function refuseNonEmptyTarget(db: Kysely<object>): Promise<void> {
const tables = await run<{ table_name: string }>(
db,
'select table_name from information_schema.tables where table_schema = current_schema() order by table_name limit 5',
);
const schemas = await run<{ nspname: string }>(
db,
'select nspname from pg_namespace where starts_with(nspname, $1) order by nspname limit 5',
[PLUGIN_SCHEMA_PREFIX],
);
const found = [...tables.map((row) => row.table_name), ...schemas.map((row) => `schema ${row.nspname}`)];
if (found.length === 0) return;
throw new MoveRefusal(
`The Postgres database is not empty (${found.join(', ')}). The move fills an empty database only, so it never ` +
'mixes two installs. Point TREK_DB_URL at a new, empty database. If an earlier move stopped part of the way, ' +
'drop that database and create it again.',
);
}
/** Opens every plugin's data file and plans its copy; the sources stay open for the copy. */
function planPlugins(root: string, warnings: string[]): Array<{ source: SqliteSource; plan: PluginPlan }> {
const { files, skipped } = findPluginDataFiles(root);
for (const file of skipped) warnings.push(`${file} is in a directory whose name is no plugin id; it was not moved.`);
const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = [];
for (const { pluginId, file } of files) {
const source = SqliteSource.open(file);
plugins.push({ source, plan: planPluginData(pluginId, source) });
}
return plugins;
}
/** Compares a copied table with what Postgres holds now: the SQLite count, the rows sent and the checksum. */
async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promise<string | null> {
const back = await readBack(trx, copied.spec);
if (back.rows !== sqliteCount || copied.rows !== sqliteCount) {
return `${copied.spec.label}: the SQLite file holds ${sqliteCount} rows, ${copied.rows} were sent, Postgres holds ${back.rows}`;
}
if (back.checksum !== copied.checksum)
return `${copied.spec.label}: the rows Postgres holds differ from the rows sent`;
return null;
}
/** Everything the copy transaction does, in order; throws on the first thing that does not hold. */
async function copyAll(
trx: Trx,
source: SqliteSource,
plugins: ReadonlyArray<{ source: SqliteSource; plan: PluginPlan }>,
options: MoveOptions,
summary: MoveSummary,
): Promise<void> {
await run(trx, "set local bytea_output = 'hex'");
const sourceTables: SourceTables = new Map(
source.tables().map((table) => [table, source.columns(table).map((column) => column.name)]),
);
const plan = planTrekTables(await readTargetSchema(trx), sourceTables);
if (plan.problems.length > 0) {
throw new MoveRefusal(
`The SQLite file and this TREK's Postgres schema differ:\n${listed(plan.problems, plan.problems.length)}`,
);
}
// The bootstrap's own rows (default settings, addons, providers) give way to the file's.
await run(trx, `truncate table ${plan.specs.map((spec) => spec.target).join(', ')}`);
const copied: CopiedTable[] = [];
let refused = 0;
const copyOne = (from: SqliteSource) => async (spec: CopiedTable['spec']) => {
options.log(`Copying ${spec.label}`);
const result = await copyTable(trx, from, spec, { insert: refused === 0 });
refused += result.problemCount;
copied.push(result);
};
await inOrder(plan.specs, copyOne(source));
await inOrder(copied, (table) => applyDeferred(trx, table));
const sequences: CopiedSequence[] = [...plan.sequences];
const pluginTables = new Map<string, CopiedTable[]>();
await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => {
if (refused > 0) return;
await createPluginSchema(trx, pluginPlan);
const before = copied.length;
await inOrder(pluginPlan.specs, copyOne(pluginSource));
pluginTables.set(pluginPlan.pluginId, copied.slice(before));
});
if (refused > 0) {
const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS);
throw new MoveRefusal(
`${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` +
`${listed(problems, refused)}\nCorrect those rows (or ask for help with them), then run the move again.`,
);
}
options.log('Moving the id sequences past the copied rows');
summary.sequences = await advanceSequences(trx, sequences, source.autoincrementSeen());
await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => {
summary.sequences += await advanceSequences(
trx,
await resolveIdentitySequences(trx, pluginPlan),
pluginSource.autoincrementSeen(),
);
});
options.log('Verifying every table');
const mismatches: string[] = [];
const sourceOf = (table: CopiedTable) =>
plugins.find(({ plan: pluginPlan }) => pluginPlan.specs.includes(table.spec))?.source ?? source;
await inOrder(copied, async (table) => {
const mismatch = await verify(trx, table, sourceOf(table).count(table.spec.source));
if (mismatch) mismatches.push(mismatch);
});
if (mismatches.length > 0) {
throw new Error(
`The copy does not match the SQLite file; nothing was written:\n${listed(mismatches, mismatches.length)}`,
);
}
const moved = (table: CopiedTable): MovedTable => ({
label: table.spec.source,
rows: table.rows,
checksum: table.checksum,
});
const pluginCopies = new Set([...pluginTables.values()].flat());
summary.tables = copied.filter((table) => !pluginCopies.has(table)).map(moved);
summary.plugins = plugins.map(({ plan: pluginPlan }) => ({
pluginId: pluginPlan.pluginId,
schema: pluginPlan.schema,
tables: (pluginTables.get(pluginPlan.pluginId) ?? []).map(moved),
notCarried: pluginPlan.notCarried,
}));
const unsafe = copied.filter((table) => table.unsafeIntegers > 0);
for (const table of unsafe) {
summary.warnings.push(
`${table.spec.label} holds ${table.unsafeIntegers} integer(s) beyond 2^53, which TREK refuses to read on Postgres.`,
);
}
}
/**
* Drops what the bootstrap created after a move failed (the database was
* empty before it, so everything in its schema is the move's). When even that
* fails, the error says so, since the database then needs dropping by hand.
*/
async function emptyAgain(db: Kysely<object>, cause: unknown): Promise<void> {
try {
await run(db, CLEAR_SCHEMA_SQL);
} catch (err) {
throw new Error(
`The move failed (${cause instanceof Error ? cause.message : String(cause)}), and the schema it had created ` +
`could not be dropped (${err instanceof Error ? err.message : String(err)}). Drop the Postgres database and ` +
'create it again before the next attempt.',
{ cause },
);
}
}
/** Runs the move. Throws a `MoveRefusal` for what the operator can fix, and leaves the target as it found it. */
export async function moveSqliteToPostgres(options: MoveOptions): Promise<MoveSummary> {
const started = Date.now();
const summary: MoveSummary = {
sqliteFile: options.sqliteFile,
tables: [],
plugins: [],
sequences: 0,
warnings: [],
seconds: 0,
};
const db = options.orm.em.getConnection().getClient<object>();
let source: SqliteSource;
try {
source = SqliteSource.open(options.sqliteFile);
} catch (err) {
throw new MoveRefusal(
`Cannot read the SQLite file ${options.sqliteFile}: ${err instanceof Error ? err.message : String(err)}. ` +
'Stop TREK first, and point --from at the file if it is not data/travel.db.',
);
}
const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = [];
try {
const level = migrationLevelProblem(source.migrationNames(), options.known);
if (level) throw new MoveRefusal(`Cannot move ${options.sqliteFile}: ${level}`);
const orphans = source.orphans(MAX_LISTED_PROBLEMS);
if (orphans.length > 0) {
throw new MoveRefusal(
`${options.sqliteFile} holds rows whose reference points at nothing, which Postgres refuses:\n` +
listed(
orphans.map((row) => `${row.table} row ${String(row.rowid)} refers to a missing row of ${row.parent}`),
orphans.length,
),
);
}
plugins.push(...planPlugins(options.pluginsDataDir, summary.warnings));
await refuseNonEmptyTarget(db);
try {
options.log('Creating the schema as a first boot does');
await bootstrapPostgresSchema(options.orm, {
baseline: options.baseline,
known: new Set(options.known),
safetyNet: NO_SAFETY_NET,
// This is the move itself.
sqliteLeftBehind: () => null,
});
await db.transaction().execute((trx) => copyAll(trx, source, plugins, options, summary));
} catch (err) {
// The copy rolled back; what the bootstrap created goes too, so the database is empty again.
await emptyAgain(db, err);
throw err;
}
} finally {
source.close();
for (const plugin of plugins) plugin.source.close();
}
summary.seconds = Math.round((Date.now() - started) / 100) / 10;
return summary;
}
/** The summary the command prints once the move committed. */
export function renderSummary(summary: MoveSummary, target: string): string {
const rows = summary.tables.reduce((sum, table) => sum + table.rows, 0);
const lines = [
`Moved ${summary.sqliteFile} into ${target} in ${summary.seconds} s.`,
` ${summary.tables.length} tables, ${rows} rows, each table verified by row count and checksum.`,
` ${summary.sequences} id sequence(s) moved past the copied rows.`,
];
for (const table of summary.tables.filter((entry) => entry.rows > 0)) {
lines.push(` ${table.label}: ${table.rows}`);
}
for (const plugin of summary.plugins) {
const pluginRows = plugin.tables.reduce((sum, table) => sum + table.rows, 0);
lines.push(
` Plugin ${plugin.pluginId}: ${plugin.tables.length} table(s), ${pluginRows} rows, into schema ${plugin.schema}.`,
);
for (const item of plugin.notCarried) lines.push(` not carried over: ${item}`);
}
for (const warning of summary.warnings) lines.push(` Warning: ${warning}`);
lines.push(
'',
'The SQLite file and the uploads were not changed. Next: set TREK_DB_URL for TREK, start it, sign in and check',
`your trips. To go back, unset TREK_DB_URL: TREK starts on ${summary.sqliteFile} as before.`,
);
return lines.join('\n');
}
@@ -0,0 +1,56 @@
/**
* The order the move fills the tables in, from the target's foreign keys: a
* table after every table it points at, so each row it inserts finds the row
* it references. Ties go by name, so two runs over one schema copy in the
* same order.
*
* A foreign key that cannot be satisfied by the order alone (a table pointing
* at itself, like a reply at its message, or a cycle between tables) is
* deferred: its columns are inserted as NULL and set once every row of the
* tables is in, still inside the move's one transaction. Postgres checks each
* foreign key when the row is written, so this is what lets a parent row
* arrive after its child.
*/
/** One foreign key of the target: `columns` of `table` point at `references`. */
export interface ForeignKey {
table: string;
columns: readonly string[];
references: string;
}
interface CopyPlan {
/** Every table, each after the tables it points at. */
order: string[];
/** Per table, the foreign key columns written as NULL first and set after all rows are in. */
deferred: Map<string, string[]>;
}
function defer(plan: CopyPlan, fk: ForeignKey): void {
const columns = plan.deferred.get(fk.table) ?? [];
for (const column of fk.columns) if (!columns.includes(column)) columns.push(column);
plan.deferred.set(fk.table, columns);
}
/** Plans the order for `tables`. A foreign key to a table outside the list (one the move does not copy) is ignored. */
export function planCopyOrder(tables: readonly string[], foreignKeys: readonly ForeignKey[]): CopyPlan {
const plan: CopyPlan = { order: [], deferred: new Map() };
const known = new Set(tables);
const pending = new Set([...tables].sort());
const keys = foreignKeys.filter((fk) => known.has(fk.table) && known.has(fk.references));
for (const fk of keys) if (fk.table === fk.references) defer(plan, fk);
const between = keys.filter((fk) => fk.table !== fk.references);
while (pending.size > 0) {
const waitsOn = (table: string) => between.filter((fk) => fk.table === table && pending.has(fk.references));
let next = [...pending].find((table) => waitsOn(table).length === 0);
if (next === undefined) {
// A cycle: break it at the first table by name, deferring its keys into the rest of the cycle.
next = [...pending][0];
for (const fk of waitsOn(next)) defer(plan, fk);
}
plan.order.push(next);
pending.delete(next);
}
return plan;
}
@@ -0,0 +1,226 @@
import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source';
import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema';
import { portableDefault } from '../schema-adjust';
import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy';
import type { ColumnKind } from './values';
import fs from 'node:fs';
import path from 'node:path';
/**
* A plugin's own database (`db:own`) moved from its `plugin.db` file into the
* schema TREK gives it on Postgres (`plugin-schema.ts`), so a plugin finds its
* rows where the Postgres store (`plugin-data-store.ts`) reads them.
*
* A plugin's tables are its own, created by its own SQL, so the move derives
* each Postgres table from the SQLite one: the column's type from SQLite's
* type affinity (and from the values it holds where the affinity leaves it
* open), NOT NULL, the primary key, a default that has a Postgres spelling,
* and the indexes over plain columns. The plugin's migration record
* (`_plugin_migrations`) is copied as it is, so the plugin does not run its
* migrations again over the copied tables. What has no Postgres counterpart
* here (foreign keys between its tables, views, triggers, partial or
* expression indexes, other defaults) is not created and is listed in the
* summary. The file itself stays where it is, untouched.
*/
/** The plugin store's own record of the migrations it ran, created exactly as the store creates it. */
const LEDGER = '_plugin_migrations';
const LEDGER_DDL = (schema: string) => `create table ${schema}.${LEDGER} (id text primary key, applied_at bigint)`;
/** The Postgres type the move gives each kind. */
const PG_TYPE: Record<ColumnKind, string> = {
integer: 'bigint',
float: 'double precision',
text: 'text',
json: 'jsonb',
bytes: 'bytea',
};
/**
* The kind of a plugin's column, by SQLite's affinity rules (the declared
* type's text decides, in this order: INT, then CHAR/CLOB/TEXT, then BLOB or
* none, then REAL/FLOA/DOUB, else NUMERIC). Where the affinity leaves the
* storage open (none, or NUMERIC), the values the column holds decide, and
* text is the answer that holds every one of them.
*/
export function pluginColumnKind(declaredType: string, storage: ReadonlySet<string>): ColumnKind {
const declared = declaredType.toUpperCase();
const only = (...classes: string[]) => storage.size > 0 && [...storage].every((value) => classes.includes(value));
if (declared.includes('INT')) return 'integer';
if (/CHAR|CLOB|TEXT/.test(declared)) return 'text';
if (declared.includes('BLOB')) return only('blob', 'text') ? 'bytes' : 'text';
if (/REAL|FLOA|DOUB/.test(declared)) return only('integer', 'real') || storage.size === 0 ? 'float' : 'text';
if (only('integer')) return 'integer';
if (only('integer', 'real')) return 'float';
if (only('blob')) return 'bytes';
return 'text';
}
const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const STRING_LITERAL = /^'(?:[^']|'')*'$/s;
/** The Postgres spelling of a column default, or null when it has none the move can trust. */
export function pluginDefault(sql: string | null, kind: ColumnKind): string | null {
if (sql === null) return null;
const text = sql
.trim()
.replace(/^\((.*)\)$/s, '$1')
.trim();
const ported = portableDefault(text);
if (ported !== text) return ported;
if (/^null$/i.test(text)) return 'null';
if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text;
if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null;
if (STRING_LITERAL.test(text)) return text;
return null;
}
/** One plugin's database as the move plans to copy it. */
export interface PluginPlan {
pluginId: string;
file: string;
schema: string;
/** The DDL that creates its tables and indexes, in order. */
ddl: string[];
specs: TableSpec[];
sequences: CopiedSequence[];
/** What the move does not carry over, for the summary. */
notCarried: string[];
}
function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, notes: string[], table: string): string {
const parts = [quoteIdentifier(column.name), PG_TYPE[kind]];
if (identity) parts.push('generated by default as identity');
if (column.notNull && !identity) parts.push('not null');
if (column.defaultSql !== null && !identity) {
const value = pluginDefault(column.defaultSql, kind);
if (value === null) notes.push(`the default ${column.defaultSql} of ${table}.${column.name}`);
else parts.push(`default ${value}`);
}
return parts.join(' ');
}
function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null {
if (index.origin === 'pk') return null;
if (index.partial || index.columns === null) {
notes.push(`the index ${index.name} on ${table} (partial or over an expression)`);
return null;
}
const columns = index.columns.map(quoteIdentifier).join(', ');
return `create ${index.unique ? 'unique ' : ''}index on ${target} (${columns})`;
}
/** Plans the copy of one plugin's `plugin.db`: what to create in its schema and what to fill. */
export function planPluginData(pluginId: string, source: SqliteSource): PluginPlan {
const schemaName = pluginSchemaName(pluginId);
const schema = quotePluginSchema(schemaName);
const plan: PluginPlan = {
pluginId,
file: source.file,
schema: schemaName,
ddl: [`create schema ${schema}`],
specs: [],
sequences: [],
notCarried: [],
};
for (const item of source.viewsAndTriggers()) plan.notCarried.push(`the ${item.type} ${item.name}`);
const tables = source.tables();
if (!tables.includes(LEDGER)) plan.ddl.push(LEDGER_DDL(schema));
for (const table of tables) {
const target = `${schema}.${quoteIdentifier(table)}`;
const label = `plugin ${pluginId}: ${table}`;
const columns = source.columns(table);
if (table === LEDGER) {
plan.ddl.push(LEDGER_DDL(schema));
plan.specs.push({
source: table,
target,
label,
columns: [
{ name: 'id', kind: 'text' },
{ name: 'applied_at', kind: 'integer' },
],
deferred: [],
primaryKey: ['id'],
});
continue;
}
const keys = columns.filter((column) => column.primaryKey > 0).sort((a, b) => a.primaryKey - b.primaryKey);
const kinds = columns.map((column) =>
pluginColumnKind(column.declaredType, source.storageClasses(table, column.name)),
);
// SQLite's rowid alias: the one INTEGER PRIMARY KEY column, which hands out its own ids.
const rowid = keys.length === 1 && keys[0].declaredType.toUpperCase() === 'INTEGER' ? keys[0].name : null;
const definitions = columns.map((column, index) =>
columnDdl(column, kinds[index], column.name === rowid, plan.notCarried, table),
);
if (keys.length > 0) definitions.push(`primary key (${keys.map((key) => quoteIdentifier(key.name)).join(', ')})`);
plan.ddl.push(`create table ${target} (${definitions.join(', ')})`);
for (const index of source.indexes(table)) {
const ddl = indexDdl(index, target, plan.notCarried, table);
if (ddl) plan.ddl.push(ddl);
}
plan.specs.push({
source: table,
target,
label,
columns: columns.map((column, index) => ({ name: column.name, kind: kinds[index] })),
deferred: [],
primaryKey: keys.map((key) => key.name),
});
// The identity's sequence is named by Postgres; resolveIdentitySequences reads it once the table exists.
if (rowid) plan.sequences.push({ table, target, column: rowid, sequence: '' });
}
return plan;
}
/** A plugin's data file under the plugins' data directory, by plugin id. */
interface PluginDataFile {
pluginId: string;
file: string;
}
/**
* The `plugin.db` files under `root`, one per plugin directory. A directory
* whose name is no plugin id cannot have a schema, and is reported instead.
*/
export function findPluginDataFiles(root: string): { files: PluginDataFile[]; skipped: string[] } {
if (!fs.existsSync(root)) return { files: [], skipped: [] };
const files: PluginDataFile[] = [];
const skipped: string[] = [];
for (const entry of fs.readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
const file = path.join(root, entry.name, 'plugin.db');
if (!entry.isDirectory() || !fs.existsSync(file)) continue;
if (isPluginId(entry.name)) files.push({ pluginId: entry.name, file });
else skipped.push(file);
}
return { files, skipped };
}
/** Creates a plugin's schema, tables and indexes in the move's transaction. */
export async function createPluginSchema(trx: Trx, plan: PluginPlan): Promise<void> {
await inOrder(plan.ddl, async (statement) => {
try {
await run(trx, statement);
} catch (err) {
throw new Error(
`Postgres refused the table of plugin ${plan.pluginId} (${statement}): ${err instanceof Error ? err.message : String(err)}`,
{ cause: err },
);
}
});
}
/** The plan's identity columns with the sequence Postgres gave each, read from the catalog. */
export async function resolveIdentitySequences(trx: Trx, plan: PluginPlan): Promise<CopiedSequence[]> {
const resolved: CopiedSequence[] = [];
await inOrder(plan.sequences, async (entry) => {
const [row] = await run<{ sequence: string | null }>(trx, 'select pg_get_serial_sequence($1, $2) as sequence', [
entry.target,
entry.column,
]);
if (row?.sequence) resolved.push({ ...entry, sequence: row.sequence });
});
return resolved;
}
@@ -0,0 +1,234 @@
import { createHash } from 'node:crypto';
/**
* How a value read from a SQLite file becomes the value a Postgres column of
* the move's target takes, and the one text form both sides are compared in.
*
* SQLite stores a value by what it is, not by the column's declared type, so
* a column may hold text where the schema says integer. The move never
* guesses: a value the Postgres column cannot hold exactly is refused with
* the reason, and the move stops before it commits anything.
*
* The SQLite side is read with better-sqlite3's safe integers, so an integer
* arrives as a `bigint` and keeps all 64 bits.
*/
/** The Postgres column types the move writes, by what they hold. */
export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes';
/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */
export type SqliteValue = null | bigint | number | string | Uint8Array;
/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */
export type PostgresValue = null | string | number | Buffer;
type Converted = { ok: true; value: PostgresValue } | { ok: false; reason: string };
const INT64_MIN = -(2n ** 63n);
const INT64_MAX = 2n ** 63n - 1n;
const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER);
const DECIMAL_INTEGER = /^[+-]?\d+$/;
const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i;
const NUL = '\u0000';
/**
* The kind of a Postgres column, from `information_schema.columns.data_type`.
* Null for a type the move has no rule for: the caller refuses the move then,
* so a new column type fails loudly here instead of being copied by guess.
*/
export function columnKind(dataType: string): ColumnKind | null {
switch (dataType) {
case 'bigint':
case 'integer':
case 'smallint':
return 'integer';
case 'double precision':
case 'real':
return 'float';
case 'text':
case 'character varying':
case 'character':
return 'text';
case 'json':
case 'jsonb':
return 'json';
case 'bytea':
return 'bytes';
default:
return null;
}
}
function refuse(reason: string): Converted {
return { ok: false, reason };
}
function describe(value: SqliteValue): string {
if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`;
if (typeof value === 'string')
return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`;
return `the value ${String(value)}`;
}
function toInteger(value: Exclude<SqliteValue, null>): Converted {
let whole: bigint;
if (typeof value === 'bigint') whole = value;
else if (typeof value === 'number') {
if (!Number.isInteger(value)) return refuse(`${describe(value)} is not a whole number`);
whole = BigInt(value);
} else if (typeof value === 'string' && DECIMAL_INTEGER.test(value.trim())) {
whole = BigInt(value.trim());
} else {
return refuse(`${describe(value)} is not a whole number`);
}
if (whole < INT64_MIN || whole > INT64_MAX) return refuse(`${describe(value)} does not fit in 64 bits`);
return { ok: true, value: whole.toString() };
}
function toFloat(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'number') return { ok: true, value };
if (typeof value === 'bigint') {
const number = Number(value);
if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`);
return { ok: true, value: number };
}
if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) {
const number = Number(value.trim());
if (Number.isFinite(number)) return { ok: true, value: number };
}
return refuse(`${describe(value)} is not a number`);
}
function toText(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'string') {
return value.includes(NUL)
? refuse('the text holds a NUL character, which Postgres text cannot store')
: { ok: true, value };
}
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
return refuse(`${describe(value)} is not text`);
}
function toJson(value: Exclude<SqliteValue, null>): Converted {
if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) };
if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`);
try {
JSON.parse(value);
} catch {
return refuse(`${describe(value)} is not valid JSON`);
}
// jsonb refuses the escaped NUL character as well as the raw one.
if (value.includes(NUL) || value.includes('\\u0000'))
return refuse('the JSON holds a NUL character, which jsonb cannot store');
return { ok: true, value };
}
function toBytes(value: Exclude<SqliteValue, null>): Converted {
if (value instanceof Uint8Array) return { ok: true, value: Buffer.from(value) };
// A text value in a BLOB column: its UTF-8 bytes, what SQLite's CAST(x AS BLOB) answers.
if (typeof value === 'string') return { ok: true, value: Buffer.from(value, 'utf8') };
return refuse(`${describe(value)} is not a BLOB`);
}
/** `value` as the Postgres column of `kind` takes it, or the reason it cannot hold it exactly. */
export function convertValue(value: SqliteValue, kind: ColumnKind): Converted {
if (value === null) return { ok: true, value: null };
switch (kind) {
case 'integer':
return toInteger(value);
case 'float':
return toFloat(value);
case 'text':
return toText(value);
case 'json':
return toJson(value);
case 'bytes':
return toBytes(value);
}
}
/** Whether an integer value read back would be beyond what TREK's Postgres driver reads (2^53). */
export function beyondSafeInteger(value: PostgresValue): boolean {
if (typeof value !== 'string' || !DECIMAL_INTEGER.test(value)) return false;
const whole = BigInt(value);
return whole > SAFE_MAX || whole < -SAFE_MAX;
}
/** JSON text with its object keys sorted, so two spellings of one value compare equal (jsonb reorders keys). */
export function stableJson(text: string): string {
const sort = (value: unknown): unknown => {
if (Array.isArray(value)) return value.map(sort);
if (value !== null && typeof value === 'object') {
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, sort((value as Record<string, unknown>)[key])]),
);
}
return value;
};
return JSON.stringify(sort(JSON.parse(text)));
}
/**
* The text a value is compared in, from the value the move sent to Postgres.
* `canonicalFromPostgres` must answer the same for what Postgres stored.
*/
export function canonicalFromSent(value: PostgresValue, kind: ColumnKind): string | null {
if (value === null) return null;
switch (kind) {
case 'integer':
return BigInt(String(value)).toString();
case 'float':
return String(Number(value));
case 'text':
return String(value);
case 'json':
return stableJson(String(value));
case 'bytes':
return Buffer.isBuffer(value) ? value.toString('hex') : Buffer.from(String(value), 'utf8').toString('hex');
}
}
/**
* The text a value is compared in, from the column read back as `::text`
* (with `bytea_output = hex`, so bytes read as `\x` and their hex digits).
*/
export function canonicalFromPostgres(text: string | null, kind: ColumnKind): string | null {
if (text === null) return null;
switch (kind) {
case 'integer':
return BigInt(text).toString();
case 'float':
return String(Number(text));
case 'text':
return text;
case 'json':
return stableJson(text);
case 'bytes':
return text.startsWith('\\x') ? text.slice(2).toLowerCase() : text;
}
}
const CHECKSUM_MODULUS = 2n ** 256n;
/**
* An order-free checksum over a table's rows: the sum of each row's SHA-256
* modulo 2^256. Postgres and SQLite need not hand the rows back in the same
* order (their collations differ), and a row that is missing, added, doubled
* or changed changes the sum.
*/
export class TableChecksum {
private sum = 0n;
rows = 0;
add(canonicalRow: ReadonlyArray<string | null>): void {
const digest = createHash('sha256').update(JSON.stringify(canonicalRow)).digest('hex');
this.sum = (this.sum + BigInt(`0x${digest}`)) % CHECKSUM_MODULUS;
this.rows++;
}
hex(): string {
return this.sum.toString(16).padStart(64, '0');
}
}