diff --git a/knip.jsonc b/knip.jsonc index a8804f5fd..ef94760ca 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -48,6 +48,8 @@ "entry": [ "src/index.ts", "src/bootstrap.ts", + // The move of a SQLite install into Postgres, run by hand (npm run db:move-to-postgres, dist/ in the image). + "src/move-to-postgres.ts", // Read by the MikroORM CLI and at boot, by path. "src/mikro-orm.config.ts", // Started with child_process.fork() by the plugin supervisor (nest/plugins/paths.ts). diff --git a/server/package.json b/server/package.json index 5fbf7a76c..6e5a8244c 100644 --- a/server/package.json +++ b/server/package.json @@ -11,6 +11,7 @@ "build": "node scripts/build.mjs", "start:prod": "node --require tsconfig-paths/register dist/index.js", "reset-admin": "node reset-admin.js", + "db:move-to-postgres": "node --import tsx src/move-to-postgres.ts", "typecheck": "tsc --noEmit", "typecheck:tests": "tsc -p tsconfig.tests.json --noEmit", "typecheck:scripts": "tsc -p tsconfig.scripts.json --noEmit", diff --git a/server/src/db/engine/postgres/sqlite-move/catalog.ts b/server/src/db/engine/postgres/sqlite-move/catalog.ts new file mode 100644 index 000000000..7881cea0a --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/catalog.ts @@ -0,0 +1,163 @@ +import { quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy'; +import { planCopyOrder, type ForeignKey } from './order'; +import { columnKind } from './values'; + +/** + * TREK's tables as the target database holds them after the engine's + * bootstrap, and how each one is filled from the table of the same name in + * the SQLite file. The target's catalog decides the column types; the SQLite + * file only has to hold the same tables and columns, which a file at this + * build's migration level does. + */ + +/** The migration history: each side keeps its own, and the bootstrap wrote the target's. */ +const HISTORY_TABLE = 'mikro_orm_migrations'; + +interface TargetColumn { + name: string; + dataType: string; + nullable: boolean; +} + +interface TargetTable { + name: string; + columns: TargetColumn[]; + primaryKey: string[]; +} + +export interface TargetSchema { + tables: TargetTable[]; + foreignKeys: ForeignKey[]; + /** Every column a sequence fills, by table. */ + sequences: Array<{ table: string; column: string; sequence: string }>; +} + +/** Reads the tables, keys and sequences of the current schema. */ +export async function readTargetSchema(trx: Trx): Promise { + const columns = await run<{ table_name: string; column_name: string; data_type: string; is_nullable: string }>( + trx, + 'select c.table_name, c.column_name, c.data_type, c.is_nullable from information_schema.columns c ' + + 'join information_schema.tables t on t.table_schema = c.table_schema and t.table_name = c.table_name ' + + "where c.table_schema = current_schema() and t.table_type = 'BASE TABLE' order by c.table_name, c.ordinal_position", + ); + const constraints = await run<{ kind: string; table_name: string; target_name: string | null; columns: string[] }>( + trx, + 'select c.contype as kind, src.relname as table_name, dst.relname as target_name, ' + + 'array_agg(a.attname::text order by k.ord) as columns from pg_constraint c ' + + 'join pg_class src on src.oid = c.conrelid left join pg_class dst on dst.oid = c.confrelid ' + + 'cross join lateral unnest(c.conkey) with ordinality as k(attnum, ord) ' + + 'join pg_attribute a on a.attrelid = c.conrelid and a.attnum = k.attnum ' + + "where c.connamespace = current_schema()::regnamespace and c.contype in ('p', 'f') " + + 'group by c.oid, c.contype, src.relname, dst.relname', + ); + const sequences = await run<{ table_name: string; column_name: string; sequence: string }>( + trx, + "select table_name, column_name, pg_get_serial_sequence(format('%I', table_name), column_name) as sequence " + + 'from information_schema.columns where table_schema = current_schema() ' + + "and pg_get_serial_sequence(format('%I', table_name), column_name) is not null order by table_name, column_name", + ); + + const tables = new Map(); + for (const column of columns) { + const table = tables.get(column.table_name) ?? { name: column.table_name, columns: [], primaryKey: [] }; + table.columns.push({ + name: column.column_name, + dataType: column.data_type, + nullable: column.is_nullable === 'YES', + }); + tables.set(column.table_name, table); + } + const foreignKeys: ForeignKey[] = []; + for (const constraint of constraints) { + if (constraint.kind === 'p') { + const table = tables.get(constraint.table_name); + if (table) table.primaryKey = constraint.columns; + } else if (constraint.target_name) { + foreignKeys.push({ + table: constraint.table_name, + columns: constraint.columns, + references: constraint.target_name, + }); + } + } + return { + tables: [...tables.values()], + foreignKeys, + sequences: sequences.map((row) => ({ table: row.table_name, column: row.column_name, sequence: row.sequence })), + }; +} + +/** The tables and columns of the SQLite file, as the move compares them with the target. */ +export type SourceTables = ReadonlyMap; + +interface TrekCopyPlan { + /** The tables in the order they are filled. */ + specs: TableSpec[]; + sequences: CopiedSequence[]; + /** Why the two schemas do not match; the move refuses when there is any. */ + problems: string[]; +} + +/** + * Pairs every table of the target with the SQLite table of the same name and + * puts them in foreign key order. Anything the two do not share, or a column + * type the move has no rule for, is a problem: copying around it would lose + * data or guess. + */ +export function planTrekTables(target: TargetSchema, source: SourceTables): TrekCopyPlan { + const problems: string[] = []; + const tables = target.tables.filter((table) => table.name !== HISTORY_TABLE); + const names = new Set(tables.map((table) => table.name)); + for (const name of source.keys()) { + if (name !== HISTORY_TABLE && !names.has(name)) problems.push(`the SQLite table ${name} has no Postgres table`); + } + const plan = planCopyOrder( + tables.map((table) => table.name), + target.foreignKeys, + ); + const byName = new Map(tables.map((table) => [table.name, table])); + const specs: TableSpec[] = []; + for (const name of plan.order) { + const table = byName.get(name); + const sourceColumns = source.get(name); + if (!table) continue; + if (!sourceColumns) { + problems.push(`the Postgres table ${name} has no SQLite table`); + continue; + } + for (const column of sourceColumns) { + if (!table.columns.some((candidate) => candidate.name === column)) { + problems.push(`the SQLite column ${name}.${column} has no Postgres column`); + } + } + const columns = table.columns.flatMap((column) => { + const kind = columnKind(column.dataType); + if (!sourceColumns.includes(column.name)) + problems.push(`the Postgres column ${name}.${column.name} has no SQLite column`); + else if (kind === null) + problems.push( + `the Postgres column ${name}.${column.name} has the type ${column.dataType}, which the move does not know`, + ); + else return [{ name: column.name, kind }]; + return []; + }); + const deferred = plan.deferred.get(name) ?? []; + for (const column of deferred) { + if (!table.columns.find((candidate) => candidate.name === column)?.nullable) { + problems.push(`${name}.${column} refers back to a table filled later and cannot be NULL meanwhile`); + } + } + specs.push({ + source: name, + target: quoteIdentifier(name), + label: name, + columns, + deferred, + primaryKey: table.primaryKey, + }); + } + const sequences = target.sequences + .filter((entry) => names.has(entry.table)) + .map((entry) => ({ ...entry, target: quoteIdentifier(entry.table) })); + return { specs, sequences, problems }; +} diff --git a/server/src/db/engine/postgres/sqlite-move/command.ts b/server/src/db/engine/postgres/sqlite-move/command.ts new file mode 100644 index 000000000..1d461dada --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/command.ts @@ -0,0 +1,104 @@ +import { readEnv, type AppEnv } from '../../../../app-config'; +import { resolveDataPaths } from '../../../../app-config/data-paths'; +import { knownMigrationNames } from '../../../known-migrations'; +import type { AnyOrm } from '../../../orm'; +import { buildOrmConfig, resolveEngine } from '../../index'; +import { redactUrl } from '../../redact-url'; +import { readPostgresBaseline } from '../baseline'; +import { MoveRefusal, moveSqliteToPostgres, renderSummary, type MoveOptions } from './move'; +import { MikroORM } from '@mikro-orm/core'; + +import path from 'node:path'; + +/** + * The command around the move (`src/move-to-postgres.ts` runs it): reads + * TREK_DB_URL as the server does, takes the SQLite file from the data + * directory or `--from=`, the plugin data from the plugins' data + * directory, runs the move and prints its summary. Answers the exit code. + */ + +export const USAGE = + 'Usage: move-to-postgres [--from=]\n' + + 'Copies a TREK SQLite database (data/travel.db by default) into the empty Postgres database TREK_DB_URL names.'; + +export interface MoveCommandIo { + env: () => AppEnv; + out: (line: string) => void; + err: (line: string) => void; + /** An ORM on the target database, from the Postgres engine's own config. */ + openOrm: (env: AppEnv) => Promise; + move: (options: MoveOptions) => ReturnType; +} + +/** The ORM the move runs on: the engine's config, without entity discovery (the move runs SQL only). */ +async function openTargetOrm(env: AppEnv): Promise { + const config = buildOrmConfig(resolveEngine(env.db)); + return MikroORM.init({ + ...config, + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + migrations: { ...config.migrations, silent: true }, + }); +} + +const defaultIo: MoveCommandIo = { + env: readEnv, + out: (line) => process.stdout.write(`${line}\n`), + err: (line) => process.stderr.write(`${line}\n`), + openOrm: openTargetOrm, + move: moveSqliteToPostgres, +}; + +const FROM = '--from='; + +async function run(argv: readonly string[], io: MoveCommandIo): Promise { + if (argv.includes('--help') || argv.includes('-h')) { + io.out(USAGE); + return 0; + } + const unknown = argv.find((arg) => !arg.startsWith(FROM)); + if (unknown !== undefined) throw new MoveRefusal(`Unknown argument ${unknown}.\n${USAGE}`); + const env = io.env(); + if (env.db.engine !== 'postgres') + throw new MoveRefusal('Set TREK_DB_URL to the empty Postgres database to move into.'); + + const paths = resolveDataPaths(env); + const from = argv.find((arg) => arg.startsWith(FROM))?.slice(FROM.length); + const sqliteFile = path.resolve(from || paths.dbFile); + const target = redactUrl(env.db.url); + io.out(`Moving ${sqliteFile} into ${target}`); + const orm = await io.openOrm(env); + try { + const summary = await io.move({ + sqliteFile, + pluginsDataDir: env.plugins.dataDir || path.join(paths.dataDir, 'plugins-data'), + orm, + known: knownMigrationNames(), + baseline: readPostgresBaseline, + log: (line) => io.out(` ${line}`), + }); + io.out(''); + io.out(renderSummary(summary, target)); + return 0; + } finally { + await orm.close(true); + } +} + +/** Runs the command and answers its exit code: 0 once the move committed, 1 for a refusal or a failure. */ +export async function runMoveCommand(argv: readonly string[], io: Partial = {}): Promise { + const resolved = { ...defaultIo, ...io }; + try { + return await run(argv, resolved); + } catch (err) { + if (err instanceof MoveRefusal) resolved.err(`\nThe move was refused: ${err.message}`); + else { + const text = err instanceof Error ? (err.stack ?? err.message) : String(err); + resolved.err( + `\nThe move failed; the Postgres database is left as the move found it unless this says otherwise: ${text}`, + ); + } + return 1; + } +} diff --git a/server/src/db/engine/postgres/sqlite-move/copy.ts b/server/src/db/engine/postgres/sqlite-move/copy.ts new file mode 100644 index 000000000..0497766a5 --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/copy.ts @@ -0,0 +1,301 @@ +import type { SqliteSource } from '../../sqlite/sqlite-source'; +import { + beyondSafeInteger, + canonicalFromPostgres, + canonicalFromSent, + convertValue, + TableChecksum, + type ColumnKind, + type PostgresValue, + type SqliteValue, +} from './values'; + +import { CompiledQuery, type Kysely, type Transaction } from 'kysely'; + +/** + * Copying one table from the SQLite file into Postgres and proving the copy, + * inside the move's transaction. The same steps serve TREK's own tables and a + * plugin's: read the rows one at a time, convert each value to its column's + * type, insert in batches, then read the table back and compare its row count + * and checksum with what was sent. + */ + +export type Trx = Transaction; + +/** Rows per INSERT at most; fewer for a wide table, so a batch stays far below Postgres's 65535 parameters. */ +const MAX_BATCH_ROWS = 500; +const MAX_BATCH_PARAMS = 30_000; +/** Rows a verification cursor fetches at once. */ +const FETCH_ROWS = 2_000; +/** The most refused values a refusal lists one by one. */ +export const MAX_LISTED_PROBLEMS = 20; + +/** One column of a table the move copies. */ +interface CopyColumn { + name: string; + kind: ColumnKind; +} + +/** What the move copies from one SQLite table into one Postgres table. */ +export interface TableSpec { + /** The table in the SQLite file. */ + source: string; + /** The Postgres table, quoted and qualified as SQL needs it. */ + target: string; + /** A name for messages: `users`, or `plugin my-plugin: notes`. */ + label: string; + columns: CopyColumn[]; + /** Foreign key columns inserted as NULL first and set once every table is in (`order.ts`). */ + deferred: string[]; + /** The primary key's columns, which the deferred update finds a row by. */ + primaryKey: string[]; +} + +/** A row's deferred values, kept until every table is in. */ +interface DeferredRow { + key: PostgresValue[]; + values: PostgresValue[]; +} + +/** What copying one table found. */ +export interface CopiedTable { + spec: TableSpec; + rows: number; + checksum: string; + /** Values the Postgres column cannot hold exactly, listed up to `MAX_LISTED_PROBLEMS`; the move refuses on any. */ + problems: string[]; + problemCount: number; + /** Integer values beyond 2^53, which TREK's Postgres driver refuses to read back. */ + unsafeIntegers: number; + deferredRows: DeferredRow[]; +} + +/** `name` as a Postgres identifier. */ +export function quoteIdentifier(name: string): string { + return `"${name.replaceAll('"', '""')}"`; +} + +/** Runs one statement, in the move's transaction or on the pool, and returns its rows. */ +export async function run>( + trx: Kysely, + sql: string, + params: unknown[] = [], +): Promise { + const result = await trx.executeQuery(CompiledQuery.raw(sql, params)); + return result.rows; +} + +/** + * Runs `work` for each item, one after the other, each started once the one + * before it has finished: the move's statements share one transaction, which + * runs one statement at a time. + */ +export async function inOrder(items: readonly T[], work: (item: T) => Promise): Promise { + await items.reduce>((previous, item) => previous.then(() => work(item)), Promise.resolve()); +} + +/** A Postgres error with its detail and the constraint it broke, which `pg` keeps apart from the message. */ +export function describePgError(err: unknown): string { + if (!(err instanceof Error)) return String(err); + const extra = err as Error & { detail?: string; constraint?: string }; + const parts = [err.message]; + if (extra.constraint) parts.push(`constraint ${extra.constraint}`); + if (extra.detail) parts.push(extra.detail); + return parts.join('; '); +} + +/** How many rows one INSERT carries for a table of `columns` columns. */ +export function batchRows(columns: number): number { + return Math.max(1, Math.min(MAX_BATCH_ROWS, Math.floor(MAX_BATCH_PARAMS / Math.max(1, columns)))); +} + +/** The INSERT of `rows` rows into the table, with numbered parameters row by row. */ +export function insertSql(spec: Pick, rows: number): string { + const names = spec.columns.map((column) => quoteIdentifier(column.name)).join(', '); + const width = spec.columns.length; + const tuples = Array.from( + { length: rows }, + (_, row) => `(${spec.columns.map((_column, index) => `$${row * width + index + 1}`).join(', ')})`, + ); + return `insert into ${spec.target} (${names}) values ${tuples.join(', ')}`; +} + +async function insertBatch(trx: Trx, spec: TableSpec, batch: PostgresValue[][]): Promise { + if (batch.length === 0) return; + try { + await run(trx, insertSql(spec, batch.length), batch.flat()); + } catch (err) { + throw new Error(`Postgres refused rows of ${spec.label}: ${describePgError(err)}`, { cause: err }); + } +} + +/** + * Copies the rows of one table. Every value is converted first; once one + * cannot be, nothing more is inserted, but the rest of the table is still read + * so the refusal can list every problem at once. With `insert: false` it only + * reads and converts, once an earlier table was refused. + */ +export async function copyTable( + trx: Trx, + source: SqliteSource, + spec: TableSpec, + options: { insert: boolean } = { insert: true }, +): Promise { + const result: CopiedTable = { + spec, + rows: 0, + checksum: '', + problems: [], + problemCount: 0, + unsafeIntegers: 0, + deferredRows: [], + }; + const checksum = new TableChecksum(); + const deferredAt = spec.deferred.map((name) => spec.columns.findIndex((column) => column.name === name)); + const keyAt = spec.primaryKey.map((name) => spec.columns.findIndex((column) => column.name === name)); + const size = batchRows(spec.columns.length); + + /** How a refusal names a row: by its primary key, or by its place in the table without one. */ + const rowName = (raw: unknown[]): string => { + if (keyAt.length === 0 || keyAt.includes(-1)) return `row ${result.rows}`; + return `row ${spec.primaryKey.map((name, index) => `${name}=${String(raw[keyAt[index]])}`).join(', ')}`; + }; + + /** One SQLite row as the values sent to Postgres, or null once a value was refused. */ + const convertRow = (raw: unknown[]): PostgresValue[] | null => { + result.rows++; + const values: PostgresValue[] = []; + spec.columns.forEach((column, index) => { + const converted = convertValue(raw[index] as SqliteValue, column.kind); + if (converted.ok) { + values.push(converted.value); + if (column.kind === 'integer' && beyondSafeInteger(converted.value)) result.unsafeIntegers++; + return; + } + values.push(null); + result.problemCount++; + if (result.problems.length < MAX_LISTED_PROBLEMS) { + result.problems.push(`${spec.label} ${rowName(raw)}, column ${column.name}: ${converted.reason}`); + } + }); + if (result.problemCount > 0) return null; + checksum.add(values.map((value, index) => canonicalFromSent(value, spec.columns[index].kind))); + if (deferredAt.length > 0) { + const later = deferredAt.map((index) => values[index]); + if (later.some((value) => value !== null)) { + result.deferredRows.push({ key: keyAt.map((index) => values[index]), values: later }); + } + for (const index of deferredAt) values[index] = null; + } + return values; + }; + + const rows = source.rows( + spec.source, + spec.columns.map((column) => column.name), + ); + /** Reads up to one batch of rows; `done` once the table has no more. */ + const nextBatch = (): { batch: PostgresValue[][]; done: boolean } => { + const batch: PostgresValue[][] = []; + while (batch.length < size) { + const next = rows.next(); + if (next.done) return { batch, done: true }; + const values = convertRow(next.value); + if (values) batch.push(values); + } + return { batch, done: false }; + }; + const pump = async (): Promise => { + const { batch, done } = nextBatch(); + // Always awaited, so the next batch starts on a fresh stack even when nothing is inserted. + await (options.insert && result.problemCount === 0 ? insertBatch(trx, spec, batch) : Promise.resolve()); + if (!done) await pump(); + }; + try { + await pump(); + } finally { + // An iterator left open keeps the SQLite handle busy. + rows.return?.(); + } + result.checksum = checksum.hex(); + return result; +} + +/** Sets the foreign key columns `copyTable` held back, once every table is in. */ +export async function applyDeferred(trx: Trx, copied: CopiedTable): Promise { + const { spec } = copied; + if (copied.deferredRows.length === 0) return; + if (spec.primaryKey.length === 0) throw new Error(`${spec.label} has no primary key to set its deferred columns by.`); + const width = spec.deferred.length; + const sets = spec.deferred.map((name, index) => `${quoteIdentifier(name)} = $${index + 1}`).join(', '); + const where = spec.primaryKey.map((name, index) => `${quoteIdentifier(name)} = $${width + index + 1}`).join(' and '); + const sql = `update ${spec.target} set ${sets} where ${where}`; + // One row at a time: only the rows of a table that points at itself, a handful in practice. + await inOrder(copied.deferredRows, async (row) => { + try { + await run(trx, sql, [...row.values, ...row.key]); + } catch (err) { + throw new Error(`Postgres refused a reference in ${spec.label}: ${describePgError(err)}`, { cause: err }); + } + }); +} + +/** + * Reads a table back through a cursor, every column as text, and answers its + * row count and checksum, which must equal what `copyTable` sent. + */ +export async function readBack(trx: Trx, spec: TableSpec): Promise<{ rows: number; checksum: string }> { + const checksum = new TableChecksum(); + const list = spec.columns.map((column, index) => `${quoteIdentifier(column.name)}::text as c${index}`).join(', '); + const fetchAll = async (): Promise => { + const rows = await run>(trx, `fetch forward ${FETCH_ROWS} from trek_move_verify`); + for (const row of rows) { + checksum.add(spec.columns.map((column, index) => canonicalFromPostgres(row[`c${index}`] ?? null, column.kind))); + } + if (rows.length === FETCH_ROWS) await fetchAll(); + }; + await run(trx, `declare trek_move_verify no scroll cursor for select ${list} from ${spec.target}`); + try { + await fetchAll(); + } finally { + await run(trx, 'close trek_move_verify'); + } + return { rows: checksum.rows, checksum: checksum.hex() }; +} + +/** A sequence behind a copied column: `table` names it in the SQLite file, `target` in Postgres. */ +export interface CopiedSequence { + table: string; + target: string; + column: string; + sequence: string; +} + +/** + * Moves every sequence of the copied tables past the ids they now hold, and + * past the highest id an AUTOINCREMENT table ever handed out in the SQLite + * file, so a new row never takes an id SQLite would not have reused. + */ +export async function advanceSequences( + trx: Trx, + sequences: readonly CopiedSequence[], + autoincrementSeen: ReadonlyMap, +): Promise { + await inOrder(sequences, async (entry) => { + const [row] = await run<{ max: string | null }>( + trx, + `select max(${quoteIdentifier(entry.column)})::text as max from ${entry.target}`, + ); + const seen = autoincrementSeen.get(entry.table) ?? null; + const copied = row?.max ? BigInt(row.max) : null; + const highest = seen !== null && (copied === null || seen > copied) ? seen : copied; + // Nothing copied: the next id is 1, as on a new database. + const called = highest !== null && highest >= 1n; + await run(trx, 'select setval($1::regclass, $2::bigint, $3::boolean)', [ + entry.sequence, + called ? highest.toString() : '1', + called, + ]); + }); + return sequences.length; +} diff --git a/server/src/db/engine/postgres/sqlite-move/move.ts b/server/src/db/engine/postgres/sqlite-move/move.ts new file mode 100644 index 000000000..de6543674 --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/move.ts @@ -0,0 +1,358 @@ +import { NO_SAFETY_NET } from '../../../legacy-baseline'; +import type { AnyOrm } from '../../../orm'; +import { SqliteSource } from '../../sqlite/sqlite-source'; +import type { PostgresBaseline } from '../baseline'; +import { CLEAR_SCHEMA_SQL } from '../clear-schema'; +import { PLUGIN_SCHEMA_PREFIX } from '../plugin-schema'; +import { bootstrapPostgresSchema } from '../schema-bootstrap'; +import { planTrekTables, readTargetSchema, type SourceTables } from './catalog'; +import { + advanceSequences, + applyDeferred, + copyTable, + inOrder, + MAX_LISTED_PROBLEMS, + readBack, + run, + type CopiedSequence, + type CopiedTable, + type Trx, +} from './copy'; +import { + createPluginSchema, + findPluginDataFiles, + planPluginData, + resolveIdentitySequences, + type PluginPlan, +} from './plugin-data'; + +import type { Kysely } from 'kysely'; + +/** + * Moves a SQLite install into an empty Postgres database (`npm run + * db:move-to-postgres`, `dist/move-to-postgres.js` in the image). + * + * 1. The SQLite file must be at this build's migration level: an older one is + * migrated by starting this TREK on it once, a newer one belongs to the + * TREK that wrote it. Nothing else is checked by guess. + * 2. The Postgres database must hold no table and no plugin schema. + * 3. The schema is created exactly as a first boot creates it (the engine's + * bootstrap: the baseline and the migrations after its cut), without the + * seeders, since every row comes from the file. + * 4. In one transaction: the rows the bootstrap seeded are cleared, every + * table is filled in foreign key order, each plugin's `plugin.db` goes into + * its schema, the sequences are moved past the copied ids, and every table + * is read back and compared with what was sent (row count and a checksum + * over all rows). Only a copy that matches is committed. + * 5. Anything that fails leaves the database as it was found: the + * transaction rolls back and the schema the bootstrap created is dropped. + * + * The SQLite files are opened read-only and never written. Uploads are not + * the database's: they stay where the storage driver keeps them. + */ + +/** A refusal the operator can act on: printed without a stack. */ +export class MoveRefusal extends Error {} + +export interface MoveOptions { + /** The SQLite file to move: `data/travel.db` by default. */ + sqliteFile: string; + /** Where the plugins keep their data directories (`plugin.db` files). */ + pluginsDataDir: string; + /** An ORM on the target database, from the Postgres engine's own config. */ + orm: AnyOrm; + /** The migrations this build ships. */ + known: ReadonlySet; + baseline: () => PostgresBaseline; + /** Where progress lines go. */ + log: (line: string) => void; +} + +interface MovedTable { + label: string; + rows: number; + checksum: string; +} + +export interface MoveSummary { + sqliteFile: string; + tables: MovedTable[]; + plugins: Array<{ pluginId: string; schema: string; tables: MovedTable[]; notCarried: string[] }>; + sequences: number; + warnings: string[]; + seconds: number; +} + +/** + * Why the SQLite file is not at this build's migration level, or null when it + * is: it must have recorded exactly the migrations this build ships. + */ +export function migrationLevelProblem(recorded: readonly string[] | null, known: ReadonlySet): string | null { + const startOnce = + 'Start this TREK version on it once, without TREK_DB_URL, so it brings the file up to date; stop it; then run the move again.'; + if (recorded === null || recorded.length === 0) { + return `the SQLite file has no TREK migration history (an install from before 5.0, or not a TREK database). ${startOnce}`; + } + const unknown = recorded.filter((name) => !known.has(name)); + if (unknown.length > 0) { + return ( + `a newer TREK migrated the SQLite file (${unknown.length} migration(s) this version does not know, ` + + `the last ${[...unknown].sort().at(-1)}). Run the move with the TREK version that last ran on it.` + ); + } + const have = new Set(recorded); + const missing = [...known].filter((name) => !have.has(name)); + if (missing.length > 0) { + return `the SQLite file is ${missing.length} migration(s) behind this TREK version. ${startOnce}`; + } + return null; +} + +function listed(problems: readonly string[], total: number): string { + const lines = problems.map((problem) => ` - ${problem}`); + if (total > problems.length) lines.push(` - and ${total - problems.length} more`); + return lines.join('\n'); +} + +/** Refuses a database that holds a table or a plugin schema: the move fills an empty database only. */ +async function refuseNonEmptyTarget(db: Kysely): Promise { + const tables = await run<{ table_name: string }>( + db, + 'select table_name from information_schema.tables where table_schema = current_schema() order by table_name limit 5', + ); + const schemas = await run<{ nspname: string }>( + db, + 'select nspname from pg_namespace where starts_with(nspname, $1) order by nspname limit 5', + [PLUGIN_SCHEMA_PREFIX], + ); + const found = [...tables.map((row) => row.table_name), ...schemas.map((row) => `schema ${row.nspname}`)]; + if (found.length === 0) return; + throw new MoveRefusal( + `The Postgres database is not empty (${found.join(', ')}). The move fills an empty database only, so it never ` + + 'mixes two installs. Point TREK_DB_URL at a new, empty database. If an earlier move stopped part of the way, ' + + 'drop that database and create it again.', + ); +} + +/** Opens every plugin's data file and plans its copy; the sources stay open for the copy. */ +function planPlugins(root: string, warnings: string[]): Array<{ source: SqliteSource; plan: PluginPlan }> { + const { files, skipped } = findPluginDataFiles(root); + for (const file of skipped) warnings.push(`${file} is in a directory whose name is no plugin id; it was not moved.`); + const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = []; + for (const { pluginId, file } of files) { + const source = SqliteSource.open(file); + plugins.push({ source, plan: planPluginData(pluginId, source) }); + } + return plugins; +} + +/** Compares a copied table with what Postgres holds now: the SQLite count, the rows sent and the checksum. */ +async function verify(trx: Trx, copied: CopiedTable, sqliteCount: number): Promise { + const back = await readBack(trx, copied.spec); + if (back.rows !== sqliteCount || copied.rows !== sqliteCount) { + return `${copied.spec.label}: the SQLite file holds ${sqliteCount} rows, ${copied.rows} were sent, Postgres holds ${back.rows}`; + } + if (back.checksum !== copied.checksum) + return `${copied.spec.label}: the rows Postgres holds differ from the rows sent`; + return null; +} + +/** Everything the copy transaction does, in order; throws on the first thing that does not hold. */ +async function copyAll( + trx: Trx, + source: SqliteSource, + plugins: ReadonlyArray<{ source: SqliteSource; plan: PluginPlan }>, + options: MoveOptions, + summary: MoveSummary, +): Promise { + await run(trx, "set local bytea_output = 'hex'"); + const sourceTables: SourceTables = new Map( + source.tables().map((table) => [table, source.columns(table).map((column) => column.name)]), + ); + const plan = planTrekTables(await readTargetSchema(trx), sourceTables); + if (plan.problems.length > 0) { + throw new MoveRefusal( + `The SQLite file and this TREK's Postgres schema differ:\n${listed(plan.problems, plan.problems.length)}`, + ); + } + // The bootstrap's own rows (default settings, addons, providers) give way to the file's. + await run(trx, `truncate table ${plan.specs.map((spec) => spec.target).join(', ')}`); + + const copied: CopiedTable[] = []; + let refused = 0; + const copyOne = (from: SqliteSource) => async (spec: CopiedTable['spec']) => { + options.log(`Copying ${spec.label}`); + const result = await copyTable(trx, from, spec, { insert: refused === 0 }); + refused += result.problemCount; + copied.push(result); + }; + await inOrder(plan.specs, copyOne(source)); + await inOrder(copied, (table) => applyDeferred(trx, table)); + + const sequences: CopiedSequence[] = [...plan.sequences]; + const pluginTables = new Map(); + await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => { + if (refused > 0) return; + await createPluginSchema(trx, pluginPlan); + const before = copied.length; + await inOrder(pluginPlan.specs, copyOne(pluginSource)); + pluginTables.set(pluginPlan.pluginId, copied.slice(before)); + }); + if (refused > 0) { + const problems = copied.flatMap((table) => table.problems).slice(0, MAX_LISTED_PROBLEMS); + throw new MoveRefusal( + `${refused} value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n` + + `${listed(problems, refused)}\nCorrect those rows (or ask for help with them), then run the move again.`, + ); + } + + options.log('Moving the id sequences past the copied rows'); + summary.sequences = await advanceSequences(trx, sequences, source.autoincrementSeen()); + await inOrder(plugins, async ({ source: pluginSource, plan: pluginPlan }) => { + summary.sequences += await advanceSequences( + trx, + await resolveIdentitySequences(trx, pluginPlan), + pluginSource.autoincrementSeen(), + ); + }); + + options.log('Verifying every table'); + const mismatches: string[] = []; + const sourceOf = (table: CopiedTable) => + plugins.find(({ plan: pluginPlan }) => pluginPlan.specs.includes(table.spec))?.source ?? source; + await inOrder(copied, async (table) => { + const mismatch = await verify(trx, table, sourceOf(table).count(table.spec.source)); + if (mismatch) mismatches.push(mismatch); + }); + if (mismatches.length > 0) { + throw new Error( + `The copy does not match the SQLite file; nothing was written:\n${listed(mismatches, mismatches.length)}`, + ); + } + + const moved = (table: CopiedTable): MovedTable => ({ + label: table.spec.source, + rows: table.rows, + checksum: table.checksum, + }); + const pluginCopies = new Set([...pluginTables.values()].flat()); + summary.tables = copied.filter((table) => !pluginCopies.has(table)).map(moved); + summary.plugins = plugins.map(({ plan: pluginPlan }) => ({ + pluginId: pluginPlan.pluginId, + schema: pluginPlan.schema, + tables: (pluginTables.get(pluginPlan.pluginId) ?? []).map(moved), + notCarried: pluginPlan.notCarried, + })); + const unsafe = copied.filter((table) => table.unsafeIntegers > 0); + for (const table of unsafe) { + summary.warnings.push( + `${table.spec.label} holds ${table.unsafeIntegers} integer(s) beyond 2^53, which TREK refuses to read on Postgres.`, + ); + } +} + +/** + * Drops what the bootstrap created after a move failed (the database was + * empty before it, so everything in its schema is the move's). When even that + * fails, the error says so, since the database then needs dropping by hand. + */ +async function emptyAgain(db: Kysely, cause: unknown): Promise { + try { + await run(db, CLEAR_SCHEMA_SQL); + } catch (err) { + throw new Error( + `The move failed (${cause instanceof Error ? cause.message : String(cause)}), and the schema it had created ` + + `could not be dropped (${err instanceof Error ? err.message : String(err)}). Drop the Postgres database and ` + + 'create it again before the next attempt.', + { cause }, + ); + } +} + +/** Runs the move. Throws a `MoveRefusal` for what the operator can fix, and leaves the target as it found it. */ +export async function moveSqliteToPostgres(options: MoveOptions): Promise { + const started = Date.now(); + const summary: MoveSummary = { + sqliteFile: options.sqliteFile, + tables: [], + plugins: [], + sequences: 0, + warnings: [], + seconds: 0, + }; + const db = options.orm.em.getConnection().getClient(); + let source: SqliteSource; + try { + source = SqliteSource.open(options.sqliteFile); + } catch (err) { + throw new MoveRefusal( + `Cannot read the SQLite file ${options.sqliteFile}: ${err instanceof Error ? err.message : String(err)}. ` + + 'Stop TREK first, and point --from at the file if it is not data/travel.db.', + ); + } + const plugins: Array<{ source: SqliteSource; plan: PluginPlan }> = []; + try { + const level = migrationLevelProblem(source.migrationNames(), options.known); + if (level) throw new MoveRefusal(`Cannot move ${options.sqliteFile}: ${level}`); + const orphans = source.orphans(MAX_LISTED_PROBLEMS); + if (orphans.length > 0) { + throw new MoveRefusal( + `${options.sqliteFile} holds rows whose reference points at nothing, which Postgres refuses:\n` + + listed( + orphans.map((row) => `${row.table} row ${String(row.rowid)} refers to a missing row of ${row.parent}`), + orphans.length, + ), + ); + } + plugins.push(...planPlugins(options.pluginsDataDir, summary.warnings)); + await refuseNonEmptyTarget(db); + + try { + options.log('Creating the schema as a first boot does'); + await bootstrapPostgresSchema(options.orm, { + baseline: options.baseline, + known: new Set(options.known), + safetyNet: NO_SAFETY_NET, + // This is the move itself. + sqliteLeftBehind: () => null, + }); + await db.transaction().execute((trx) => copyAll(trx, source, plugins, options, summary)); + } catch (err) { + // The copy rolled back; what the bootstrap created goes too, so the database is empty again. + await emptyAgain(db, err); + throw err; + } + } finally { + source.close(); + for (const plugin of plugins) plugin.source.close(); + } + summary.seconds = Math.round((Date.now() - started) / 100) / 10; + return summary; +} + +/** The summary the command prints once the move committed. */ +export function renderSummary(summary: MoveSummary, target: string): string { + const rows = summary.tables.reduce((sum, table) => sum + table.rows, 0); + const lines = [ + `Moved ${summary.sqliteFile} into ${target} in ${summary.seconds} s.`, + ` ${summary.tables.length} tables, ${rows} rows, each table verified by row count and checksum.`, + ` ${summary.sequences} id sequence(s) moved past the copied rows.`, + ]; + for (const table of summary.tables.filter((entry) => entry.rows > 0)) { + lines.push(` ${table.label}: ${table.rows}`); + } + for (const plugin of summary.plugins) { + const pluginRows = plugin.tables.reduce((sum, table) => sum + table.rows, 0); + lines.push( + ` Plugin ${plugin.pluginId}: ${plugin.tables.length} table(s), ${pluginRows} rows, into schema ${plugin.schema}.`, + ); + for (const item of plugin.notCarried) lines.push(` not carried over: ${item}`); + } + for (const warning of summary.warnings) lines.push(` Warning: ${warning}`); + lines.push( + '', + 'The SQLite file and the uploads were not changed. Next: set TREK_DB_URL for TREK, start it, sign in and check', + `your trips. To go back, unset TREK_DB_URL: TREK starts on ${summary.sqliteFile} as before.`, + ); + return lines.join('\n'); +} diff --git a/server/src/db/engine/postgres/sqlite-move/order.ts b/server/src/db/engine/postgres/sqlite-move/order.ts new file mode 100644 index 000000000..e12d7b561 --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/order.ts @@ -0,0 +1,56 @@ +/** + * The order the move fills the tables in, from the target's foreign keys: a + * table after every table it points at, so each row it inserts finds the row + * it references. Ties go by name, so two runs over one schema copy in the + * same order. + * + * A foreign key that cannot be satisfied by the order alone (a table pointing + * at itself, like a reply at its message, or a cycle between tables) is + * deferred: its columns are inserted as NULL and set once every row of the + * tables is in, still inside the move's one transaction. Postgres checks each + * foreign key when the row is written, so this is what lets a parent row + * arrive after its child. + */ + +/** One foreign key of the target: `columns` of `table` point at `references`. */ +export interface ForeignKey { + table: string; + columns: readonly string[]; + references: string; +} + +interface CopyPlan { + /** Every table, each after the tables it points at. */ + order: string[]; + /** Per table, the foreign key columns written as NULL first and set after all rows are in. */ + deferred: Map; +} + +function defer(plan: CopyPlan, fk: ForeignKey): void { + const columns = plan.deferred.get(fk.table) ?? []; + for (const column of fk.columns) if (!columns.includes(column)) columns.push(column); + plan.deferred.set(fk.table, columns); +} + +/** Plans the order for `tables`. A foreign key to a table outside the list (one the move does not copy) is ignored. */ +export function planCopyOrder(tables: readonly string[], foreignKeys: readonly ForeignKey[]): CopyPlan { + const plan: CopyPlan = { order: [], deferred: new Map() }; + const known = new Set(tables); + const pending = new Set([...tables].sort()); + const keys = foreignKeys.filter((fk) => known.has(fk.table) && known.has(fk.references)); + for (const fk of keys) if (fk.table === fk.references) defer(plan, fk); + const between = keys.filter((fk) => fk.table !== fk.references); + + while (pending.size > 0) { + const waitsOn = (table: string) => between.filter((fk) => fk.table === table && pending.has(fk.references)); + let next = [...pending].find((table) => waitsOn(table).length === 0); + if (next === undefined) { + // A cycle: break it at the first table by name, deferring its keys into the rest of the cycle. + next = [...pending][0]; + for (const fk of waitsOn(next)) defer(plan, fk); + } + plan.order.push(next); + pending.delete(next); + } + return plan; +} diff --git a/server/src/db/engine/postgres/sqlite-move/plugin-data.ts b/server/src/db/engine/postgres/sqlite-move/plugin-data.ts new file mode 100644 index 000000000..9108f78cf --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/plugin-data.ts @@ -0,0 +1,226 @@ +import { SqliteSource, type SqliteColumn, type SqliteIndex } from '../../sqlite/sqlite-source'; +import { isPluginId, pluginSchemaName, quotePluginSchema } from '../plugin-schema'; +import { portableDefault } from '../schema-adjust'; +import { inOrder, quoteIdentifier, run, type CopiedSequence, type TableSpec, type Trx } from './copy'; +import type { ColumnKind } from './values'; + +import fs from 'node:fs'; +import path from 'node:path'; + +/** + * A plugin's own database (`db:own`) moved from its `plugin.db` file into the + * schema TREK gives it on Postgres (`plugin-schema.ts`), so a plugin finds its + * rows where the Postgres store (`plugin-data-store.ts`) reads them. + * + * A plugin's tables are its own, created by its own SQL, so the move derives + * each Postgres table from the SQLite one: the column's type from SQLite's + * type affinity (and from the values it holds where the affinity leaves it + * open), NOT NULL, the primary key, a default that has a Postgres spelling, + * and the indexes over plain columns. The plugin's migration record + * (`_plugin_migrations`) is copied as it is, so the plugin does not run its + * migrations again over the copied tables. What has no Postgres counterpart + * here (foreign keys between its tables, views, triggers, partial or + * expression indexes, other defaults) is not created and is listed in the + * summary. The file itself stays where it is, untouched. + */ + +/** The plugin store's own record of the migrations it ran, created exactly as the store creates it. */ +const LEDGER = '_plugin_migrations'; +const LEDGER_DDL = (schema: string) => `create table ${schema}.${LEDGER} (id text primary key, applied_at bigint)`; + +/** The Postgres type the move gives each kind. */ +const PG_TYPE: Record = { + integer: 'bigint', + float: 'double precision', + text: 'text', + json: 'jsonb', + bytes: 'bytea', +}; + +/** + * The kind of a plugin's column, by SQLite's affinity rules (the declared + * type's text decides, in this order: INT, then CHAR/CLOB/TEXT, then BLOB or + * none, then REAL/FLOA/DOUB, else NUMERIC). Where the affinity leaves the + * storage open (none, or NUMERIC), the values the column holds decide, and + * text is the answer that holds every one of them. + */ +export function pluginColumnKind(declaredType: string, storage: ReadonlySet): ColumnKind { + const declared = declaredType.toUpperCase(); + const only = (...classes: string[]) => storage.size > 0 && [...storage].every((value) => classes.includes(value)); + if (declared.includes('INT')) return 'integer'; + if (/CHAR|CLOB|TEXT/.test(declared)) return 'text'; + if (declared.includes('BLOB')) return only('blob', 'text') ? 'bytes' : 'text'; + if (/REAL|FLOA|DOUB/.test(declared)) return only('integer', 'real') || storage.size === 0 ? 'float' : 'text'; + if (only('integer')) return 'integer'; + if (only('integer', 'real')) return 'float'; + if (only('blob')) return 'bytes'; + return 'text'; +} + +const NUMBER_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i; +const STRING_LITERAL = /^'(?:[^']|'')*'$/s; + +/** The Postgres spelling of a column default, or null when it has none the move can trust. */ +export function pluginDefault(sql: string | null, kind: ColumnKind): string | null { + if (sql === null) return null; + const text = sql + .trim() + .replace(/^\((.*)\)$/s, '$1') + .trim(); + const ported = portableDefault(text); + if (ported !== text) return ported; + if (/^null$/i.test(text)) return 'null'; + if (NUMBER_LITERAL.test(text)) return kind === 'text' ? `'${text}'` : text; + if (/^(true|false)$/i.test(text)) return kind === 'integer' ? (/^true$/i.test(text) ? '1' : '0') : null; + if (STRING_LITERAL.test(text)) return text; + return null; +} + +/** One plugin's database as the move plans to copy it. */ +export interface PluginPlan { + pluginId: string; + file: string; + schema: string; + /** The DDL that creates its tables and indexes, in order. */ + ddl: string[]; + specs: TableSpec[]; + sequences: CopiedSequence[]; + /** What the move does not carry over, for the summary. */ + notCarried: string[]; +} + +function columnDdl(column: SqliteColumn, kind: ColumnKind, identity: boolean, notes: string[], table: string): string { + const parts = [quoteIdentifier(column.name), PG_TYPE[kind]]; + if (identity) parts.push('generated by default as identity'); + if (column.notNull && !identity) parts.push('not null'); + if (column.defaultSql !== null && !identity) { + const value = pluginDefault(column.defaultSql, kind); + if (value === null) notes.push(`the default ${column.defaultSql} of ${table}.${column.name}`); + else parts.push(`default ${value}`); + } + return parts.join(' '); +} + +function indexDdl(index: SqliteIndex, target: string, notes: string[], table: string): string | null { + if (index.origin === 'pk') return null; + if (index.partial || index.columns === null) { + notes.push(`the index ${index.name} on ${table} (partial or over an expression)`); + return null; + } + const columns = index.columns.map(quoteIdentifier).join(', '); + return `create ${index.unique ? 'unique ' : ''}index on ${target} (${columns})`; +} + +/** Plans the copy of one plugin's `plugin.db`: what to create in its schema and what to fill. */ +export function planPluginData(pluginId: string, source: SqliteSource): PluginPlan { + const schemaName = pluginSchemaName(pluginId); + const schema = quotePluginSchema(schemaName); + const plan: PluginPlan = { + pluginId, + file: source.file, + schema: schemaName, + ddl: [`create schema ${schema}`], + specs: [], + sequences: [], + notCarried: [], + }; + for (const item of source.viewsAndTriggers()) plan.notCarried.push(`the ${item.type} ${item.name}`); + const tables = source.tables(); + if (!tables.includes(LEDGER)) plan.ddl.push(LEDGER_DDL(schema)); + for (const table of tables) { + const target = `${schema}.${quoteIdentifier(table)}`; + const label = `plugin ${pluginId}: ${table}`; + const columns = source.columns(table); + if (table === LEDGER) { + plan.ddl.push(LEDGER_DDL(schema)); + plan.specs.push({ + source: table, + target, + label, + columns: [ + { name: 'id', kind: 'text' }, + { name: 'applied_at', kind: 'integer' }, + ], + deferred: [], + primaryKey: ['id'], + }); + continue; + } + const keys = columns.filter((column) => column.primaryKey > 0).sort((a, b) => a.primaryKey - b.primaryKey); + const kinds = columns.map((column) => + pluginColumnKind(column.declaredType, source.storageClasses(table, column.name)), + ); + // SQLite's rowid alias: the one INTEGER PRIMARY KEY column, which hands out its own ids. + const rowid = keys.length === 1 && keys[0].declaredType.toUpperCase() === 'INTEGER' ? keys[0].name : null; + const definitions = columns.map((column, index) => + columnDdl(column, kinds[index], column.name === rowid, plan.notCarried, table), + ); + if (keys.length > 0) definitions.push(`primary key (${keys.map((key) => quoteIdentifier(key.name)).join(', ')})`); + plan.ddl.push(`create table ${target} (${definitions.join(', ')})`); + for (const index of source.indexes(table)) { + const ddl = indexDdl(index, target, plan.notCarried, table); + if (ddl) plan.ddl.push(ddl); + } + plan.specs.push({ + source: table, + target, + label, + columns: columns.map((column, index) => ({ name: column.name, kind: kinds[index] })), + deferred: [], + primaryKey: keys.map((key) => key.name), + }); + // The identity's sequence is named by Postgres; resolveIdentitySequences reads it once the table exists. + if (rowid) plan.sequences.push({ table, target, column: rowid, sequence: '' }); + } + return plan; +} + +/** A plugin's data file under the plugins' data directory, by plugin id. */ +interface PluginDataFile { + pluginId: string; + file: string; +} + +/** + * The `plugin.db` files under `root`, one per plugin directory. A directory + * whose name is no plugin id cannot have a schema, and is reported instead. + */ +export function findPluginDataFiles(root: string): { files: PluginDataFile[]; skipped: string[] } { + if (!fs.existsSync(root)) return { files: [], skipped: [] }; + const files: PluginDataFile[] = []; + const skipped: string[] = []; + for (const entry of fs.readdirSync(root, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const file = path.join(root, entry.name, 'plugin.db'); + if (!entry.isDirectory() || !fs.existsSync(file)) continue; + if (isPluginId(entry.name)) files.push({ pluginId: entry.name, file }); + else skipped.push(file); + } + return { files, skipped }; +} + +/** Creates a plugin's schema, tables and indexes in the move's transaction. */ +export async function createPluginSchema(trx: Trx, plan: PluginPlan): Promise { + await inOrder(plan.ddl, async (statement) => { + try { + await run(trx, statement); + } catch (err) { + throw new Error( + `Postgres refused the table of plugin ${plan.pluginId} (${statement}): ${err instanceof Error ? err.message : String(err)}`, + { cause: err }, + ); + } + }); +} + +/** The plan's identity columns with the sequence Postgres gave each, read from the catalog. */ +export async function resolveIdentitySequences(trx: Trx, plan: PluginPlan): Promise { + const resolved: CopiedSequence[] = []; + await inOrder(plan.sequences, async (entry) => { + const [row] = await run<{ sequence: string | null }>(trx, 'select pg_get_serial_sequence($1, $2) as sequence', [ + entry.target, + entry.column, + ]); + if (row?.sequence) resolved.push({ ...entry, sequence: row.sequence }); + }); + return resolved; +} diff --git a/server/src/db/engine/postgres/sqlite-move/values.ts b/server/src/db/engine/postgres/sqlite-move/values.ts new file mode 100644 index 000000000..779ed0fe0 --- /dev/null +++ b/server/src/db/engine/postgres/sqlite-move/values.ts @@ -0,0 +1,234 @@ +import { createHash } from 'node:crypto'; + +/** + * How a value read from a SQLite file becomes the value a Postgres column of + * the move's target takes, and the one text form both sides are compared in. + * + * SQLite stores a value by what it is, not by the column's declared type, so + * a column may hold text where the schema says integer. The move never + * guesses: a value the Postgres column cannot hold exactly is refused with + * the reason, and the move stops before it commits anything. + * + * The SQLite side is read with better-sqlite3's safe integers, so an integer + * arrives as a `bigint` and keeps all 64 bits. + */ + +/** The Postgres column types the move writes, by what they hold. */ +export type ColumnKind = 'integer' | 'float' | 'text' | 'json' | 'bytes'; + +/** What a SQLite cell can be: NULL, INTEGER, REAL, TEXT or BLOB. */ +export type SqliteValue = null | bigint | number | string | Uint8Array; + +/** A value Postgres takes as a parameter: a decimal string for an integer, a Buffer for bytes. */ +export type PostgresValue = null | string | number | Buffer; + +type Converted = { ok: true; value: PostgresValue } | { ok: false; reason: string }; + +const INT64_MIN = -(2n ** 63n); +const INT64_MAX = 2n ** 63n - 1n; +const SAFE_MAX = BigInt(Number.MAX_SAFE_INTEGER); +const DECIMAL_INTEGER = /^[+-]?\d+$/; +const DECIMAL_NUMBER = /^[+-]?(\d+\.?\d*|\.\d+)(e[+-]?\d+)?$/i; +const NUL = '\u0000'; + +/** + * The kind of a Postgres column, from `information_schema.columns.data_type`. + * Null for a type the move has no rule for: the caller refuses the move then, + * so a new column type fails loudly here instead of being copied by guess. + */ +export function columnKind(dataType: string): ColumnKind | null { + switch (dataType) { + case 'bigint': + case 'integer': + case 'smallint': + return 'integer'; + case 'double precision': + case 'real': + return 'float'; + case 'text': + case 'character varying': + case 'character': + return 'text'; + case 'json': + case 'jsonb': + return 'json'; + case 'bytea': + return 'bytes'; + default: + return null; + } +} + +function refuse(reason: string): Converted { + return { ok: false, reason }; +} + +function describe(value: SqliteValue): string { + if (value instanceof Uint8Array) return `a ${value.length}-byte BLOB`; + if (typeof value === 'string') + return `the text ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}`; + return `the value ${String(value)}`; +} + +function toInteger(value: Exclude): Converted { + let whole: bigint; + if (typeof value === 'bigint') whole = value; + else if (typeof value === 'number') { + if (!Number.isInteger(value)) return refuse(`${describe(value)} is not a whole number`); + whole = BigInt(value); + } else if (typeof value === 'string' && DECIMAL_INTEGER.test(value.trim())) { + whole = BigInt(value.trim()); + } else { + return refuse(`${describe(value)} is not a whole number`); + } + if (whole < INT64_MIN || whole > INT64_MAX) return refuse(`${describe(value)} does not fit in 64 bits`); + return { ok: true, value: whole.toString() }; +} + +function toFloat(value: Exclude): Converted { + if (typeof value === 'number') return { ok: true, value }; + if (typeof value === 'bigint') { + const number = Number(value); + if (BigInt(number) !== value) return refuse(`${describe(value)} has more digits than a double holds`); + return { ok: true, value: number }; + } + if (typeof value === 'string' && DECIMAL_NUMBER.test(value.trim())) { + const number = Number(value.trim()); + if (Number.isFinite(number)) return { ok: true, value: number }; + } + return refuse(`${describe(value)} is not a number`); +} + +function toText(value: Exclude): Converted { + if (typeof value === 'string') { + return value.includes(NUL) + ? refuse('the text holds a NUL character, which Postgres text cannot store') + : { ok: true, value }; + } + if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; + return refuse(`${describe(value)} is not text`); +} + +function toJson(value: Exclude): Converted { + if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; + if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`); + try { + JSON.parse(value); + } catch { + return refuse(`${describe(value)} is not valid JSON`); + } + // jsonb refuses the escaped NUL character as well as the raw one. + if (value.includes(NUL) || value.includes('\\u0000')) + return refuse('the JSON holds a NUL character, which jsonb cannot store'); + return { ok: true, value }; +} + +function toBytes(value: Exclude): Converted { + if (value instanceof Uint8Array) return { ok: true, value: Buffer.from(value) }; + // A text value in a BLOB column: its UTF-8 bytes, what SQLite's CAST(x AS BLOB) answers. + if (typeof value === 'string') return { ok: true, value: Buffer.from(value, 'utf8') }; + return refuse(`${describe(value)} is not a BLOB`); +} + +/** `value` as the Postgres column of `kind` takes it, or the reason it cannot hold it exactly. */ +export function convertValue(value: SqliteValue, kind: ColumnKind): Converted { + if (value === null) return { ok: true, value: null }; + switch (kind) { + case 'integer': + return toInteger(value); + case 'float': + return toFloat(value); + case 'text': + return toText(value); + case 'json': + return toJson(value); + case 'bytes': + return toBytes(value); + } +} + +/** Whether an integer value read back would be beyond what TREK's Postgres driver reads (2^53). */ +export function beyondSafeInteger(value: PostgresValue): boolean { + if (typeof value !== 'string' || !DECIMAL_INTEGER.test(value)) return false; + const whole = BigInt(value); + return whole > SAFE_MAX || whole < -SAFE_MAX; +} + +/** JSON text with its object keys sorted, so two spellings of one value compare equal (jsonb reorders keys). */ +export function stableJson(text: string): string { + const sort = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sort); + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, sort((value as Record)[key])]), + ); + } + return value; + }; + return JSON.stringify(sort(JSON.parse(text))); +} + +/** + * The text a value is compared in, from the value the move sent to Postgres. + * `canonicalFromPostgres` must answer the same for what Postgres stored. + */ +export function canonicalFromSent(value: PostgresValue, kind: ColumnKind): string | null { + if (value === null) return null; + switch (kind) { + case 'integer': + return BigInt(String(value)).toString(); + case 'float': + return String(Number(value)); + case 'text': + return String(value); + case 'json': + return stableJson(String(value)); + case 'bytes': + return Buffer.isBuffer(value) ? value.toString('hex') : Buffer.from(String(value), 'utf8').toString('hex'); + } +} + +/** + * The text a value is compared in, from the column read back as `::text` + * (with `bytea_output = hex`, so bytes read as `\x` and their hex digits). + */ +export function canonicalFromPostgres(text: string | null, kind: ColumnKind): string | null { + if (text === null) return null; + switch (kind) { + case 'integer': + return BigInt(text).toString(); + case 'float': + return String(Number(text)); + case 'text': + return text; + case 'json': + return stableJson(text); + case 'bytes': + return text.startsWith('\\x') ? text.slice(2).toLowerCase() : text; + } +} + +const CHECKSUM_MODULUS = 2n ** 256n; + +/** + * An order-free checksum over a table's rows: the sum of each row's SHA-256 + * modulo 2^256. Postgres and SQLite need not hand the rows back in the same + * order (their collations differ), and a row that is missing, added, doubled + * or changed changes the sum. + */ +export class TableChecksum { + private sum = 0n; + rows = 0; + + add(canonicalRow: ReadonlyArray): void { + const digest = createHash('sha256').update(JSON.stringify(canonicalRow)).digest('hex'); + this.sum = (this.sum + BigInt(`0x${digest}`)) % CHECKSUM_MODULUS; + this.rows++; + } + + hex(): string { + return this.sum.toString(16).padStart(64, '0'); + } +} diff --git a/server/src/move-to-postgres.ts b/server/src/move-to-postgres.ts new file mode 100644 index 000000000..3eefba3a5 --- /dev/null +++ b/server/src/move-to-postgres.ts @@ -0,0 +1,20 @@ +import 'reflect-metadata'; +import 'dotenv/config'; +// The same fail-fast validation as a server start: TREK_DB_URL is read as the server reads it. +import './app-config/boot-validate'; + +import { runMoveCommand } from './db/engine/postgres/sqlite-move/command'; + +/** + * Moves a SQLite install into an empty Postgres database, with TREK stopped: + * + * docker compose run --rm --user node -w /app/server app \ + * node --require tsconfig-paths/register dist/move-to-postgres.js + * npm run db:move-to-postgres (in server/, outside Docker) + * + * The target is TREK_DB_URL; the source is data/travel.db, or --from=. + * The SQLite files and the uploads are never changed. The wiki page + * Database-Postgres ("Moving an existing install to Postgres") has the steps, + * and src/db/engine/postgres/sqlite-move/ the move itself. + */ +void runMoveCommand(process.argv.slice(2)).then((code) => process.exit(code)); diff --git a/server/tests/helpers/fake-postgres-move.ts b/server/tests/helpers/fake-postgres-move.ts new file mode 100644 index 000000000..3c0d45a6c --- /dev/null +++ b/server/tests/helpers/fake-postgres-move.ts @@ -0,0 +1,261 @@ +import type { AnyOrm } from '../../src/db/orm'; + +import type { CompiledQuery, Kysely } from 'kysely'; +import { vi } from 'vitest'; + +/** + * A stand-in for the Postgres side of the SQLite move + * (`src/db/engine/postgres/sqlite-move/`), for the unit tests in the SQLite + * lane: it answers exactly the statements the move sends, keeps the rows it + * is given in memory and reads them back as Postgres's `::text` would. The + * real database is the Postgres lane's (`tests/postgres/sqlite-move.test.ts`). + */ + +interface FakeColumn { + name: string; + dataType: string; + nullable?: boolean; +} + +export interface FakeTableDef { + name: string; + columns: FakeColumn[]; + primaryKey: string[]; + /** Foreign keys: `columns` of this table point at `references`. */ + foreignKeys?: Array<{ columns: string[]; references: string }>; + /** Columns a sequence fills. */ + serial?: string[]; +} + +interface FakeTable { + columns: string[]; + rows: unknown[][]; +} + +const QUOTED = /"((?:[^"]|"")*)"/g; + +function unquote(text: string): string[] { + return [...text.matchAll(QUOTED)].map((match) => match[1].replaceAll('""', '"')); +} + +/** A value as Postgres would print it with `::text` and `bytea_output = hex`. */ +function asText(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (Buffer.isBuffer(value)) return `\\x${value.toString('hex')}`; + return String(value); +} + +export class FakePostgres { + /** Every statement, in order. */ + readonly statements: string[] = []; + readonly tables = new Map(); + readonly sequences = new Map(); + /** Tables the database holds before anything runs (the move refuses them). */ + existing: string[] = []; + pluginSchemas: string[] = []; + /** A statement matching this fails, as Postgres would refuse it. */ + failOn: RegExp | null = null; + /** Rows the read-back answers in place of the stored ones, by table: a copy that came out wrong. */ + tamper = new Map unknown[][]>(); + private cursor: Array> = []; + private bootstrapped = false; + + constructor(readonly schema: FakeTableDef[]) {} + + /** What the engine's bootstrap leaves: every table of the schema, with a seeded row or two. */ + bootstrap(): void { + this.bootstrapped = true; + for (const table of this.schema) { + this.tables.set(`"${table.name}"`, { columns: table.columns.map((column) => column.name), rows: [] }); + } + const first = this.schema[0]; + this.tables.get(`"${first.name}"`)?.rows.push(first.columns.map(() => null)); + } + + rowsOf(target: string): unknown[][] { + return this.tables.get(target)?.rows ?? []; + } + + async executeQuery(query: CompiledQuery): Promise<{ rows: R[] }> { + const sql = query.sql; + const params = [...query.parameters]; + this.statements.push(sql); + if (this.failOn?.test(sql)) { + throw Object.assign(new Error('refused by the fake'), { detail: 'Key (id)=(1)', constraint: 'fake_key' }); + } + return { rows: this.answer(sql, params) as R[] }; + } + + /** `db.transaction().execute(fn)`: the fake has no isolation, only the order of statements. */ + transaction() { + return { execute: (work: (trx: Kysely) => Promise) => work(this.asKysely()) }; + } + + asKysely(): Kysely { + return this as unknown as Kysely; + } + + private answer(sql: string, params: unknown[]): unknown[] { + if (sql.startsWith('select table_name from information_schema.tables')) { + return [...this.existing, ...(this.bootstrapped ? this.schema.map((table) => table.name) : [])].map( + (table_name) => ({ table_name }), + ); + } + if (sql.startsWith('select nspname from pg_namespace')) return this.pluginSchemas.map((nspname) => ({ nspname })); + if (sql.includes('from information_schema.columns c')) { + return this.schema.flatMap((table) => + table.columns.map((column) => ({ + table_name: table.name, + column_name: column.name, + data_type: column.dataType, + is_nullable: column.nullable === false ? 'NO' : 'YES', + })), + ); + } + if (sql.includes('from pg_constraint')) { + return this.schema.flatMap((table) => [ + { kind: 'p', table_name: table.name, target_name: null, columns: table.primaryKey }, + ...(table.foreignKeys ?? []).map((fk) => ({ + kind: 'f', + table_name: table.name, + target_name: fk.references, + columns: fk.columns, + })), + ]); + } + if (sql.includes("pg_get_serial_sequence(format('%I'")) { + return this.schema.flatMap((table) => + (table.serial ?? []).map((column) => ({ + table_name: table.name, + column_name: column, + sequence: `public.${table.name}_${column}_seq`, + })), + ); + } + if (sql.startsWith('DO $clear$')) { + // The current schema is emptied: the database is as the move found it. + this.bootstrapped = false; + this.tables.clear(); + return []; + } + if (sql.startsWith('select pg_get_serial_sequence($1, $2)')) return [{ sequence: `${String(params[0])}_seq` }]; + if (sql.startsWith('truncate table ')) { + for (const target of sql.slice('truncate table '.length).split(', ')) this.tableOf(target).rows = []; + return []; + } + if (sql.startsWith('insert into ')) return this.insert(sql, params); + if (sql.startsWith('update ')) return this.update(sql, params); + if (sql.startsWith('create table ')) return this.createTable(sql); + if (sql.startsWith('declare trek_move_verify')) return this.declare(sql); + if (sql.startsWith('fetch forward ')) { + const size = Number(/fetch forward (\d+)/.exec(sql)?.[1]); + return this.cursor.splice(0, size); + } + if (sql.startsWith('select max(')) { + const [column] = unquote(sql.slice(0, sql.indexOf(')'))); + const table = this.tableOf(sql.slice(sql.indexOf(' from ') + 6)); + const index = table.columns.indexOf(column); + const values = table.rows + .map((row) => row[index]) + .filter((value) => value !== null) + .map((value) => BigInt(String(value))); + const max = values.reduce((best, value) => (best === null || value > best ? value : best), null); + return [{ max: max === null ? null : max.toString() }]; + } + if (sql.startsWith('select setval(')) { + this.sequences.set(String(params[0]), { value: String(params[1]), called: Boolean(params[2]) }); + return []; + } + return []; + } + + private tableOf(target: string): FakeTable { + const table = this.tables.get(target.trim()); + if (!table) throw new Error(`relation ${target} does not exist`); + return table; + } + + private insert(sql: string, params: unknown[]): unknown[] { + const match = /^insert into (.+?) \((.+?)\) values /.exec(sql); + if (!match) throw new Error(`unparsed insert: ${sql}`); + const table = this.tableOf(match[1]); + const columns = unquote(match[2]); + for (let at = 0; at < params.length; at += columns.length) { + const values = params.slice(at, at + columns.length); + table.rows.push(table.columns.map((name) => values[columns.indexOf(name)] ?? null)); + } + return []; + } + + private update(sql: string, params: unknown[]): unknown[] { + const match = /^update (.+?) set (.+) where (.+)$/.exec(sql); + if (!match) throw new Error(`unparsed update: ${sql}`); + const table = this.tableOf(match[1]); + const sets = unquote(match[2]); + const keys = unquote(match[3]); + const keyValues = params.slice(sets.length); + const row = table.rows.find((candidate) => + keys.every((key, index) => String(candidate[table.columns.indexOf(key)]) === String(keyValues[index])), + ); + if (!row) throw new Error(`no row in ${match[1]} to update`); + sets.forEach((column, index) => { + row[table.columns.indexOf(column)] = params[index]; + }); + return []; + } + + private createTable(sql: string): unknown[] { + const match = /^create table (\S+?\.\S+?|\S+) \((.*)\)$/s.exec(sql); + if (!match) throw new Error(`unparsed create table: ${sql}`); + const definitions = match[2].split(/, (?![^(]*\))/).filter((part) => !part.startsWith('primary key')); + const columns = definitions.map((part) => (part.startsWith('"') ? unquote(part)[0] : part.split(' ')[0])); + const target = match[1].replace(/\._plugin_migrations$/, '."_plugin_migrations"'); + this.tables.set(target, { columns, rows: [] }); + return []; + } + + private declare(sql: string): unknown[] { + const target = sql.slice(sql.lastIndexOf(' from ') + 6); + const table = this.tableOf(target); + const selected = [...sql.matchAll(/"((?:[^"]|"")*)"::text/g)].map((match) => match[1]); + const tamper = this.tamper.get(target) ?? ((rows: unknown[][]) => rows); + this.cursor = tamper(table.rows).map((row) => + Object.fromEntries(selected.map((name, index) => [`c${index}`, asText(row[table.columns.indexOf(name)])])), + ); + return []; + } +} + +/** + * An ORM whose connection is the fake: enough for the engine's bootstrap + * (`schema-bootstrap.ts`) to run its empty-database path, and for the move to + * reach the fake through `getClient()`. + */ +export function fakeMoveOrm(pg: FakePostgres, options: { pending?: string[]; known: string[] }): AnyOrm { + const executed: string[] = []; + const connection = { + execute: vi.fn(async (sql: string) => { + if (sql.includes('server_version_num')) return [{ schema: 'public', version: '170004', privileged: false }]; + const answer = await pg.executeQuery({ sql, parameters: [], query: undefined } as unknown as CompiledQuery); + if (sql.startsWith('create table "baseline"')) pg.bootstrap(); + return answer.rows; + }), + transactional: vi.fn(async (work: (ctx: unknown) => Promise) => work({})), + getClient: () => pg.asKysely(), + }; + const migrator = { + getStorage: () => ({ + ensureTable: vi.fn(async () => undefined), + logMigration: vi.fn(async ({ name }: { name: string }) => { + executed.push(name); + }), + }), + getExecuted: vi.fn(async () => executed.map((name) => ({ name }))), + getPending: vi.fn(async () => (options.pending ?? []).map((name) => ({ name }))), + up: vi.fn(async () => undefined), + }; + return { + config: { getExtension: () => migrator, get: () => ({ tableName: 'mikro_orm_migrations' }) }, + em: { getConnection: () => connection }, + } as unknown as AnyOrm; +} diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts new file mode 100644 index 000000000..077cf2e5c --- /dev/null +++ b/server/tests/unit/db/engine/postgres/sqlite-move/move.test.ts @@ -0,0 +1,455 @@ +/** + * MOVE-001..016: the move of a SQLite install into Postgres + * (src/db/engine/postgres/sqlite-move/), over real SQLite files and a + * stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts): + * the refusals before anything is written, the copy in foreign key order with + * a table that points at itself, the sequences, the verification, a plugin's + * own database, the cleanup after a failure, and the command around it. The + * Postgres lane runs the same move against a real server + * (tests/postgres/sqlite-move.test.ts). + */ +import type { AppEnv } from '../../../../../../src/app-config'; +import { CLEAR_SCHEMA_SQL } from '../../../../../../src/db/engine/postgres/clear-schema'; +import { + runMoveCommand, + USAGE, + type MoveCommandIo, +} from '../../../../../../src/db/engine/postgres/sqlite-move/command'; +import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy'; +import { + migrationLevelProblem, + MoveRefusal, + moveSqliteToPostgres, + renderSummary, + type MoveOptions, + type MoveSummary, +} from '../../../../../../src/db/engine/postgres/sqlite-move/move'; +import { + findPluginDataFiles, + pluginColumnKind, + pluginDefault, +} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data'; +import { sqliteInstallHoldsAccounts, SqliteSource } from '../../../../../../src/db/engine/sqlite/sqlite-source'; +import type { AnyOrm } from '../../../../../../src/db/orm'; +import { FakePostgres, fakeMoveOrm, type FakeTableDef } from '../../../../../helpers/fake-postgres-move'; + +import Database from 'better-sqlite3'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const M1 = 'Migration20200101000000_baseline_schema'; +const M2 = 'Migration20200101000100_second'; + +/** TREK's schema in miniature, as the bootstrap leaves it on Postgres. */ +const SCHEMA: FakeTableDef[] = [ + { + name: 'messages', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'trip_id', dataType: 'bigint' }, + { name: 'reply_to', dataType: 'bigint' }, + { name: 'body', dataType: 'text' }, + ], + primaryKey: ['id'], + foreignKeys: [ + { columns: ['trip_id'], references: 'trips' }, + { columns: ['reply_to'], references: 'messages' }, + ], + serial: ['id'], + }, + { + name: 'trips', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'user_id', dataType: 'bigint' }, + { name: 'title', dataType: 'text' }, + { name: 'meta', dataType: 'jsonb' }, + { name: 'cover', dataType: 'bytea' }, + { name: 'score', dataType: 'double precision' }, + ], + primaryKey: ['id'], + foreignKeys: [{ columns: ['user_id'], references: 'users' }], + serial: ['id'], + }, + { + name: 'users', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'name', dataType: 'text', nullable: false }, + ], + primaryKey: ['id'], + serial: ['id'], + }, +]; + +const SOURCE_SQL = ` + create table mikro_orm_migrations (id integer primary key autoincrement, name text, executed_at text); + insert into mikro_orm_migrations (name) values ('${M1}'), ('${M2}'); + create table users (id integer primary key autoincrement, name text not null); + create table trips (id integer primary key, user_id integer references users (id), title text, meta text, cover blob, score real); + create table messages (id integer primary key, trip_id integer references trips (id), reply_to integer references messages (id), body text); + insert into users (name) values ('Ada'), ('Grace'), ('Gone'); + delete from users where name = 'Gone'; + insert into trips values (1, 1, 'Lisbon', '{"b":1,"a":2}', x'00ff', 1.5), (2, 2, 'Porto', null, null, null); + insert into messages values (1, 1, 2, 'I do'), (2, 1, null, 'Who books the tram?'); +`; + +let dir: string; +let pg: FakePostgres; +let orm: AnyOrm; +const log = vi.fn(); + +function sqlite(file: string, sql: string): string { + const db = new Database(file); + db.exec(sql); + db.close(); + return file; +} + +function options(overrides: Partial = {}): MoveOptions { + return { + sqliteFile: path.join(dir, 'travel.db'), + pluginsDataDir: path.join(dir, 'plugins-data'), + orm, + known: new Set([M1, M2]), + baseline: () => ({ through: M2, migrations: [M1, M2], statements: ['create table "baseline" ("id" bigint)'] }), + log, + ...overrides, + }; +} + +beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-move-')); + sqlite(path.join(dir, 'travel.db'), SOURCE_SQL); + pg = new FakePostgres(SCHEMA); + orm = fakeMoveOrm(pg, { known: [M1, M2] }); +}); + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe('moveSqliteToPostgres', () => { + it('MOVE-001: fills every table in foreign key order, sets the self-references afterwards and verifies the copy', async () => { + const summary = await moveSqliteToPostgres(options()); + expect(summary.tables.map((table) => [table.label, table.rows])).toEqual([ + ['users', 2], + ['trips', 2], + ['messages', 2], + ]); + expect(pg.rowsOf('"users"')).toEqual([ + ['1', 'Ada'], + ['2', 'Grace'], + ]); + expect(pg.rowsOf('"trips"')).toEqual([ + ['1', '1', 'Lisbon', '{"b":1,"a":2}', Buffer.from([0, 255]), 1.5], + ['2', '2', 'Porto', null, null, null], + ]); + // The reply was inserted without its reference, then pointed at the message it answers. + expect(pg.rowsOf('"messages"')).toEqual([ + ['1', '1', '2', 'I do'], + ['2', '1', null, 'Who books the tram?'], + ]); + const statements = pg.statements.join('\n'); + expect(statements).toMatch(/truncate table "users", "trips", "messages"/); + expect(statements).toMatch(/update "messages" set "reply_to" = \$1 where "id" = \$2/); + expect(statements).not.toContain(CLEAR_SCHEMA_SQL); + expect(summary.warnings).toEqual([]); + }); + + it('MOVE-002: moves every sequence past the copied ids, and past ids AUTOINCREMENT handed out before', async () => { + const summary = await moveSqliteToPostgres(options()); + expect(summary.sequences).toBe(3); + expect(pg.sequences.get('public.users_id_seq')).toEqual({ value: '3', called: true }); + expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '2', called: true }); + expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '2', called: true }); + }); + + it('MOVE-003: an empty table starts its sequence at 1', async () => { + sqlite(path.join(dir, 'travel.db'), 'delete from messages;'); + await moveSqliteToPostgres(options()); + expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '1', called: false }); + }); + + it('MOVE-004: a copy that does not read back as sent is refused and the schema dropped again', async () => { + pg.tamper.set('"trips"', (rows) => rows.map((row) => row.map((value) => (value === 'Porto' ? 'Braga' : value)))); + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + /does not match the SQLite file; nothing was written:\n {2}- trips: the rows Postgres holds differ/, + ); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + }); + + it('MOVE-005: a row count that differs is named with all three counts', async () => { + pg.tamper.set('"users"', (rows) => rows.slice(1)); + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + 'users: the SQLite file holds 2 rows, 2 were sent, Postgres holds 1', + ); + }); + + it('MOVE-006: values Postgres cannot hold are listed by row and column, and nothing is inserted', async () => { + sqlite(path.join(dir, 'travel.db'), "update users set id = id; update trips set score = 'high' where id = 2;"); + const failure = moveSqliteToPostgres(options()); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + /1 value\(s\) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n {2}- trips row id=2, column score: the text "high" is not a number/, + ); + expect(pg.statements.some((sql) => sql.startsWith('insert into "messages"'))).toBe(false); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + }); + + it('MOVE-007: a statement Postgres refuses is named with its table, detail and constraint', async () => { + pg.failOn = /^insert into "trips"/; + await expect(moveSqliteToPostgres(options())).rejects.toThrow( + 'Postgres refused rows of trips: refused by the fake; constraint fake_key; Key (id)=(1)', + ); + expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL); + }); + + it('MOVE-008: when the schema cannot be dropped after a failure, the error says the database needs dropping', async () => { + pg.failOn = /^(insert into "trips"|DO \$clear\$)/; + await expect(moveSqliteToPostgres(options())).rejects.toThrow(/could not be dropped .* Drop the Postgres database/); + }); + + it('MOVE-009: refuses a target that holds a table or a plugin schema, before it writes anything', async () => { + pg.existing = ['customers']; + await expect(moveSqliteToPostgres(options())).rejects.toThrow(/The Postgres database is not empty \(customers\)/); + pg.existing = []; + pg.pluginSchemas = ['trek_plugin_notes']; + await expect(moveSqliteToPostgres(options())).rejects.toThrow(/schema trek_plugin_notes/); + expect(pg.statements.some((sql) => sql.startsWith('create') || sql.startsWith('insert'))).toBe(false); + }); + + it('MOVE-010: refuses a SQLite file at another migration level, one with orphaned rows, and one it cannot read', async () => { + await expect( + moveSqliteToPostgres(options({ known: new Set([M1, M2, 'Migration20200101000200_third']) })), + ).rejects.toThrow(/1 migration\(s\) behind this TREK version\. Start this TREK version on it once/); + sqlite( + path.join(dir, 'travel.db'), + "pragma foreign_keys = off; insert into trips (id, user_id, title) values (9, 77, 'Nowhere');", + ); + await expect(moveSqliteToPostgres(options())).rejects.toThrow(/trips row 9 refers to a missing row of users/); + await expect(moveSqliteToPostgres(options({ sqliteFile: path.join(dir, 'missing.db') }))).rejects.toThrow( + /Cannot read the SQLite file .*missing\.db: There is no SQLite file/, + ); + }); + + it('MOVE-011: a plugin database goes into its schema with its migration record, ids and indexes', async () => { + const pluginDir = path.join(dir, 'plugins-data', 'move-notes'); + fs.mkdirSync(pluginDir, { recursive: true }); + fs.mkdirSync(path.join(dir, 'plugins-data', 'Not_An_Id'), { recursive: true }); + sqlite(path.join(dir, 'plugins-data', 'Not_An_Id', 'plugin.db'), 'create table t (a);'); + sqlite( + path.join(pluginDir, 'plugin.db'), + `create table _plugin_migrations (id text primary key, applied_at integer); + insert into _plugin_migrations values ('001', 1700000000000); + create table notes (id integer primary key autoincrement, body text not null default 'x', score real, raw blob, + added text default current_timestamp, odd text default (lower('A'))); + create unique index notes_body on notes (body); + create index notes_hot on notes (score) where score > 1; + create view recent as select * from notes; + insert into notes (body, score, raw) values ('a', 1.5, x'01'), ('b', null, null), ('c', 3, null); + delete from notes where body = 'c';`, + ); + const summary = await moveSqliteToPostgres(options()); + expect(summary.plugins).toEqual([ + { + pluginId: 'move-notes', + schema: 'trek_plugin_move_notes', + tables: [ + { label: '_plugin_migrations', rows: 1, checksum: expect.any(String) }, + { label: 'notes', rows: 2, checksum: expect.any(String) }, + ], + notCarried: [ + 'the view recent', + expect.stringMatching(/^the default \(?lower\('A'\)\)? of notes\.odd$/), + 'the index notes_hot on notes (partial or over an expression)', + ], + }, + ]); + expect(summary.warnings).toEqual([ + expect.stringMatching(/Not_An_Id.*plugin\.db is in a directory whose name is no plugin id/), + ]); + const statements = pg.statements.join('\n'); + expect(statements).toContain('create schema "trek_plugin_move_notes"'); + expect(statements).toContain( + 'create table "trek_plugin_move_notes"."notes" ("id" bigint generated by default as identity, ' + + '"body" text not null default \'x\', "score" double precision, "raw" bytea, "added" text default ' + + 'to_char((CURRENT_TIMESTAMP AT TIME ZONE \'UTC\'), \'YYYY-MM-DD HH24:MI:SS\'), "odd" text, primary key ("id"))', + ); + expect(statements).toContain('create unique index on "trek_plugin_move_notes"."notes" ("body")'); + expect(pg.rowsOf('"trek_plugin_move_notes"."_plugin_migrations"')).toEqual([['001', '1700000000000']]); + expect(pg.sequences.get('"trek_plugin_move_notes"."notes"_seq')).toEqual({ value: '3', called: true }); + expect(renderSummary(summary, 'postgres://db/trek')).toContain( + 'Plugin move-notes: 2 table(s), 3 rows, into schema trek_plugin_move_notes.', + ); + }); + + it("MOVE-012: a plugin database without a migration record gets the store's empty one", async () => { + const pluginDir = path.join(dir, 'plugins-data', 'bare-notes'); + fs.mkdirSync(pluginDir, { recursive: true }); + sqlite(path.join(pluginDir, 'plugin.db'), 'create table kv (k text primary key, v);'); + await moveSqliteToPostgres(options()); + expect(pg.statements).toContain( + 'create table "trek_plugin_bare_notes"._plugin_migrations (id text primary key, applied_at bigint)', + ); + expect(findPluginDataFiles(path.join(dir, 'nowhere'))).toEqual({ files: [], skipped: [] }); + }); + + it('MOVE-013: copies an integer past 2^53 exactly and warns that TREK cannot read it back on Postgres', async () => { + sqlite(path.join(dir, 'travel.db'), 'update trips set id = 9007199254740993 where id = 2;'); + const summary = await moveSqliteToPostgres(options()); + expect(pg.rowsOf('"trips"').map((row) => row[0])).toEqual(['1', '9007199254740993']); + expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '9007199254740993', called: true }); + expect(summary.warnings).toEqual(['trips holds 1 integer(s) beyond 2^53, which TREK refuses to read on Postgres.']); + }); +}); + +describe('migrationLevelProblem', () => { + it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => { + const known = new Set([M1, M2]); + expect(migrationLevelProblem([M1, M2], known)).toBeNull(); + expect(migrationLevelProblem(null, known)).toMatch(/no TREK migration history/); + expect(migrationLevelProblem([], known)).toMatch(/no TREK migration history/); + expect(migrationLevelProblem([M1, M2, 'Migration20990101000000_future'], known)).toMatch( + /a newer TREK migrated the SQLite file \(1 migration\(s\) this version does not know, the last Migration20990101000000_future\)/, + ); + expect(migrationLevelProblem([M1], known)).toMatch(/1 migration\(s\) behind/); + }); +}); + +describe('the pieces', () => { + it('MOVE-015: the plugin column rules, defaults and batches', () => { + const classes = (...names: string[]) => new Set(names); + expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer'); + expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text'); + expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes'); + expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text'); + expect(pluginColumnKind('DOUBLE', classes())).toBe('float'); + expect(pluginColumnKind('REAL', classes('text'))).toBe('text'); + expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer'); + expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float'); + expect(pluginColumnKind('', classes('blob'))).toBe('bytes'); + expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text'); + + expect(pluginDefault(null, 'text')).toBeNull(); + expect(pluginDefault('NULL', 'text')).toBe('null'); + expect(pluginDefault('(42)', 'integer')).toBe('42'); + expect(pluginDefault('42', 'text')).toBe("'42'"); + expect(pluginDefault('TRUE', 'integer')).toBe('1'); + expect(pluginDefault('false', 'integer')).toBe('0'); + expect(pluginDefault('true', 'text')).toBeNull(); + expect(pluginDefault("'it''s'", 'text')).toBe("'it''s'"); + expect(pluginDefault("strftime('%s','now')", 'integer')).toBe( + 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)', + ); + + expect(batchRows(2)).toBe(500); + expect(batchRows(200)).toBe(150); + expect( + insertSql( + { + target: '"t"', + columns: [ + { name: 'a"b', kind: 'text' }, + { name: 'c', kind: 'integer' }, + ], + }, + 2, + ), + ).toBe('insert into "t" ("a""b", "c") values ($1, $2), ($3, $4)'); + expect(describePgError('plain')).toBe('plain'); + expect(describePgError(new Error('bare'))).toBe('bare'); + }); + + it('MOVE-016: a SQLite install holds accounts only with a user row; an unreadable file counts as holding them', () => { + const empty = path.join(dir, 'empty.db'); + fs.writeFileSync(empty, ''); + expect(sqliteInstallHoldsAccounts(empty)).toBe(false); + expect(sqliteInstallHoldsAccounts(path.join(dir, 'none.db'))).toBe(false); + expect(sqliteInstallHoldsAccounts(path.join(dir, 'travel.db'))).toBe(true); + expect(sqliteInstallHoldsAccounts(sqlite(path.join(dir, 'other.db'), 'create table notes (a);'))).toBe(false); + const broken = path.join(dir, 'broken.db'); + fs.writeFileSync(broken, 'not a database at all, but long enough to be read as one'); + expect(sqliteInstallHoldsAccounts(broken)).toBe(true); + // Reading never writes the file. + const before = fs.readFileSync(path.join(dir, 'travel.db')); + const source = SqliteSource.open(path.join(dir, 'travel.db')); + expect(source.count('users')).toBe(2); + source.close(); + source.close(); + expect(fs.readFileSync(path.join(dir, 'travel.db')).equals(before)).toBe(true); + }); +}); + +describe('runMoveCommand', () => { + const env = (db: Record) => () => + ({ db: { trekDbFile: path.join(dir, 'travel.db'), ...db }, plugins: { dataDir: '' } }) as unknown as AppEnv; + const postgres = env({ engine: 'postgres', url: 'postgres://trek:secret@db:5432/trek' }); + const summary: MoveSummary = { + sqliteFile: 'travel.db', + tables: [{ label: 'users', rows: 2, checksum: 'x' }], + plugins: [], + sequences: 1, + warnings: ['careful'], + seconds: 0.4, + }; + + function io(overrides: Partial = {}) { + const out: string[] = []; + const err: string[] = []; + const close = vi.fn(async () => undefined); + return { + out, + err, + close, + io: { + env: postgres, + out: (line: string) => out.push(line), + err: (line: string) => err.push(line), + openOrm: async () => ({ close }) as unknown as AnyOrm, + move: vi.fn(async () => summary), + ...overrides, + }, + }; + } + + it('MOVE-017: prints the usage, refuses an unknown argument and a missing TREK_DB_URL', async () => { + const help = io(); + expect(await runMoveCommand(['--help'], help.io)).toBe(0); + expect(help.out).toEqual([USAGE]); + const unknown = io(); + expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1); + expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/); + const onSqlite = io({ env: env({ engine: 'sqlite' }) }); + expect(await runMoveCommand([], onSqlite.io)).toBe(1); + expect(onSqlite.err[0]).toMatch(/Set TREK_DB_URL/); + }); + + it('MOVE-018: moves the given file, prints the summary without the password and closes the ORM', async () => { + const run = io(); + expect(await runMoveCommand([`--from=${path.join(dir, 'other.db')}`], run.io)).toBe(0); + expect(run.io.move).toHaveBeenCalledWith( + expect.objectContaining({ + sqliteFile: path.join(dir, 'other.db'), + pluginsDataDir: expect.stringMatching(/plugins-data$/), + }), + ); + const printed = run.out.join('\n'); + expect(printed).toContain('postgres://trek:***@db:5432/trek'); + expect(printed).not.toContain('secret'); + expect(printed).toMatch( + /1 tables, 2 rows, each table verified by row count and checksum[\s\S]*users: 2[\s\S]*Warning: careful/, + ); + expect(run.close).toHaveBeenCalledWith(true); + }); + + it('MOVE-019: a failure prints its cause and still closes the ORM', async () => { + const failing = io({ move: vi.fn(async () => Promise.reject(new Error('connection lost'))) }); + expect(await runMoveCommand([], failing.io)).toBe(1); + expect(failing.err[0]).toMatch(/The move failed; .*connection lost/s); + expect(failing.close).toHaveBeenCalled(); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts new file mode 100644 index 000000000..c8515e4e8 --- /dev/null +++ b/server/tests/unit/db/engine/postgres/sqlite-move/order.test.ts @@ -0,0 +1,175 @@ +/** + * MOVEORD-001..005: the order the move fills the tables in, and the foreign + * keys it sets only once every row is in (src/db/engine/postgres/sqlite-move/order.ts), + * plus how the target's tables are paired with the SQLite file's (catalog.ts). + */ +import { + planTrekTables, + readTargetSchema, + type TargetSchema, +} from '../../../../../../src/db/engine/postgres/sqlite-move/catalog'; +import { planCopyOrder } from '../../../../../../src/db/engine/postgres/sqlite-move/order'; +import { FakePostgres } from '../../../../../helpers/fake-postgres-move'; + +import type { Transaction } from 'kysely'; +import { describe, expect, it } from 'vitest'; + +describe('planCopyOrder', () => { + it('MOVEORD-001: puts every table after the tables it points at, ties by name', () => { + const plan = planCopyOrder( + ['places', 'trips', 'users', 'days', 'categories'], + [ + { table: 'trips', columns: ['user_id'], references: 'users' }, + { table: 'days', columns: ['trip_id'], references: 'trips' }, + { table: 'places', columns: ['trip_id'], references: 'trips' }, + { table: 'places', columns: ['category_id'], references: 'categories' }, + ], + ); + expect(plan.order).toEqual(['categories', 'users', 'trips', 'days', 'places']); + expect(plan.deferred.size).toBe(0); + }); + + it('MOVEORD-002: a table pointing at itself has those columns set after its rows are in', () => { + const plan = planCopyOrder( + ['collab_messages'], + [ + { table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' }, + { table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' }, + ], + ); + expect(plan.order).toEqual(['collab_messages']); + expect(plan.deferred.get('collab_messages')).toEqual(['reply_to']); + }); + + it('MOVEORD-003: a cycle is broken at the first table by name, deferring its keys into the cycle', () => { + const plan = planCopyOrder( + ['b', 'a', 'c'], + [ + { table: 'a', columns: ['b_id'], references: 'b' }, + { table: 'b', columns: ['a_id'], references: 'a' }, + { table: 'c', columns: ['a_id'], references: 'a' }, + ], + ); + expect(plan.order).toEqual(['a', 'b', 'c']); + expect([...plan.deferred]).toEqual([['a', ['b_id']]]); + }); + + it('MOVEORD-004: a key to a table the move does not copy is ignored', () => { + const plan = planCopyOrder(['trips'], [{ table: 'trips', columns: ['user_id'], references: 'users' }]); + expect(plan.order).toEqual(['trips']); + }); +}); + +describe('planTrekTables', () => { + const target: TargetSchema = { + tables: [ + { name: 'mikro_orm_migrations', columns: [{ name: 'name', dataType: 'text', nullable: false }], primaryKey: [] }, + { + name: 'messages', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'reply_to', dataType: 'bigint', nullable: true }, + ], + primaryKey: ['id'], + }, + { + name: 'users', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'avatar', dataType: 'bytea', nullable: true }, + ], + primaryKey: ['id'], + }, + ], + foreignKeys: [{ table: 'messages', columns: ['reply_to'], references: 'messages' }], + sequences: [{ table: 'users', column: 'id', sequence: 'public.users_id_seq' }], + }; + + it('MOVEORD-005: pairs the tables by name, leaves the history out and quotes every name', () => { + const plan = planTrekTables( + target, + new Map([ + ['mikro_orm_migrations', ['name']], + ['messages', ['id', 'reply_to']], + ['users', ['id', 'avatar']], + ]), + ); + expect(plan.problems).toEqual([]); + expect(plan.specs.map((spec) => spec.target)).toEqual(['"messages"', '"users"']); + expect(plan.specs[0]).toMatchObject({ deferred: ['reply_to'], primaryKey: ['id'], label: 'messages' }); + expect(plan.specs[1].columns).toEqual([ + { name: 'id', kind: 'integer' }, + { name: 'avatar', kind: 'bytes' }, + ]); + expect(plan.sequences).toEqual([ + { table: 'users', column: 'id', sequence: 'public.users_id_seq', target: '"users"' }, + ]); + }); + + it('MOVEORD-006: names every difference between the two schemas instead of copying around it', () => { + const odd: TargetSchema = { + ...target, + tables: [ + ...target.tables.filter((table) => table.name !== 'messages'), + { + name: 'messages', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'reply_to', dataType: 'bigint', nullable: false }, + { name: 'sent_at', dataType: 'timestamp with time zone', nullable: true }, + ], + primaryKey: ['id'], + }, + { name: 'extra', columns: [], primaryKey: [] }, + ], + }; + const plan = planTrekTables( + odd, + new Map([ + ['messages', ['id', 'reply_to', 'sent_at', 'legacy']], + ['users', ['id']], + ['leftover', ['id']], + ]), + ); + expect(plan.problems).toEqual([ + 'the SQLite table leftover has no Postgres table', + 'the Postgres table extra has no SQLite table', + 'the SQLite column messages.legacy has no Postgres column', + 'the Postgres column messages.sent_at has the type timestamp with time zone, which the move does not know', + 'messages.reply_to refers back to a table filled later and cannot be NULL meanwhile', + 'the Postgres column users.avatar has no SQLite column', + ]); + }); +}); + +describe('readTargetSchema', () => { + it('MOVEORD-007: reads the columns, keys and sequences of the current schema', async () => { + const pg = new FakePostgres([ + { + name: 'trips', + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'user_id', dataType: 'bigint' }, + ], + primaryKey: ['id'], + foreignKeys: [{ columns: ['user_id'], references: 'users' }], + serial: ['id'], + }, + { name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] }, + ]); + const schema = await readTargetSchema(pg.asKysely() as Transaction); + expect(schema.tables).toEqual([ + { + name: 'trips', + primaryKey: ['id'], + columns: [ + { name: 'id', dataType: 'bigint', nullable: false }, + { name: 'user_id', dataType: 'bigint', nullable: true }, + ], + }, + { name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }] }, + ]); + expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]); + expect(schema.sequences).toEqual([{ table: 'trips', column: 'id', sequence: 'public.trips_id_seq' }]); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts new file mode 100644 index 000000000..832497dd0 --- /dev/null +++ b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts @@ -0,0 +1,179 @@ +/** + * MOVEVAL-001..010: how a SQLite value becomes the value a Postgres column of + * the move's target takes, the text both sides are compared in, and the + * order-free table checksum (src/db/engine/postgres/sqlite-move/values.ts). + */ +import { + beyondSafeInteger, + canonicalFromPostgres, + canonicalFromSent, + columnKind, + convertValue, + stableJson, + TableChecksum, + type ColumnKind, +} from '../../../../../../src/db/engine/postgres/sqlite-move/values'; + +import fs from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const ok = (value: unknown) => ({ ok: true, value }); + +describe('columnKind', () => { + it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => { + const sql = fs.readFileSync(path.join(__dirname, '../../../../../../assets/postgres/baseline.sql'), 'utf8'); + const types = new Set( + [ + ...sql.matchAll( + /" (bigserial|bigint|integer|smallint|text|varchar\(\d+\)|double precision|real|jsonb|json|bytea|boolean|numeric|timestamptz|date|uuid)(?=[ ,)])/g, + ), + ].map((match) => match[1]), + ); + const asCatalog: Record = { bigserial: 'bigint' }; + for (const type of types) { + const dataType = asCatalog[type] ?? (type.startsWith('varchar') ? 'character varying' : type); + expect(columnKind(dataType), type).not.toBeNull(); + } + }); + + it('MOVEVAL-002: has no rule for a type it does not know, so the move refuses instead of guessing', () => { + expect(columnKind('timestamp with time zone')).toBeNull(); + expect(columnKind('boolean')).toBeNull(); + expect(columnKind('smallint')).toBe('integer'); + expect(columnKind('real')).toBe('float'); + expect(columnKind('character')).toBe('text'); + expect(columnKind('json')).toBe('json'); + }); +}); + +describe('convertValue', () => { + it('MOVEVAL-003: integers keep all 64 bits and accept whole numbers in any spelling SQLite stored them in', () => { + expect(convertValue(9007199254740993n, 'integer')).toEqual(ok('9007199254740993')); + expect(convertValue(42, 'integer')).toEqual(ok('42')); + expect(convertValue(' -7 ', 'integer')).toEqual(ok('-7')); + expect(convertValue(null, 'integer')).toEqual(ok(null)); + expect(convertValue(1.5, 'integer')).toMatchObject({ ok: false, reason: 'the value 1.5 is not a whole number' }); + expect(convertValue('abc', 'integer')).toMatchObject({ ok: false, reason: 'the text "abc" is not a whole number' }); + expect(convertValue('99999999999999999999', 'integer')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/does not fit in 64 bits/), + }); + expect(convertValue(new Uint8Array([1]), 'integer')).toMatchObject({ + ok: false, + reason: 'a 1-byte BLOB is not a whole number', + }); + }); + + it('MOVEVAL-004: floats take numbers and numeric text, never a value a double cannot hold', () => { + expect(convertValue(1.25, 'float')).toEqual(ok(1.25)); + expect(convertValue(12n, 'float')).toEqual(ok(12)); + expect(convertValue('3.5e2', 'float')).toEqual(ok(350)); + expect(convertValue(9007199254740993n, 'float')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/more digits than a double holds/), + }); + expect(convertValue('north', 'float')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/is not a number/), + }); + expect(convertValue('', 'float')).toMatchObject({ ok: false }); + }); + + it('MOVEVAL-005: text takes strings and numbers, and refuses a NUL Postgres cannot store or a BLOB', () => { + expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa')); + expect(convertValue(7n, 'text')).toEqual(ok('7')); + expect(convertValue(2.5, 'text')).toEqual(ok('2.5')); + expect(convertValue('a\u0000b', 'text')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/NUL character/), + }); + expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/is not text/), + }); + const long = 'x'.repeat(60); + expect(convertValue(long, 'integer')).toMatchObject({ + reason: `the text "${'x'.repeat(40)}..." is not a whole number`, + }); + }); + + it('MOVEVAL-006: JSON must parse and hold no NUL, escaped or not', () => { + expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}')); + expect(convertValue(5n, 'json')).toEqual(ok('5')); + expect(convertValue('{broken', 'json')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/is not valid JSON/), + }); + expect(convertValue('"\\u0000"', 'json')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/jsonb cannot store/), + }); + expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({ + ok: false, + reason: expect.stringMatching(/is not JSON text/), + }); + }); + + it('MOVEVAL-007: bytes take a BLOB, or the UTF-8 of text, and refuse a number', () => { + expect(convertValue(new Uint8Array([0, 255]), 'bytes')).toEqual(ok(Buffer.from([0, 255]))); + expect(convertValue('hé', 'bytes')).toEqual(ok(Buffer.from('hé', 'utf8'))); + expect(convertValue(3n, 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not a BLOB/) }); + }); + + it('MOVEVAL-008: flags an integer past 2^53, which the Postgres driver refuses to read', () => { + expect(beyondSafeInteger('9007199254740993')).toBe(true); + expect(beyondSafeInteger('-9007199254740993')).toBe(true); + expect(beyondSafeInteger('9007199254740991')).toBe(false); + expect(beyondSafeInteger(12)).toBe(false); + expect(beyondSafeInteger('text')).toBe(false); + }); +}); + +describe('the compared text', () => { + const cases: Array<[ColumnKind, Parameters[0], string]> = [ + ['integer', '-12', '-12'], + ['float', 0.30000000000000004, '0.30000000000000004'], + ['float', 1e21, '1e+21'], + ['text', 'São Paulo', 'São Paulo'], + ['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'], + ['bytes', Buffer.from([0, 171]), '\\x00ab'], + ]; + + it.each(cases)('MOVEVAL-009: %s values compare equal as sent and as Postgres prints them', (kind, sent, printed) => { + expect(canonicalFromSent(sent, kind)).toBe(canonicalFromPostgres(printed, kind)); + expect(canonicalFromSent(null, kind)).toBeNull(); + expect(canonicalFromPostgres(null, kind)).toBeNull(); + }); + + it('MOVEVAL-009b: text sent for a BLOB compares as its bytes, and a float as Postgres spells it', () => { + expect(canonicalFromSent('ab', 'bytes')).toBe('6162'); + expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSent(1e20, 'float')); + expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain'); + expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}'); + }); + + it('MOVEVAL-010: the checksum ignores row order and catches a changed, missing or doubled row', () => { + const sum = (rows: Array>) => { + const checksum = new TableChecksum(); + for (const row of rows) checksum.add(row); + return checksum.hex(); + }; + const rows = [ + ['1', 'a'], + ['2', null], + ]; + expect(sum(rows)).toBe(sum([...rows].reverse())); + expect(sum(rows)).toHaveLength(64); + expect(sum(rows)).not.toBe( + sum([ + ['1', 'a'], + ['2', 'b'], + ]), + ); + expect(sum(rows)).not.toBe(sum([['1', 'a']])); + expect(sum(rows)).not.toBe(sum([...rows, ['2', null]])); + // A NULL and the text "null" are different values. + expect(sum([['null']])).not.toBe(sum([[null]])); + }); +});