feat(db): move a SQLite install into an empty Postgres database

npm run db:move-to-postgres (dist/move-to-postgres.js in the image) copies
a SQLite file at this build's migration level into an empty Postgres
database. It refuses a file at another level, rows whose references point
at nothing and a target holding any table or plugin schema, creates the
schema with the engine's own bootstrap, then in one transaction copies every
table in foreign key order (setting a self-reference once its rows are in),
converts each value to its column's type and refuses one Postgres cannot
hold exactly, copies each plugin's plugin.db into its schema, moves the
sequences past the copied ids and compares every table's row count and an
order-free checksum over all rows before it commits. A failure drops the
schema again. The SQLite files and the uploads are never written.
This commit is contained in:
Maurice
2026-11-17 13:41:20 +01:00
parent 1ea08dcf0a
commit 55607ff2f9
14 changed files with 2535 additions and 0 deletions
+261
View File
@@ -0,0 +1,261 @@
import type { AnyOrm } from '../../src/db/orm';
import type { CompiledQuery, Kysely } from 'kysely';
import { vi } from 'vitest';
/**
* A stand-in for the Postgres side of the SQLite move
* (`src/db/engine/postgres/sqlite-move/`), for the unit tests in the SQLite
* lane: it answers exactly the statements the move sends, keeps the rows it
* is given in memory and reads them back as Postgres's `::text` would. The
* real database is the Postgres lane's (`tests/postgres/sqlite-move.test.ts`).
*/
interface FakeColumn {
name: string;
dataType: string;
nullable?: boolean;
}
export interface FakeTableDef {
name: string;
columns: FakeColumn[];
primaryKey: string[];
/** Foreign keys: `columns` of this table point at `references`. */
foreignKeys?: Array<{ columns: string[]; references: string }>;
/** Columns a sequence fills. */
serial?: string[];
}
interface FakeTable {
columns: string[];
rows: unknown[][];
}
const QUOTED = /"((?:[^"]|"")*)"/g;
function unquote(text: string): string[] {
return [...text.matchAll(QUOTED)].map((match) => match[1].replaceAll('""', '"'));
}
/** A value as Postgres would print it with `::text` and `bytea_output = hex`. */
function asText(value: unknown): string | null {
if (value === null || value === undefined) return null;
if (Buffer.isBuffer(value)) return `\\x${value.toString('hex')}`;
return String(value);
}
export class FakePostgres {
/** Every statement, in order. */
readonly statements: string[] = [];
readonly tables = new Map<string, FakeTable>();
readonly sequences = new Map<string, { value: string; called: boolean }>();
/** Tables the database holds before anything runs (the move refuses them). */
existing: string[] = [];
pluginSchemas: string[] = [];
/** A statement matching this fails, as Postgres would refuse it. */
failOn: RegExp | null = null;
/** Rows the read-back answers in place of the stored ones, by table: a copy that came out wrong. */
tamper = new Map<string, (rows: unknown[][]) => unknown[][]>();
private cursor: Array<Record<string, string | null>> = [];
private bootstrapped = false;
constructor(readonly schema: FakeTableDef[]) {}
/** What the engine's bootstrap leaves: every table of the schema, with a seeded row or two. */
bootstrap(): void {
this.bootstrapped = true;
for (const table of this.schema) {
this.tables.set(`"${table.name}"`, { columns: table.columns.map((column) => column.name), rows: [] });
}
const first = this.schema[0];
this.tables.get(`"${first.name}"`)?.rows.push(first.columns.map(() => null));
}
rowsOf(target: string): unknown[][] {
return this.tables.get(target)?.rows ?? [];
}
async executeQuery<R>(query: CompiledQuery): Promise<{ rows: R[] }> {
const sql = query.sql;
const params = [...query.parameters];
this.statements.push(sql);
if (this.failOn?.test(sql)) {
throw Object.assign(new Error('refused by the fake'), { detail: 'Key (id)=(1)', constraint: 'fake_key' });
}
return { rows: this.answer(sql, params) as R[] };
}
/** `db.transaction().execute(fn)`: the fake has no isolation, only the order of statements. */
transaction() {
return { execute: <T>(work: (trx: Kysely<object>) => Promise<T>) => work(this.asKysely()) };
}
asKysely(): Kysely<object> {
return this as unknown as Kysely<object>;
}
private answer(sql: string, params: unknown[]): unknown[] {
if (sql.startsWith('select table_name from information_schema.tables')) {
return [...this.existing, ...(this.bootstrapped ? this.schema.map((table) => table.name) : [])].map(
(table_name) => ({ table_name }),
);
}
if (sql.startsWith('select nspname from pg_namespace')) return this.pluginSchemas.map((nspname) => ({ nspname }));
if (sql.includes('from information_schema.columns c')) {
return this.schema.flatMap((table) =>
table.columns.map((column) => ({
table_name: table.name,
column_name: column.name,
data_type: column.dataType,
is_nullable: column.nullable === false ? 'NO' : 'YES',
})),
);
}
if (sql.includes('from pg_constraint')) {
return this.schema.flatMap((table) => [
{ kind: 'p', table_name: table.name, target_name: null, columns: table.primaryKey },
...(table.foreignKeys ?? []).map((fk) => ({
kind: 'f',
table_name: table.name,
target_name: fk.references,
columns: fk.columns,
})),
]);
}
if (sql.includes("pg_get_serial_sequence(format('%I'")) {
return this.schema.flatMap((table) =>
(table.serial ?? []).map((column) => ({
table_name: table.name,
column_name: column,
sequence: `public.${table.name}_${column}_seq`,
})),
);
}
if (sql.startsWith('DO $clear$')) {
// The current schema is emptied: the database is as the move found it.
this.bootstrapped = false;
this.tables.clear();
return [];
}
if (sql.startsWith('select pg_get_serial_sequence($1, $2)')) return [{ sequence: `${String(params[0])}_seq` }];
if (sql.startsWith('truncate table ')) {
for (const target of sql.slice('truncate table '.length).split(', ')) this.tableOf(target).rows = [];
return [];
}
if (sql.startsWith('insert into ')) return this.insert(sql, params);
if (sql.startsWith('update ')) return this.update(sql, params);
if (sql.startsWith('create table ')) return this.createTable(sql);
if (sql.startsWith('declare trek_move_verify')) return this.declare(sql);
if (sql.startsWith('fetch forward ')) {
const size = Number(/fetch forward (\d+)/.exec(sql)?.[1]);
return this.cursor.splice(0, size);
}
if (sql.startsWith('select max(')) {
const [column] = unquote(sql.slice(0, sql.indexOf(')')));
const table = this.tableOf(sql.slice(sql.indexOf(' from ') + 6));
const index = table.columns.indexOf(column);
const values = table.rows
.map((row) => row[index])
.filter((value) => value !== null)
.map((value) => BigInt(String(value)));
const max = values.reduce<bigint | null>((best, value) => (best === null || value > best ? value : best), null);
return [{ max: max === null ? null : max.toString() }];
}
if (sql.startsWith('select setval(')) {
this.sequences.set(String(params[0]), { value: String(params[1]), called: Boolean(params[2]) });
return [];
}
return [];
}
private tableOf(target: string): FakeTable {
const table = this.tables.get(target.trim());
if (!table) throw new Error(`relation ${target} does not exist`);
return table;
}
private insert(sql: string, params: unknown[]): unknown[] {
const match = /^insert into (.+?) \((.+?)\) values /.exec(sql);
if (!match) throw new Error(`unparsed insert: ${sql}`);
const table = this.tableOf(match[1]);
const columns = unquote(match[2]);
for (let at = 0; at < params.length; at += columns.length) {
const values = params.slice(at, at + columns.length);
table.rows.push(table.columns.map((name) => values[columns.indexOf(name)] ?? null));
}
return [];
}
private update(sql: string, params: unknown[]): unknown[] {
const match = /^update (.+?) set (.+) where (.+)$/.exec(sql);
if (!match) throw new Error(`unparsed update: ${sql}`);
const table = this.tableOf(match[1]);
const sets = unquote(match[2]);
const keys = unquote(match[3]);
const keyValues = params.slice(sets.length);
const row = table.rows.find((candidate) =>
keys.every((key, index) => String(candidate[table.columns.indexOf(key)]) === String(keyValues[index])),
);
if (!row) throw new Error(`no row in ${match[1]} to update`);
sets.forEach((column, index) => {
row[table.columns.indexOf(column)] = params[index];
});
return [];
}
private createTable(sql: string): unknown[] {
const match = /^create table (\S+?\.\S+?|\S+) \((.*)\)$/s.exec(sql);
if (!match) throw new Error(`unparsed create table: ${sql}`);
const definitions = match[2].split(/, (?![^(]*\))/).filter((part) => !part.startsWith('primary key'));
const columns = definitions.map((part) => (part.startsWith('"') ? unquote(part)[0] : part.split(' ')[0]));
const target = match[1].replace(/\._plugin_migrations$/, '."_plugin_migrations"');
this.tables.set(target, { columns, rows: [] });
return [];
}
private declare(sql: string): unknown[] {
const target = sql.slice(sql.lastIndexOf(' from ') + 6);
const table = this.tableOf(target);
const selected = [...sql.matchAll(/"((?:[^"]|"")*)"::text/g)].map((match) => match[1]);
const tamper = this.tamper.get(target) ?? ((rows: unknown[][]) => rows);
this.cursor = tamper(table.rows).map((row) =>
Object.fromEntries(selected.map((name, index) => [`c${index}`, asText(row[table.columns.indexOf(name)])])),
);
return [];
}
}
/**
* An ORM whose connection is the fake: enough for the engine's bootstrap
* (`schema-bootstrap.ts`) to run its empty-database path, and for the move to
* reach the fake through `getClient()`.
*/
export function fakeMoveOrm(pg: FakePostgres, options: { pending?: string[]; known: string[] }): AnyOrm {
const executed: string[] = [];
const connection = {
execute: vi.fn(async (sql: string) => {
if (sql.includes('server_version_num')) return [{ schema: 'public', version: '170004', privileged: false }];
const answer = await pg.executeQuery({ sql, parameters: [], query: undefined } as unknown as CompiledQuery);
if (sql.startsWith('create table "baseline"')) pg.bootstrap();
return answer.rows;
}),
transactional: vi.fn(async (work: (ctx: unknown) => Promise<void>) => work({})),
getClient: () => pg.asKysely(),
};
const migrator = {
getStorage: () => ({
ensureTable: vi.fn(async () => undefined),
logMigration: vi.fn(async ({ name }: { name: string }) => {
executed.push(name);
}),
}),
getExecuted: vi.fn(async () => executed.map((name) => ({ name }))),
getPending: vi.fn(async () => (options.pending ?? []).map((name) => ({ name }))),
up: vi.fn(async () => undefined),
};
return {
config: { getExtension: () => migrator, get: () => ({ tableName: 'mikro_orm_migrations' }) },
em: { getConnection: () => connection },
} as unknown as AnyOrm;
}
@@ -0,0 +1,455 @@
/**
* MOVE-001..016: the move of a SQLite install into Postgres
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
* the refusals before anything is written, the copy in foreign key order with
* a table that points at itself, the sequences, the verification, a plugin's
* own database, the cleanup after a failure, and the command around it. The
* Postgres lane runs the same move against a real server
* (tests/postgres/sqlite-move.test.ts).
*/
import type { AppEnv } from '../../../../../../src/app-config';
import { CLEAR_SCHEMA_SQL } from '../../../../../../src/db/engine/postgres/clear-schema';
import {
runMoveCommand,
USAGE,
type MoveCommandIo,
} from '../../../../../../src/db/engine/postgres/sqlite-move/command';
import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
import {
migrationLevelProblem,
MoveRefusal,
moveSqliteToPostgres,
renderSummary,
type MoveOptions,
type MoveSummary,
} from '../../../../../../src/db/engine/postgres/sqlite-move/move';
import {
findPluginDataFiles,
pluginColumnKind,
pluginDefault,
} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data';
import { sqliteInstallHoldsAccounts, SqliteSource } from '../../../../../../src/db/engine/sqlite/sqlite-source';
import type { AnyOrm } from '../../../../../../src/db/orm';
import { FakePostgres, fakeMoveOrm, type FakeTableDef } from '../../../../../helpers/fake-postgres-move';
import Database from 'better-sqlite3';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const M1 = 'Migration20200101000000_baseline_schema';
const M2 = 'Migration20200101000100_second';
/** TREK's schema in miniature, as the bootstrap leaves it on Postgres. */
const SCHEMA: FakeTableDef[] = [
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'trip_id', dataType: 'bigint' },
{ name: 'reply_to', dataType: 'bigint' },
{ name: 'body', dataType: 'text' },
],
primaryKey: ['id'],
foreignKeys: [
{ columns: ['trip_id'], references: 'trips' },
{ columns: ['reply_to'], references: 'messages' },
],
serial: ['id'],
},
{
name: 'trips',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint' },
{ name: 'title', dataType: 'text' },
{ name: 'meta', dataType: 'jsonb' },
{ name: 'cover', dataType: 'bytea' },
{ name: 'score', dataType: 'double precision' },
],
primaryKey: ['id'],
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
serial: ['id'],
},
{
name: 'users',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'name', dataType: 'text', nullable: false },
],
primaryKey: ['id'],
serial: ['id'],
},
];
const SOURCE_SQL = `
create table mikro_orm_migrations (id integer primary key autoincrement, name text, executed_at text);
insert into mikro_orm_migrations (name) values ('${M1}'), ('${M2}');
create table users (id integer primary key autoincrement, name text not null);
create table trips (id integer primary key, user_id integer references users (id), title text, meta text, cover blob, score real);
create table messages (id integer primary key, trip_id integer references trips (id), reply_to integer references messages (id), body text);
insert into users (name) values ('Ada'), ('Grace'), ('Gone');
delete from users where name = 'Gone';
insert into trips values (1, 1, 'Lisbon', '{"b":1,"a":2}', x'00ff', 1.5), (2, 2, 'Porto', null, null, null);
insert into messages values (1, 1, 2, 'I do'), (2, 1, null, 'Who books the tram?');
`;
let dir: string;
let pg: FakePostgres;
let orm: AnyOrm;
const log = vi.fn();
function sqlite(file: string, sql: string): string {
const db = new Database(file);
db.exec(sql);
db.close();
return file;
}
function options(overrides: Partial<MoveOptions> = {}): MoveOptions {
return {
sqliteFile: path.join(dir, 'travel.db'),
pluginsDataDir: path.join(dir, 'plugins-data'),
orm,
known: new Set([M1, M2]),
baseline: () => ({ through: M2, migrations: [M1, M2], statements: ['create table "baseline" ("id" bigint)'] }),
log,
...overrides,
};
}
beforeEach(() => {
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-move-'));
sqlite(path.join(dir, 'travel.db'), SOURCE_SQL);
pg = new FakePostgres(SCHEMA);
orm = fakeMoveOrm(pg, { known: [M1, M2] });
});
afterEach(() => {
fs.rmSync(dir, { recursive: true, force: true });
vi.restoreAllMocks();
});
describe('moveSqliteToPostgres', () => {
it('MOVE-001: fills every table in foreign key order, sets the self-references afterwards and verifies the copy', async () => {
const summary = await moveSqliteToPostgres(options());
expect(summary.tables.map((table) => [table.label, table.rows])).toEqual([
['users', 2],
['trips', 2],
['messages', 2],
]);
expect(pg.rowsOf('"users"')).toEqual([
['1', 'Ada'],
['2', 'Grace'],
]);
expect(pg.rowsOf('"trips"')).toEqual([
['1', '1', 'Lisbon', '{"b":1,"a":2}', Buffer.from([0, 255]), 1.5],
['2', '2', 'Porto', null, null, null],
]);
// The reply was inserted without its reference, then pointed at the message it answers.
expect(pg.rowsOf('"messages"')).toEqual([
['1', '1', '2', 'I do'],
['2', '1', null, 'Who books the tram?'],
]);
const statements = pg.statements.join('\n');
expect(statements).toMatch(/truncate table "users", "trips", "messages"/);
expect(statements).toMatch(/update "messages" set "reply_to" = \$1 where "id" = \$2/);
expect(statements).not.toContain(CLEAR_SCHEMA_SQL);
expect(summary.warnings).toEqual([]);
});
it('MOVE-002: moves every sequence past the copied ids, and past ids AUTOINCREMENT handed out before', async () => {
const summary = await moveSqliteToPostgres(options());
expect(summary.sequences).toBe(3);
expect(pg.sequences.get('public.users_id_seq')).toEqual({ value: '3', called: true });
expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '2', called: true });
expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '2', called: true });
});
it('MOVE-003: an empty table starts its sequence at 1', async () => {
sqlite(path.join(dir, 'travel.db'), 'delete from messages;');
await moveSqliteToPostgres(options());
expect(pg.sequences.get('public.messages_id_seq')).toEqual({ value: '1', called: false });
});
it('MOVE-004: a copy that does not read back as sent is refused and the schema dropped again', async () => {
pg.tamper.set('"trips"', (rows) => rows.map((row) => row.map((value) => (value === 'Porto' ? 'Braga' : value))));
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
/does not match the SQLite file; nothing was written:\n {2}- trips: the rows Postgres holds differ/,
);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-005: a row count that differs is named with all three counts', async () => {
pg.tamper.set('"users"', (rows) => rows.slice(1));
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
'users: the SQLite file holds 2 rows, 2 were sent, Postgres holds 1',
);
});
it('MOVE-006: values Postgres cannot hold are listed by row and column, and nothing is inserted', async () => {
sqlite(path.join(dir, 'travel.db'), "update users set id = id; update trips set score = 'high' where id = 2;");
const failure = moveSqliteToPostgres(options());
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
/1 value\(s\) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n {2}- trips row id=2, column score: the text "high" is not a number/,
);
expect(pg.statements.some((sql) => sql.startsWith('insert into "messages"'))).toBe(false);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-007: a statement Postgres refuses is named with its table, detail and constraint', async () => {
pg.failOn = /^insert into "trips"/;
await expect(moveSqliteToPostgres(options())).rejects.toThrow(
'Postgres refused rows of trips: refused by the fake; constraint fake_key; Key (id)=(1)',
);
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
});
it('MOVE-008: when the schema cannot be dropped after a failure, the error says the database needs dropping', async () => {
pg.failOn = /^(insert into "trips"|DO \$clear\$)/;
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/could not be dropped .* Drop the Postgres database/);
});
it('MOVE-009: refuses a target that holds a table or a plugin schema, before it writes anything', async () => {
pg.existing = ['customers'];
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/The Postgres database is not empty \(customers\)/);
pg.existing = [];
pg.pluginSchemas = ['trek_plugin_notes'];
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/schema trek_plugin_notes/);
expect(pg.statements.some((sql) => sql.startsWith('create') || sql.startsWith('insert'))).toBe(false);
});
it('MOVE-010: refuses a SQLite file at another migration level, one with orphaned rows, and one it cannot read', async () => {
await expect(
moveSqliteToPostgres(options({ known: new Set([M1, M2, 'Migration20200101000200_third']) })),
).rejects.toThrow(/1 migration\(s\) behind this TREK version\. Start this TREK version on it once/);
sqlite(
path.join(dir, 'travel.db'),
"pragma foreign_keys = off; insert into trips (id, user_id, title) values (9, 77, 'Nowhere');",
);
await expect(moveSqliteToPostgres(options())).rejects.toThrow(/trips row 9 refers to a missing row of users/);
await expect(moveSqliteToPostgres(options({ sqliteFile: path.join(dir, 'missing.db') }))).rejects.toThrow(
/Cannot read the SQLite file .*missing\.db: There is no SQLite file/,
);
});
it('MOVE-011: a plugin database goes into its schema with its migration record, ids and indexes', async () => {
const pluginDir = path.join(dir, 'plugins-data', 'move-notes');
fs.mkdirSync(pluginDir, { recursive: true });
fs.mkdirSync(path.join(dir, 'plugins-data', 'Not_An_Id'), { recursive: true });
sqlite(path.join(dir, 'plugins-data', 'Not_An_Id', 'plugin.db'), 'create table t (a);');
sqlite(
path.join(pluginDir, 'plugin.db'),
`create table _plugin_migrations (id text primary key, applied_at integer);
insert into _plugin_migrations values ('001', 1700000000000);
create table notes (id integer primary key autoincrement, body text not null default 'x', score real, raw blob,
added text default current_timestamp, odd text default (lower('A')));
create unique index notes_body on notes (body);
create index notes_hot on notes (score) where score > 1;
create view recent as select * from notes;
insert into notes (body, score, raw) values ('a', 1.5, x'01'), ('b', null, null), ('c', 3, null);
delete from notes where body = 'c';`,
);
const summary = await moveSqliteToPostgres(options());
expect(summary.plugins).toEqual([
{
pluginId: 'move-notes',
schema: 'trek_plugin_move_notes',
tables: [
{ label: '_plugin_migrations', rows: 1, checksum: expect.any(String) },
{ label: 'notes', rows: 2, checksum: expect.any(String) },
],
notCarried: [
'the view recent',
expect.stringMatching(/^the default \(?lower\('A'\)\)? of notes\.odd$/),
'the index notes_hot on notes (partial or over an expression)',
],
},
]);
expect(summary.warnings).toEqual([
expect.stringMatching(/Not_An_Id.*plugin\.db is in a directory whose name is no plugin id/),
]);
const statements = pg.statements.join('\n');
expect(statements).toContain('create schema "trek_plugin_move_notes"');
expect(statements).toContain(
'create table "trek_plugin_move_notes"."notes" ("id" bigint generated by default as identity, ' +
'"body" text not null default \'x\', "score" double precision, "raw" bytea, "added" text default ' +
'to_char((CURRENT_TIMESTAMP AT TIME ZONE \'UTC\'), \'YYYY-MM-DD HH24:MI:SS\'), "odd" text, primary key ("id"))',
);
expect(statements).toContain('create unique index on "trek_plugin_move_notes"."notes" ("body")');
expect(pg.rowsOf('"trek_plugin_move_notes"."_plugin_migrations"')).toEqual([['001', '1700000000000']]);
expect(pg.sequences.get('"trek_plugin_move_notes"."notes"_seq')).toEqual({ value: '3', called: true });
expect(renderSummary(summary, 'postgres://db/trek')).toContain(
'Plugin move-notes: 2 table(s), 3 rows, into schema trek_plugin_move_notes.',
);
});
it("MOVE-012: a plugin database without a migration record gets the store's empty one", async () => {
const pluginDir = path.join(dir, 'plugins-data', 'bare-notes');
fs.mkdirSync(pluginDir, { recursive: true });
sqlite(path.join(pluginDir, 'plugin.db'), 'create table kv (k text primary key, v);');
await moveSqliteToPostgres(options());
expect(pg.statements).toContain(
'create table "trek_plugin_bare_notes"._plugin_migrations (id text primary key, applied_at bigint)',
);
expect(findPluginDataFiles(path.join(dir, 'nowhere'))).toEqual({ files: [], skipped: [] });
});
it('MOVE-013: copies an integer past 2^53 exactly and warns that TREK cannot read it back on Postgres', async () => {
sqlite(path.join(dir, 'travel.db'), 'update trips set id = 9007199254740993 where id = 2;');
const summary = await moveSqliteToPostgres(options());
expect(pg.rowsOf('"trips"').map((row) => row[0])).toEqual(['1', '9007199254740993']);
expect(pg.sequences.get('public.trips_id_seq')).toEqual({ value: '9007199254740993', called: true });
expect(summary.warnings).toEqual(['trips holds 1 integer(s) beyond 2^53, which TREK refuses to read on Postgres.']);
});
});
describe('migrationLevelProblem', () => {
it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => {
const known = new Set([M1, M2]);
expect(migrationLevelProblem([M1, M2], known)).toBeNull();
expect(migrationLevelProblem(null, known)).toMatch(/no TREK migration history/);
expect(migrationLevelProblem([], known)).toMatch(/no TREK migration history/);
expect(migrationLevelProblem([M1, M2, 'Migration20990101000000_future'], known)).toMatch(
/a newer TREK migrated the SQLite file \(1 migration\(s\) this version does not know, the last Migration20990101000000_future\)/,
);
expect(migrationLevelProblem([M1], known)).toMatch(/1 migration\(s\) behind/);
});
});
describe('the pieces', () => {
it('MOVE-015: the plugin column rules, defaults and batches', () => {
const classes = (...names: string[]) => new Set(names);
expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer');
expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text');
expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes');
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text');
expect(pluginColumnKind('DOUBLE', classes())).toBe('float');
expect(pluginColumnKind('REAL', classes('text'))).toBe('text');
expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer');
expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float');
expect(pluginColumnKind('', classes('blob'))).toBe('bytes');
expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text');
expect(pluginDefault(null, 'text')).toBeNull();
expect(pluginDefault('NULL', 'text')).toBe('null');
expect(pluginDefault('(42)', 'integer')).toBe('42');
expect(pluginDefault('42', 'text')).toBe("'42'");
expect(pluginDefault('TRUE', 'integer')).toBe('1');
expect(pluginDefault('false', 'integer')).toBe('0');
expect(pluginDefault('true', 'text')).toBeNull();
expect(pluginDefault("'it''s'", 'text')).toBe("'it''s'");
expect(pluginDefault("strftime('%s','now')", 'integer')).toBe(
'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)',
);
expect(batchRows(2)).toBe(500);
expect(batchRows(200)).toBe(150);
expect(
insertSql(
{
target: '"t"',
columns: [
{ name: 'a"b', kind: 'text' },
{ name: 'c', kind: 'integer' },
],
},
2,
),
).toBe('insert into "t" ("a""b", "c") values ($1, $2), ($3, $4)');
expect(describePgError('plain')).toBe('plain');
expect(describePgError(new Error('bare'))).toBe('bare');
});
it('MOVE-016: a SQLite install holds accounts only with a user row; an unreadable file counts as holding them', () => {
const empty = path.join(dir, 'empty.db');
fs.writeFileSync(empty, '');
expect(sqliteInstallHoldsAccounts(empty)).toBe(false);
expect(sqliteInstallHoldsAccounts(path.join(dir, 'none.db'))).toBe(false);
expect(sqliteInstallHoldsAccounts(path.join(dir, 'travel.db'))).toBe(true);
expect(sqliteInstallHoldsAccounts(sqlite(path.join(dir, 'other.db'), 'create table notes (a);'))).toBe(false);
const broken = path.join(dir, 'broken.db');
fs.writeFileSync(broken, 'not a database at all, but long enough to be read as one');
expect(sqliteInstallHoldsAccounts(broken)).toBe(true);
// Reading never writes the file.
const before = fs.readFileSync(path.join(dir, 'travel.db'));
const source = SqliteSource.open(path.join(dir, 'travel.db'));
expect(source.count('users')).toBe(2);
source.close();
source.close();
expect(fs.readFileSync(path.join(dir, 'travel.db')).equals(before)).toBe(true);
});
});
describe('runMoveCommand', () => {
const env = (db: Record<string, unknown>) => () =>
({ db: { trekDbFile: path.join(dir, 'travel.db'), ...db }, plugins: { dataDir: '' } }) as unknown as AppEnv;
const postgres = env({ engine: 'postgres', url: 'postgres://trek:secret@db:5432/trek' });
const summary: MoveSummary = {
sqliteFile: 'travel.db',
tables: [{ label: 'users', rows: 2, checksum: 'x' }],
plugins: [],
sequences: 1,
warnings: ['careful'],
seconds: 0.4,
};
function io(overrides: Partial<MoveCommandIo> = {}) {
const out: string[] = [];
const err: string[] = [];
const close = vi.fn(async () => undefined);
return {
out,
err,
close,
io: {
env: postgres,
out: (line: string) => out.push(line),
err: (line: string) => err.push(line),
openOrm: async () => ({ close }) as unknown as AnyOrm,
move: vi.fn(async () => summary),
...overrides,
},
};
}
it('MOVE-017: prints the usage, refuses an unknown argument and a missing TREK_DB_URL', async () => {
const help = io();
expect(await runMoveCommand(['--help'], help.io)).toBe(0);
expect(help.out).toEqual([USAGE]);
const unknown = io();
expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1);
expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/);
const onSqlite = io({ env: env({ engine: 'sqlite' }) });
expect(await runMoveCommand([], onSqlite.io)).toBe(1);
expect(onSqlite.err[0]).toMatch(/Set TREK_DB_URL/);
});
it('MOVE-018: moves the given file, prints the summary without the password and closes the ORM', async () => {
const run = io();
expect(await runMoveCommand([`--from=${path.join(dir, 'other.db')}`], run.io)).toBe(0);
expect(run.io.move).toHaveBeenCalledWith(
expect.objectContaining({
sqliteFile: path.join(dir, 'other.db'),
pluginsDataDir: expect.stringMatching(/plugins-data$/),
}),
);
const printed = run.out.join('\n');
expect(printed).toContain('postgres://trek:***@db:5432/trek');
expect(printed).not.toContain('secret');
expect(printed).toMatch(
/1 tables, 2 rows, each table verified by row count and checksum[\s\S]*users: 2[\s\S]*Warning: careful/,
);
expect(run.close).toHaveBeenCalledWith(true);
});
it('MOVE-019: a failure prints its cause and still closes the ORM', async () => {
const failing = io({ move: vi.fn(async () => Promise.reject(new Error('connection lost'))) });
expect(await runMoveCommand([], failing.io)).toBe(1);
expect(failing.err[0]).toMatch(/The move failed; .*connection lost/s);
expect(failing.close).toHaveBeenCalled();
});
});
@@ -0,0 +1,175 @@
/**
* MOVEORD-001..005: the order the move fills the tables in, and the foreign
* keys it sets only once every row is in (src/db/engine/postgres/sqlite-move/order.ts),
* plus how the target's tables are paired with the SQLite file's (catalog.ts).
*/
import {
planTrekTables,
readTargetSchema,
type TargetSchema,
} from '../../../../../../src/db/engine/postgres/sqlite-move/catalog';
import { planCopyOrder } from '../../../../../../src/db/engine/postgres/sqlite-move/order';
import { FakePostgres } from '../../../../../helpers/fake-postgres-move';
import type { Transaction } from 'kysely';
import { describe, expect, it } from 'vitest';
describe('planCopyOrder', () => {
it('MOVEORD-001: puts every table after the tables it points at, ties by name', () => {
const plan = planCopyOrder(
['places', 'trips', 'users', 'days', 'categories'],
[
{ table: 'trips', columns: ['user_id'], references: 'users' },
{ table: 'days', columns: ['trip_id'], references: 'trips' },
{ table: 'places', columns: ['trip_id'], references: 'trips' },
{ table: 'places', columns: ['category_id'], references: 'categories' },
],
);
expect(plan.order).toEqual(['categories', 'users', 'trips', 'days', 'places']);
expect(plan.deferred.size).toBe(0);
});
it('MOVEORD-002: a table pointing at itself has those columns set after its rows are in', () => {
const plan = planCopyOrder(
['collab_messages'],
[
{ table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' },
{ table: 'collab_messages', columns: ['reply_to'], references: 'collab_messages' },
],
);
expect(plan.order).toEqual(['collab_messages']);
expect(plan.deferred.get('collab_messages')).toEqual(['reply_to']);
});
it('MOVEORD-003: a cycle is broken at the first table by name, deferring its keys into the cycle', () => {
const plan = planCopyOrder(
['b', 'a', 'c'],
[
{ table: 'a', columns: ['b_id'], references: 'b' },
{ table: 'b', columns: ['a_id'], references: 'a' },
{ table: 'c', columns: ['a_id'], references: 'a' },
],
);
expect(plan.order).toEqual(['a', 'b', 'c']);
expect([...plan.deferred]).toEqual([['a', ['b_id']]]);
});
it('MOVEORD-004: a key to a table the move does not copy is ignored', () => {
const plan = planCopyOrder(['trips'], [{ table: 'trips', columns: ['user_id'], references: 'users' }]);
expect(plan.order).toEqual(['trips']);
});
});
describe('planTrekTables', () => {
const target: TargetSchema = {
tables: [
{ name: 'mikro_orm_migrations', columns: [{ name: 'name', dataType: 'text', nullable: false }], primaryKey: [] },
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'reply_to', dataType: 'bigint', nullable: true },
],
primaryKey: ['id'],
},
{
name: 'users',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'avatar', dataType: 'bytea', nullable: true },
],
primaryKey: ['id'],
},
],
foreignKeys: [{ table: 'messages', columns: ['reply_to'], references: 'messages' }],
sequences: [{ table: 'users', column: 'id', sequence: 'public.users_id_seq' }],
};
it('MOVEORD-005: pairs the tables by name, leaves the history out and quotes every name', () => {
const plan = planTrekTables(
target,
new Map([
['mikro_orm_migrations', ['name']],
['messages', ['id', 'reply_to']],
['users', ['id', 'avatar']],
]),
);
expect(plan.problems).toEqual([]);
expect(plan.specs.map((spec) => spec.target)).toEqual(['"messages"', '"users"']);
expect(plan.specs[0]).toMatchObject({ deferred: ['reply_to'], primaryKey: ['id'], label: 'messages' });
expect(plan.specs[1].columns).toEqual([
{ name: 'id', kind: 'integer' },
{ name: 'avatar', kind: 'bytes' },
]);
expect(plan.sequences).toEqual([
{ table: 'users', column: 'id', sequence: 'public.users_id_seq', target: '"users"' },
]);
});
it('MOVEORD-006: names every difference between the two schemas instead of copying around it', () => {
const odd: TargetSchema = {
...target,
tables: [
...target.tables.filter((table) => table.name !== 'messages'),
{
name: 'messages',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'reply_to', dataType: 'bigint', nullable: false },
{ name: 'sent_at', dataType: 'timestamp with time zone', nullable: true },
],
primaryKey: ['id'],
},
{ name: 'extra', columns: [], primaryKey: [] },
],
};
const plan = planTrekTables(
odd,
new Map([
['messages', ['id', 'reply_to', 'sent_at', 'legacy']],
['users', ['id']],
['leftover', ['id']],
]),
);
expect(plan.problems).toEqual([
'the SQLite table leftover has no Postgres table',
'the Postgres table extra has no SQLite table',
'the SQLite column messages.legacy has no Postgres column',
'the Postgres column messages.sent_at has the type timestamp with time zone, which the move does not know',
'messages.reply_to refers back to a table filled later and cannot be NULL meanwhile',
'the Postgres column users.avatar has no SQLite column',
]);
});
});
describe('readTargetSchema', () => {
it('MOVEORD-007: reads the columns, keys and sequences of the current schema', async () => {
const pg = new FakePostgres([
{
name: 'trips',
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint' },
],
primaryKey: ['id'],
foreignKeys: [{ columns: ['user_id'], references: 'users' }],
serial: ['id'],
},
{ name: 'users', columns: [{ name: 'id', dataType: 'bigint', nullable: false }], primaryKey: ['id'] },
]);
const schema = await readTargetSchema(pg.asKysely() as Transaction<object>);
expect(schema.tables).toEqual([
{
name: 'trips',
primaryKey: ['id'],
columns: [
{ name: 'id', dataType: 'bigint', nullable: false },
{ name: 'user_id', dataType: 'bigint', nullable: true },
],
},
{ name: 'users', primaryKey: ['id'], columns: [{ name: 'id', dataType: 'bigint', nullable: false }] },
]);
expect(schema.foreignKeys).toEqual([{ table: 'trips', columns: ['user_id'], references: 'users' }]);
expect(schema.sequences).toEqual([{ table: 'trips', column: 'id', sequence: 'public.trips_id_seq' }]);
});
});
@@ -0,0 +1,179 @@
/**
* MOVEVAL-001..010: how a SQLite value becomes the value a Postgres column of
* the move's target takes, the text both sides are compared in, and the
* order-free table checksum (src/db/engine/postgres/sqlite-move/values.ts).
*/
import {
beyondSafeInteger,
canonicalFromPostgres,
canonicalFromSent,
columnKind,
convertValue,
stableJson,
TableChecksum,
type ColumnKind,
} from '../../../../../../src/db/engine/postgres/sqlite-move/values';
import fs from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
const ok = (value: unknown) => ({ ok: true, value });
describe('columnKind', () => {
it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => {
const sql = fs.readFileSync(path.join(__dirname, '../../../../../../assets/postgres/baseline.sql'), 'utf8');
const types = new Set(
[
...sql.matchAll(
/" (bigserial|bigint|integer|smallint|text|varchar\(\d+\)|double precision|real|jsonb|json|bytea|boolean|numeric|timestamptz|date|uuid)(?=[ ,)])/g,
),
].map((match) => match[1]),
);
const asCatalog: Record<string, string> = { bigserial: 'bigint' };
for (const type of types) {
const dataType = asCatalog[type] ?? (type.startsWith('varchar') ? 'character varying' : type);
expect(columnKind(dataType), type).not.toBeNull();
}
});
it('MOVEVAL-002: has no rule for a type it does not know, so the move refuses instead of guessing', () => {
expect(columnKind('timestamp with time zone')).toBeNull();
expect(columnKind('boolean')).toBeNull();
expect(columnKind('smallint')).toBe('integer');
expect(columnKind('real')).toBe('float');
expect(columnKind('character')).toBe('text');
expect(columnKind('json')).toBe('json');
});
});
describe('convertValue', () => {
it('MOVEVAL-003: integers keep all 64 bits and accept whole numbers in any spelling SQLite stored them in', () => {
expect(convertValue(9007199254740993n, 'integer')).toEqual(ok('9007199254740993'));
expect(convertValue(42, 'integer')).toEqual(ok('42'));
expect(convertValue(' -7 ', 'integer')).toEqual(ok('-7'));
expect(convertValue(null, 'integer')).toEqual(ok(null));
expect(convertValue(1.5, 'integer')).toMatchObject({ ok: false, reason: 'the value 1.5 is not a whole number' });
expect(convertValue('abc', 'integer')).toMatchObject({ ok: false, reason: 'the text "abc" is not a whole number' });
expect(convertValue('99999999999999999999', 'integer')).toMatchObject({
ok: false,
reason: expect.stringMatching(/does not fit in 64 bits/),
});
expect(convertValue(new Uint8Array([1]), 'integer')).toMatchObject({
ok: false,
reason: 'a 1-byte BLOB is not a whole number',
});
});
it('MOVEVAL-004: floats take numbers and numeric text, never a value a double cannot hold', () => {
expect(convertValue(1.25, 'float')).toEqual(ok(1.25));
expect(convertValue(12n, 'float')).toEqual(ok(12));
expect(convertValue('3.5e2', 'float')).toEqual(ok(350));
expect(convertValue(9007199254740993n, 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/more digits than a double holds/),
});
expect(convertValue('north', 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not a number/),
});
expect(convertValue('', 'float')).toMatchObject({ ok: false });
});
it('MOVEVAL-005: text takes strings and numbers, and refuses a NUL Postgres cannot store or a BLOB', () => {
expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa'));
expect(convertValue(7n, 'text')).toEqual(ok('7'));
expect(convertValue(2.5, 'text')).toEqual(ok('2.5'));
expect(convertValue('a\u0000b', 'text')).toMatchObject({
ok: false,
reason: expect.stringMatching(/NUL character/),
});
expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not text/),
});
const long = 'x'.repeat(60);
expect(convertValue(long, 'integer')).toMatchObject({
reason: `the text "${'x'.repeat(40)}..." is not a whole number`,
});
});
it('MOVEVAL-006: JSON must parse and hold no NUL, escaped or not', () => {
expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}'));
expect(convertValue(5n, 'json')).toEqual(ok('5'));
expect(convertValue('{broken', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not valid JSON/),
});
expect(convertValue('"\\u0000"', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/jsonb cannot store/),
});
expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not JSON text/),
});
});
it('MOVEVAL-007: bytes take a BLOB, or the UTF-8 of text, and refuse a number', () => {
expect(convertValue(new Uint8Array([0, 255]), 'bytes')).toEqual(ok(Buffer.from([0, 255])));
expect(convertValue('hé', 'bytes')).toEqual(ok(Buffer.from('hé', 'utf8')));
expect(convertValue(3n, 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not a BLOB/) });
});
it('MOVEVAL-008: flags an integer past 2^53, which the Postgres driver refuses to read', () => {
expect(beyondSafeInteger('9007199254740993')).toBe(true);
expect(beyondSafeInteger('-9007199254740993')).toBe(true);
expect(beyondSafeInteger('9007199254740991')).toBe(false);
expect(beyondSafeInteger(12)).toBe(false);
expect(beyondSafeInteger('text')).toBe(false);
});
});
describe('the compared text', () => {
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSent>[0], string]> = [
['integer', '-12', '-12'],
['float', 0.30000000000000004, '0.30000000000000004'],
['float', 1e21, '1e+21'],
['text', 'São Paulo', 'São Paulo'],
['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'],
['bytes', Buffer.from([0, 171]), '\\x00ab'],
];
it.each(cases)('MOVEVAL-009: %s values compare equal as sent and as Postgres prints them', (kind, sent, printed) => {
expect(canonicalFromSent(sent, kind)).toBe(canonicalFromPostgres(printed, kind));
expect(canonicalFromSent(null, kind)).toBeNull();
expect(canonicalFromPostgres(null, kind)).toBeNull();
});
it('MOVEVAL-009b: text sent for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
expect(canonicalFromSent('ab', 'bytes')).toBe('6162');
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSent(1e20, 'float'));
expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain');
expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}');
});
it('MOVEVAL-010: the checksum ignores row order and catches a changed, missing or doubled row', () => {
const sum = (rows: Array<Array<string | null>>) => {
const checksum = new TableChecksum();
for (const row of rows) checksum.add(row);
return checksum.hex();
};
const rows = [
['1', 'a'],
['2', null],
];
expect(sum(rows)).toBe(sum([...rows].reverse()));
expect(sum(rows)).toHaveLength(64);
expect(sum(rows)).not.toBe(
sum([
['1', 'a'],
['2', 'b'],
]),
);
expect(sum(rows)).not.toBe(sum([['1', 'a']]));
expect(sum(rows)).not.toBe(sum([...rows, ['2', null]]));
// A NULL and the text "null" are different values.
expect(sum([['null']])).not.toBe(sum([[null]]));
});
});