From 621b2f17dfe60eaf1224055b764e62f0a3e96e90 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 23:28:07 +0100 Subject: [PATCH] test(db): run the move's NUL and plugin type rules on a real Postgres The plugin in the move suite gains a BLOB column holding text and an untyped column holding numbers, and the plugin store must read both back as they were stored. A new suite moves a file whose plugin and own rows hold NUL: the plugin's refuses before anything is written, TREK's refuses until --drop-nul, a username that dropping the NUL makes duplicate is named with users_username_unique, and a move without a collision lists the value it changed and stores it without the NUL. --- server/tests/postgres/sqlite-move-nul.test.ts | 159 ++++++++++++++++++ server/tests/postgres/sqlite-move.test.ts | 13 +- 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 server/tests/postgres/sqlite-move-nul.test.ts diff --git a/server/tests/postgres/sqlite-move-nul.test.ts b/server/tests/postgres/sqlite-move-nul.test.ts new file mode 100644 index 000000000..833cdf092 --- /dev/null +++ b/server/tests/postgres/sqlite-move-nul.test.ts @@ -0,0 +1,159 @@ +/** + * PGMOVENUL-001..004: the move into Postgres (`src/db/engine/postgres/sqlite-move/`) + * changes nothing silently, on the lane's server. A plugin value holding a NUL + * refuses the move before anything is written; a NUL in TREK's own text + * refuses it unless `dropNul` (`--drop-nul`) is given; a unique key that + * dropping a NUL breaks is named as that, with the constraint Postgres + * reports; and a move with `dropNul` lists every value it changed and stores + * the value TREK would have written. + */ +import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; +import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; +import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; +import { knownMigrationNames } from '../../src/db/known-migrations'; +import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm'; +import { createBoundSqliteDriver } from '../../src/db/orm-driver'; +import { makeUser } from '../helpers/factories/users'; +import { withAdmin } from './databases'; +import { MikroORM, type Options } from '@mikro-orm/core'; + +import Database from 'better-sqlite3'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +let dir: string; +let handle: Database.Database; +let sqliteOrm: AnyOrm; +let movedId: number; + +/** An ORM on its own SQLite file, migrated the way the SQLite engine brings a file to head. */ +async function migratedSqlite(file: string): Promise { + handle = new Database(file); + handle.pragma('foreign_keys = ON'); + const config = buildOrmConfig(new SqliteEngine()); + const options: Options = { + ...config, + driver: createBoundSqliteDriver(() => handle), + dbName: file, + migrations: { ...config.migrations, silent: true }, + }; + const orm = await MikroORM.init(options); + await runSchemaBootstrap(orm, { snapshot: false }); + return orm; +} + +/** Runs the move into the worker's database, which the lane hands over empty of tables until a move commits. */ +async function move(dropNul = false): Promise { + const config = buildOrmConfig(resolveEngine()); + const orm = await MikroORM.init({ + ...config, + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + migrations: { ...config.migrations, silent: true }, + }); + try { + return await moveSqliteToPostgres({ + sqliteFile: path.join(dir, 'travel.db'), + pluginsDataDir: path.join(dir, 'plugins-data'), + orm, + known: knownMigrationNames(), + baseline: readPostgresBaseline, + log: () => undefined, + dropNul, + }); + } finally { + await orm.close(true); + } +} + +/** The tables of the worker database's current schema. */ +async function tablesInTarget(): Promise { + return withAdmin(process.env.TREK_DB_URL as string, async (run) => { + const [row] = await run<{ count: string | number }>( + 'select count(*) as count from information_schema.tables where table_schema = current_schema()', + ); + return Number(row.count); + }); +} + +/** The worker's database, emptied: the lane hands each file a migrated copy, and the move fills an empty one only. */ +async function emptyTarget(): Promise { + const name = new URL(process.env.TREK_DB_URL as string).pathname.slice(1); + await withAdmin(process.env.TREK_TEST_DB_URL as string, async (run) => { + await run(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); + await run(`CREATE DATABASE "${name}"`); + }); +} + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-pg-move-nul-')); + sqliteOrm = await migratedSqlite(path.join(dir, 'travel.db')); + await makeUser(sqliteOrm, { username: 'movera' }); + const { user } = await makeUser(sqliteOrm, { username: 'mover-two' }); + movedId = user.id; + // A value an older TREK stored with a NUL, which the ORM would no longer write. + handle.exec(`update users set username = 'mover' || char(0) || 'a' where id = ${movedId}`); + fs.mkdirSync(path.join(dir, 'plugins-data', 'nul-notes'), { recursive: true }); + const plugin = new Database(path.join(dir, 'plugins-data', 'nul-notes', 'plugin.db')); + plugin.exec( + 'create table notes (k text primary key, v text);' + + "insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two');", + ); + plugin.close(); + await emptyTarget(); +}, 180_000); + +afterAll(async () => { + await sqliteOrm?.close(true); + if (handle?.open) handle.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('the move changes nothing silently', () => { + it("PGMOVENUL-001: a plugin's NUL refuses the move before anything is written", async () => { + const failure = move(true); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + /1 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/, + ); + expect(await tablesInTarget()).toBe(0); + fs.rmSync(path.join(dir, 'plugins-data'), { recursive: true, force: true }); + }); + + it("PGMOVENUL-002: a NUL in TREK's own text refuses the move by default and names --drop-nul", async () => { + const failure = move(); + await expect(failure).rejects.toBeInstanceOf(MoveRefusal); + await expect(failure).rejects.toThrow( + new RegExp( + `users row id=${movedId}, column username: the text "mover\\\\u0000a" holds a NUL character \\(U\\+0000\\)` + + '[\\s\\S]*Run the move again with --drop-nul', + ), + ); + expect(await tablesInTarget()).toBe(0); + }); + + it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks is named, with the constraint', async () => { + await expect(move(true)).rejects.toThrow( + new RegExp( + 'Postgres refused rows of users: .*constraint users_username_unique[\\s\\S]*' + + 'The key is broken because --drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its ' + + `column\\(s\\) username,[\\s\\S]* - users row id=${movedId}, column username\\n`, + ), + ); + expect(await tablesInTarget()).toBe(0); + }); + + it('PGMOVENUL-004: with --drop-nul and no collision, each changed value is listed and stored without the NUL', async () => { + handle.exec(`update users set username = 'mover' || char(0) || 'b' where id = ${movedId}`); + const summary = await move(true); + expect(summary.nulDropped).toEqual([`users row id=${movedId}, column username`]); + const stored = await withAdmin(process.env.TREK_DB_URL as string, (run) => + run<{ username: string }>('select username from users where id = ?', [movedId]), + ); + expect(stored).toEqual([{ username: 'moverb' }]); + }); +}); diff --git a/server/tests/postgres/sqlite-move.test.ts b/server/tests/postgres/sqlite-move.test.ts index 62da5673a..49151bc4c 100644 --- a/server/tests/postgres/sqlite-move.test.ts +++ b/server/tests/postgres/sqlite-move.test.ts @@ -126,7 +126,10 @@ function seedPluginData(root: string): void { "create table notes (id integer primary key autoincrement, body text not null default '', score real, raw blob);" + 'create unique index notes_body on notes (body);' + "insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" + - "delete from notes where body = 'gone';", + "delete from notes where body = 'gone';" + + // Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them. + 'create table docs (id integer primary key, payload blob, n);' + + `insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5);`, ); plugin.close(); } @@ -209,6 +212,7 @@ describe('moving a SQLite install to Postgres', () => { schema: 'trek_plugin_move_notes', tables: [ expect.objectContaining({ label: '_plugin_migrations', rows: 1 }), + expect.objectContaining({ label: 'docs', rows: 2 }), expect.objectContaining({ label: 'notes', rows: 2 }), ], }), @@ -264,7 +268,7 @@ describe('moving a SQLite install to Postgres', () => { expect(reply?.reply_to).toBe(question?.id); }); - it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows and its migration record', async () => { + it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows, typed as stored, and its migration record', async () => { const { user } = await makeUser(pgOrm); const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id)); expect(user.id).toBeGreaterThan(highest); @@ -281,6 +285,11 @@ describe('moving a SQLite install to Postgres', () => { { id: 1, body: 'first', score: 1.5 }, { id: 2, body: 'second', score: null }, ]); + // Text from a BLOB column is still a string, and numbers from a column without a type still numbers. + expect(await store.query('select id, payload, n from docs order by id')).toEqual([ + { id: 1, payload: '{"a":1}', n: 5 }, + { id: 2, payload: 'plain', n: 2.5 }, + ]); // The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out. expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false }); await store.exec("insert into notes (body) values ('third')");