mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 19:40:34 +00:00
test(db): run the move's NUL and plugin type rules on a real Postgres
The plugin in the move suite gains a BLOB column holding text and an untyped column holding numbers, and the plugin store must read both back as they were stored. A new suite moves a file whose plugin and own rows hold NUL: the plugin's refuses before anything is written, TREK's refuses until --drop-nul, a username that dropping the NUL makes duplicate is named with users_username_unique, and a move without a collision lists the value it changed and stores it without the NUL.
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
/**
|
||||
* PGMOVENUL-001..004: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
||||
* changes nothing silently, on the lane's server. A plugin value holding a NUL
|
||||
* refuses the move before anything is written; a NUL in TREK's own text
|
||||
* refuses it unless `dropNul` (`--drop-nul`) is given; a unique key that
|
||||
* dropping a NUL breaks is named as that, with the constraint Postgres
|
||||
* reports; and a move with `dropNul` lists every value it changed and stores
|
||||
* the value TREK would have written.
|
||||
*/
|
||||
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
||||
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
||||
import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
||||
import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine';
|
||||
import { knownMigrationNames } from '../../src/db/known-migrations';
|
||||
import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm';
|
||||
import { createBoundSqliteDriver } from '../../src/db/orm-driver';
|
||||
import { makeUser } from '../helpers/factories/users';
|
||||
import { withAdmin } from './databases';
|
||||
import { MikroORM, type Options } from '@mikro-orm/core';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
let dir: string;
|
||||
let handle: Database.Database;
|
||||
let sqliteOrm: AnyOrm;
|
||||
let movedId: number;
|
||||
|
||||
/** An ORM on its own SQLite file, migrated the way the SQLite engine brings a file to head. */
|
||||
async function migratedSqlite(file: string): Promise<AnyOrm> {
|
||||
handle = new Database(file);
|
||||
handle.pragma('foreign_keys = ON');
|
||||
const config = buildOrmConfig(new SqliteEngine());
|
||||
const options: Options = {
|
||||
...config,
|
||||
driver: createBoundSqliteDriver(() => handle),
|
||||
dbName: file,
|
||||
migrations: { ...config.migrations, silent: true },
|
||||
};
|
||||
const orm = await MikroORM.init(options);
|
||||
await runSchemaBootstrap(orm, { snapshot: false });
|
||||
return orm;
|
||||
}
|
||||
|
||||
/** Runs the move into the worker's database, which the lane hands over empty of tables until a move commits. */
|
||||
async function move(dropNul = false): Promise<MoveSummary> {
|
||||
const config = buildOrmConfig(resolveEngine());
|
||||
const orm = await MikroORM.init({
|
||||
...config,
|
||||
entities: [],
|
||||
entitiesTs: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
migrations: { ...config.migrations, silent: true },
|
||||
});
|
||||
try {
|
||||
return await moveSqliteToPostgres({
|
||||
sqliteFile: path.join(dir, 'travel.db'),
|
||||
pluginsDataDir: path.join(dir, 'plugins-data'),
|
||||
orm,
|
||||
known: knownMigrationNames(),
|
||||
baseline: readPostgresBaseline,
|
||||
log: () => undefined,
|
||||
dropNul,
|
||||
});
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}
|
||||
|
||||
/** The tables of the worker database's current schema. */
|
||||
async function tablesInTarget(): Promise<number> {
|
||||
return withAdmin(process.env.TREK_DB_URL as string, async (run) => {
|
||||
const [row] = await run<{ count: string | number }>(
|
||||
'select count(*) as count from information_schema.tables where table_schema = current_schema()',
|
||||
);
|
||||
return Number(row.count);
|
||||
});
|
||||
}
|
||||
|
||||
/** The worker's database, emptied: the lane hands each file a migrated copy, and the move fills an empty one only. */
|
||||
async function emptyTarget(): Promise<void> {
|
||||
const name = new URL(process.env.TREK_DB_URL as string).pathname.slice(1);
|
||||
await withAdmin(process.env.TREK_TEST_DB_URL as string, async (run) => {
|
||||
await run(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
||||
await run(`CREATE DATABASE "${name}"`);
|
||||
});
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-pg-move-nul-'));
|
||||
sqliteOrm = await migratedSqlite(path.join(dir, 'travel.db'));
|
||||
await makeUser(sqliteOrm, { username: 'movera' });
|
||||
const { user } = await makeUser(sqliteOrm, { username: 'mover-two' });
|
||||
movedId = user.id;
|
||||
// A value an older TREK stored with a NUL, which the ORM would no longer write.
|
||||
handle.exec(`update users set username = 'mover' || char(0) || 'a' where id = ${movedId}`);
|
||||
fs.mkdirSync(path.join(dir, 'plugins-data', 'nul-notes'), { recursive: true });
|
||||
const plugin = new Database(path.join(dir, 'plugins-data', 'nul-notes', 'plugin.db'));
|
||||
plugin.exec(
|
||||
'create table notes (k text primary key, v text);' +
|
||||
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two');",
|
||||
);
|
||||
plugin.close();
|
||||
await emptyTarget();
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await sqliteOrm?.close(true);
|
||||
if (handle?.open) handle.close();
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('the move changes nothing silently', () => {
|
||||
it("PGMOVENUL-001: a plugin's NUL refuses the move before anything is written", async () => {
|
||||
const failure = move(true);
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
/1 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
fs.rmSync(path.join(dir, 'plugins-data'), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("PGMOVENUL-002: a NUL in TREK's own text refuses the move by default and names --drop-nul", async () => {
|
||||
const failure = move();
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
await expect(failure).rejects.toThrow(
|
||||
new RegExp(
|
||||
`users row id=${movedId}, column username: the text "mover\\\\u0000a" holds a NUL character \\(U\\+0000\\)` +
|
||||
'[\\s\\S]*Run the move again with --drop-nul',
|
||||
),
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks is named, with the constraint', async () => {
|
||||
await expect(move(true)).rejects.toThrow(
|
||||
new RegExp(
|
||||
'Postgres refused rows of users: .*constraint users_username_unique[\\s\\S]*' +
|
||||
'The key is broken because --drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its ' +
|
||||
`column\\(s\\) username,[\\s\\S]* - users row id=${movedId}, column username\\n`,
|
||||
),
|
||||
);
|
||||
expect(await tablesInTarget()).toBe(0);
|
||||
});
|
||||
|
||||
it('PGMOVENUL-004: with --drop-nul and no collision, each changed value is listed and stored without the NUL', async () => {
|
||||
handle.exec(`update users set username = 'mover' || char(0) || 'b' where id = ${movedId}`);
|
||||
const summary = await move(true);
|
||||
expect(summary.nulDropped).toEqual([`users row id=${movedId}, column username`]);
|
||||
const stored = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
||||
run<{ username: string }>('select username from users where id = ?', [movedId]),
|
||||
);
|
||||
expect(stored).toEqual([{ username: 'moverb' }]);
|
||||
});
|
||||
});
|
||||
@@ -126,7 +126,10 @@ function seedPluginData(root: string): void {
|
||||
"create table notes (id integer primary key autoincrement, body text not null default '', score real, raw blob);" +
|
||||
'create unique index notes_body on notes (body);' +
|
||||
"insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" +
|
||||
"delete from notes where body = 'gone';",
|
||||
"delete from notes where body = 'gone';" +
|
||||
// Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them.
|
||||
'create table docs (id integer primary key, payload blob, n);' +
|
||||
`insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5);`,
|
||||
);
|
||||
plugin.close();
|
||||
}
|
||||
@@ -209,6 +212,7 @@ describe('moving a SQLite install to Postgres', () => {
|
||||
schema: 'trek_plugin_move_notes',
|
||||
tables: [
|
||||
expect.objectContaining({ label: '_plugin_migrations', rows: 1 }),
|
||||
expect.objectContaining({ label: 'docs', rows: 2 }),
|
||||
expect.objectContaining({ label: 'notes', rows: 2 }),
|
||||
],
|
||||
}),
|
||||
@@ -264,7 +268,7 @@ describe('moving a SQLite install to Postgres', () => {
|
||||
expect(reply?.reply_to).toBe(question?.id);
|
||||
});
|
||||
|
||||
it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows and its migration record', async () => {
|
||||
it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows, typed as stored, and its migration record', async () => {
|
||||
const { user } = await makeUser(pgOrm);
|
||||
const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id));
|
||||
expect(user.id).toBeGreaterThan(highest);
|
||||
@@ -281,6 +285,11 @@ describe('moving a SQLite install to Postgres', () => {
|
||||
{ id: 1, body: 'first', score: 1.5 },
|
||||
{ id: 2, body: 'second', score: null },
|
||||
]);
|
||||
// Text from a BLOB column is still a string, and numbers from a column without a type still numbers.
|
||||
expect(await store.query('select id, payload, n from docs order by id')).toEqual([
|
||||
{ id: 1, payload: '{"a":1}', n: 5 },
|
||||
{ id: 2, payload: 'plain', n: 2.5 },
|
||||
]);
|
||||
// The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out.
|
||||
expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false });
|
||||
await store.exec("insert into notes (body) values ('third')");
|
||||
|
||||
Reference in New Issue
Block a user