mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
69e791206e
The move read unique indexes by joining indkey to pg_attribute. An expression key such as lower(email) has attnum 0 there, so idx_users_email_lower was missing from the plan. The catalog query now reads each index's key as Postgres prints it, whether any part of it is an expression, and the columns it reads: the plain key columns and the Var nodes of indexprs, without INCLUDE columns or the predicate. A broken unique key no longer names --drop-nul as its cause. The refusal names the key and the shared value from Postgres's detail. It explains that the SQLite file held the two rows apart, and the hint about SQLite's lower() appears only for an expression key. Values --drop-nul changed in the key's columns are listed as rows to check first, and only when the duplicate value holds their text. An unrelated value that lost a NUL is no longer named. A plain key over a quoted column is named by its column, not as an expression. A row that breaks a constraint (SQLSTATE class 23) is now a MoveRefusal that says what to correct, in the SQLite file or in a plugin's plugin.db. Before, it was a bare error with a stack. A value Postgres refuses by its type (class 22) stays a failure: it names no row, and values.ts should have refused it first, so the message asks for a bug report. MoveRefusal and the remedy text move to refusal.ts so copy.ts can use them without an import cycle. The usage text of --drop-nul and the wiki's move section describe NUL in JSON, the key listing and these refusals.
325 lines
15 KiB
TypeScript
325 lines
15 KiB
TypeScript
/**
|
|
* PGMOVENUL-001..010: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
|
|
* changes nothing silently, on the lane's server. A plugin value holding a NUL
|
|
* or bytes that are not UTF-8 refuses the move before anything is written; a
|
|
* NUL in TREK's own text refuses it unless `dropNul` (`--drop-nul`) is given;
|
|
* a unique key that dropping a NUL may have broken lists that value, with the
|
|
* constraint Postgres reports; a move with `dropNul` lists every value it
|
|
* changed and stores the value TREK would have written, JSON with nothing
|
|
* else changed; and TREK's own text that is not UTF-8 is refused either way.
|
|
* A key over an expression (`lower(email)`) that a dropped NUL breaks lists
|
|
* only the dropped values of the columns its expression reads, and a raw NUL
|
|
* in JSON text is refused as a NUL and dropped like an escaped one. Two rows
|
|
* a Postgres key holds equal while the SQLite file held them apart are a
|
|
* refusal that names the key and says what to change, and a value that lost
|
|
* a NUL in a column of that key is listed only when the duplicate value holds
|
|
* it. The catalog names a key over an expression as one, and a plain key over
|
|
* a quoted column by that column.
|
|
*/
|
|
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
|
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
|
import { readTargetSchema } from '../../src/db/engine/postgres/sqlite-move/catalog';
|
|
import { inOrder, run } from '../../src/db/engine/postgres/sqlite-move/copy';
|
|
import { moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
|
import { MoveRefusal } from '../../src/db/engine/postgres/sqlite-move/refusal';
|
|
import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine';
|
|
import { knownMigrationNames } from '../../src/db/known-migrations';
|
|
import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm';
|
|
import { createBoundSqliteDriver } from '../../src/db/orm-driver';
|
|
import { makeUser } from '../helpers/factories/users';
|
|
import { withAdmin } from './databases';
|
|
import { MikroORM, type Options } from '@mikro-orm/core';
|
|
import type { AbstractSqlConnection } from '@mikro-orm/sql';
|
|
|
|
import Database from 'better-sqlite3';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
let dir: string;
|
|
let handle: Database.Database;
|
|
let sqliteOrm: AnyOrm;
|
|
let movedId: number;
|
|
|
|
/** An ORM on its own SQLite file, migrated the way the SQLite engine brings a file to head. */
|
|
async function migratedSqlite(file: string): Promise<AnyOrm> {
|
|
handle = new Database(file);
|
|
handle.pragma('foreign_keys = ON');
|
|
const config = buildOrmConfig(new SqliteEngine());
|
|
const options: Options = {
|
|
...config,
|
|
driver: createBoundSqliteDriver(() => handle),
|
|
dbName: file,
|
|
migrations: { ...config.migrations, silent: true },
|
|
};
|
|
const orm = await MikroORM.init(options);
|
|
await runSchemaBootstrap(orm, { snapshot: false });
|
|
return orm;
|
|
}
|
|
|
|
/** Runs the move into the worker's database, which the lane hands over empty of tables until a move commits. */
|
|
async function move(dropNul = false): Promise<MoveSummary> {
|
|
const config = buildOrmConfig(resolveEngine());
|
|
const orm = await MikroORM.init({
|
|
...config,
|
|
entities: [],
|
|
entitiesTs: [],
|
|
discovery: { warnWhenNoEntities: false },
|
|
migrations: { ...config.migrations, silent: true },
|
|
});
|
|
try {
|
|
return await moveSqliteToPostgres({
|
|
sqliteFile: path.join(dir, 'travel.db'),
|
|
pluginsDataDir: path.join(dir, 'plugins-data'),
|
|
orm,
|
|
known: knownMigrationNames(),
|
|
baseline: readPostgresBaseline,
|
|
log: () => undefined,
|
|
dropNul,
|
|
});
|
|
} finally {
|
|
await orm.close(true);
|
|
}
|
|
}
|
|
|
|
/** The tables of the worker database's current schema. */
|
|
async function tablesInTarget(): Promise<number> {
|
|
return withAdmin(process.env.TREK_DB_URL as string, async (run) => {
|
|
const [row] = await run<{ count: string | number }>(
|
|
'select count(*) as count from information_schema.tables where table_schema = current_schema()',
|
|
);
|
|
return Number(row.count);
|
|
});
|
|
}
|
|
|
|
/** The worker's database, emptied: the lane hands each file a migrated copy, and the move fills an empty one only. */
|
|
async function emptyTarget(): Promise<void> {
|
|
const name = new URL(process.env.TREK_DB_URL as string).pathname.slice(1);
|
|
await withAdmin(process.env.TREK_TEST_DB_URL as string, async (run) => {
|
|
await run(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
|
await run(`CREATE DATABASE "${name}"`);
|
|
});
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-pg-move-nul-'));
|
|
sqliteOrm = await migratedSqlite(path.join(dir, 'travel.db'));
|
|
await makeUser(sqliteOrm, { username: 'movera' });
|
|
const { user } = await makeUser(sqliteOrm, { username: 'mover-two' });
|
|
movedId = user.id;
|
|
// A value an older TREK stored with a NUL, which the ORM would no longer write.
|
|
handle.exec(`update users set username = 'mover' || char(0) || 'a' where id = ${movedId}`);
|
|
fs.mkdirSync(path.join(dir, 'plugins-data', 'nul-notes'), { recursive: true });
|
|
const plugin = new Database(path.join(dir, 'plugins-data', 'nul-notes', 'plugin.db'));
|
|
plugin.exec(
|
|
'create table notes (k text primary key, v text);' +
|
|
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two'), ('bytes', cast(x'ff41' as text));",
|
|
);
|
|
plugin.close();
|
|
await emptyTarget();
|
|
}, 180_000);
|
|
|
|
afterAll(async () => {
|
|
await sqliteOrm?.close(true);
|
|
if (handle?.open) handle.close();
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
describe('the move changes nothing silently', () => {
|
|
it("PGMOVENUL-001: a plugin's NUL or bytes that are not UTF-8 refuse the move before anything is written", async () => {
|
|
const failure = move(true);
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
/2 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
|
|
);
|
|
await expect(failure).rejects.toThrow(
|
|
'plugin nul-notes: notes row k=bytes, column v: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
fs.rmSync(path.join(dir, 'plugins-data'), { recursive: true, force: true });
|
|
});
|
|
|
|
it("PGMOVENUL-002: a NUL in TREK's own text refuses the move by default and names --drop-nul", async () => {
|
|
const failure = move();
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
new RegExp(
|
|
`users row id=${movedId}, column username: the text "mover\\\\u0000a" holds a NUL character \\(U\\+0000\\)` +
|
|
'[\\s\\S]*Run the move again with --drop-nul',
|
|
),
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
});
|
|
|
|
it('PGMOVENUL-003: with --drop-nul, a unique key the dropped NUL breaks lists that value, with the constraint', async () => {
|
|
await expect(move(true)).rejects.toThrow(
|
|
new RegExp(
|
|
'Postgres refused rows of users: .*constraint users_username_unique[\\s\\S]*' +
|
|
'--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) username ' +
|
|
`that may hold it;[\\s\\S]* - users row id=${movedId}, column username\\n`,
|
|
),
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
});
|
|
|
|
it('PGMOVENUL-004: with --drop-nul and no collision, each changed value is listed and stored without the NUL', async () => {
|
|
handle.exec(`update users set username = 'mover' || char(0) || 'b' where id = ${movedId}`);
|
|
// JSON loses the escaped NUL and nothing else: the digits of a large number and the spelling 1.0 stay.
|
|
handle.exec(
|
|
`update addons set config = '{"note":"a\\u0000b","big":12345678901234567890,"ratio":1.0}' where id = 'collab'`,
|
|
);
|
|
const summary = await move(true);
|
|
expect([...summary.nulDropped].sort()).toEqual([
|
|
'addons row id=collab, column config',
|
|
`users row id=${movedId}, column username`,
|
|
]);
|
|
const stored = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
|
run<{ username: string }>('select username from users where id = ?', [movedId]),
|
|
);
|
|
expect(stored).toEqual([{ username: 'moverb' }]);
|
|
const config = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
|
run<{ config: string }>("select config::text as config from addons where id = 'collab'"),
|
|
);
|
|
expect(config).toEqual([{ config: '{"big": 12345678901234567890, "note": "ab", "ratio": 1.0}' }]);
|
|
});
|
|
|
|
it("PGMOVENUL-005: TREK's own text whose bytes are not UTF-8 is refused, with --drop-nul too", async () => {
|
|
await emptyTarget();
|
|
handle.exec("insert into app_settings (key, value) values ('move-bytes', cast(x'ff41' as text))");
|
|
const failure = move(true);
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
'app_settings row key=move-bytes, column value: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
handle.exec("delete from app_settings where key = 'move-bytes'");
|
|
});
|
|
|
|
it('PGMOVENUL-006: a key over an expression that a dropped NUL breaks lists only the values its expression reads', async () => {
|
|
// The row loses a NUL in username and in email; only email is read by lower(email), the key that breaks.
|
|
handle.exec(
|
|
"update users set email = 'Same@x.test' where username = 'movera';" +
|
|
`update users set username = 'mover' || char(0) || 'c', email = 'same' || char(0) || '@x.test' where id = ${movedId}`,
|
|
);
|
|
const failure = move(true);
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
new RegExp(
|
|
'Postgres refused rows of users: .*constraint idx_users_email_lower; ' +
|
|
'Key \\(lower\\(email\\)\\)=\\(same@x\\.test\\) already exists\\.\\n' +
|
|
'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' +
|
|
'[^\\n]*\\n' +
|
|
'--drop-nul removed NUL characters \\(U\\+0000\\) from 1 value\\(s\\) of its column\\(s\\) email ' +
|
|
'\\(read by its unique key lower\\(email\\)\\) that may hold it; [^\\n]*\\n' +
|
|
` - users row id=${movedId}, column email\\n` +
|
|
'Change or remove one of those rows',
|
|
),
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
});
|
|
|
|
it("PGMOVENUL-007: a raw NUL in TREK's JSON is refused as a NUL, and --drop-nul drops it like an escaped one", async () => {
|
|
handle.exec(
|
|
`update users set email = 'mover@x.test' where id = ${movedId};` +
|
|
`update addons set config = '{"note":"a' || char(0) || 'b",' || char(0) || '"ratio":1.0}' where id = 'collab'`,
|
|
);
|
|
const failure = move();
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
'addons row id=collab, column config: the text "{\\"note\\":\\"a\\u0000b\\",\\u0000\\"ratio\\":1.0}" holds a ' +
|
|
'NUL character (U+0000), which Postgres text cannot hold',
|
|
);
|
|
await expect(failure).rejects.toThrow(/Run the move again with --drop-nul/);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
|
|
const summary = await move(true);
|
|
expect(summary.nulDropped).toContain('addons row id=collab, column config');
|
|
const config = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
|
|
run<{ config: string }>("select config::text as config from addons where id = 'collab'"),
|
|
);
|
|
expect(config).toEqual([{ config: '{"note": "ab", "ratio": 1.0}' }]);
|
|
});
|
|
|
|
it('PGMOVENUL-008: two rows the SQLite file held apart but a Postgres key does not are a refusal that says what to do', async () => {
|
|
await emptyTarget();
|
|
// An install whose case-insensitive email index is still deferred: two accounts differ only in case.
|
|
handle.exec(
|
|
'drop index idx_users_email_lower;' +
|
|
"update users set email = 'Dup@x.test' where username = 'movera';" +
|
|
`update users set email = 'dup@x.test' where id = ${movedId}`,
|
|
);
|
|
const failure = move(true);
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
new RegExp(
|
|
'constraint idx_users_email_lower; Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' +
|
|
'Two rows of users share a value of its column\\(s\\) email \\(read by its unique key lower\\(email\\)\\), ' +
|
|
'which Postgres holds unique and the SQLite file did not [\\s\\S]*Change or remove one of those rows in the ' +
|
|
'SQLite file \\(through TREK while it still runs on SQLite, or with the sqlite3 tool while TREK is ' +
|
|
'stopped\\), then run the move again\\.$',
|
|
),
|
|
);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
});
|
|
|
|
it('PGMOVENUL-009: a value that lost a NUL is not blamed for a duplicate that does not hold it', async () => {
|
|
await emptyTarget();
|
|
// The two accounts of PGMOVENUL-008 still differ only in case; a third address loses a NUL and collides with nothing.
|
|
const { user } = await makeUser(sqliteOrm, { username: 'mover-three' });
|
|
handle.exec(`update users set email = 'other' || char(0) || '@x.test' where id = ${user.id}`);
|
|
const failure = move(true);
|
|
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(failure).rejects.toThrow(
|
|
new RegExp(
|
|
'Key \\(lower\\(email\\)\\)=\\(dup@x\\.test\\) already exists\\.\\n' +
|
|
"Two rows of users share [^\\n]*SQLite's lower\\(\\) changes only A to Z\\)\\. The detail above names the " +
|
|
'value\\.\\nChange or remove one of those rows',
|
|
),
|
|
);
|
|
await expect(failure).rejects.not.toThrow(/--drop-nul/);
|
|
expect(await tablesInTarget()).toBe(0);
|
|
});
|
|
|
|
it('PGMOVENUL-010: the catalog names a key over an expression as one, and a plain key over a quoted column by it', async () => {
|
|
const orm = await MikroORM.init({
|
|
...buildOrmConfig(resolveEngine()),
|
|
entities: [],
|
|
entitiesTs: [],
|
|
discovery: { warnWhenNoEntities: false },
|
|
});
|
|
const db = (orm.em.getConnection() as AbstractSqlConnection).getClient<object>();
|
|
const rollback = new Error('roll back');
|
|
let keys: unknown;
|
|
try {
|
|
await db.transaction().execute(async (trx) => {
|
|
// Rolled back below, so the schema never outlives the test.
|
|
await inOrder(
|
|
[
|
|
'create schema move_catalog_probe',
|
|
'set local search_path to move_catalog_probe',
|
|
'create table p (id bigint primary key, "userId" text, email text, note text)',
|
|
'create unique index p_user on p ("userId")',
|
|
'create unique index p_email on p (lower(email), "userId") include (note) where note is not null',
|
|
],
|
|
async (statement) => {
|
|
await run(trx, statement);
|
|
},
|
|
);
|
|
const schema = await readTargetSchema(trx);
|
|
keys = schema.tables.find((table) => table.name === 'p')?.uniqueKeys;
|
|
throw rollback;
|
|
});
|
|
} catch (err) {
|
|
if (err !== rollback) throw err;
|
|
} finally {
|
|
await orm.close(true);
|
|
}
|
|
expect(keys).toEqual([
|
|
{ name: 'p_email', key: 'lower(email), "userId"', columns: ['userId', 'email'], expression: true },
|
|
{ name: 'p_pkey', key: 'id', columns: ['id'], expression: false },
|
|
{ name: 'p_user', key: '"userId"', columns: ['userId'], expression: false },
|
|
]);
|
|
});
|
|
});
|