fix(db): close the move's remaining silent changes

An attack on the move to Postgres with real SQLite files found five values
that changed without a refusal or a listing, or a valid file that failed:

- Negative zero in a plugin column became 0: pg sends the number -0 as "0",
  and the checksum printed both as "0". It is sent as text and kept apart in
  the checksum on both sides.
- Text whose bytes are not valid UTF-8 was read with each bad sequence
  replaced by U+FFFD, and the checksum was built from the replaced text.
  Text is now read as its bytes and decoded strictly in the file's encoding
  (UTF-8, or UTF-16 for a file created so); such a value is refused in
  TREK's and in plugin data, with --drop-nul too, and listed with its bytes.
- --drop-nul serialized a JSON value again, so a large number lost digits
  and 1.0 became 1. Only the \u0000 escapes are removed now.
- A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB
  column) failed the move, and DEFAULT 1.5 on an integer column would have
  been rounded. Such a default is left off and listed as not carried over.
- A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM
  became one that compares exactly. It is still created, and the summary
  lists the collation it lost.

Each case has a regression test that fails without its fix, on SQLite and
on the Postgres lane.
This commit is contained in:
Maurice
2026-11-17 23:30:35 +00:00
parent 636fc7d0a6
commit f2cfbc4041
9 changed files with 315 additions and 46 deletions
+36 -10
View File
@@ -1,11 +1,12 @@
/**
* PGMOVENUL-001..004: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
* PGMOVENUL-001..005: the move into Postgres (`src/db/engine/postgres/sqlite-move/`)
* changes nothing silently, on the lane's server. A plugin value holding a NUL
* refuses the move before anything is written; a NUL in TREK's own text
* refuses it unless `dropNul` (`--drop-nul`) is given; a unique key that
* dropping a NUL breaks is named as that, with the constraint Postgres
* reports; and a move with `dropNul` lists every value it changed and stores
* the value TREK would have written.
* or bytes that are not UTF-8 refuses the move before anything is written; a
* NUL in TREK's own text refuses it unless `dropNul` (`--drop-nul`) is given;
* a unique key that dropping a NUL breaks is named as that, with the
* constraint Postgres reports; a move with `dropNul` lists every value it
* changed and stores the value TREK would have written, JSON with nothing
* else changed; and TREK's own text that is not UTF-8 is refused either way.
*/
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
@@ -101,7 +102,7 @@ beforeAll(async () => {
const plugin = new Database(path.join(dir, 'plugins-data', 'nul-notes', 'plugin.db'));
plugin.exec(
'create table notes (k text primary key, v text);' +
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two');",
"insert into notes values ('team' || char(0) || 'a', 'one'), ('teama', 'two'), ('bytes', cast(x'ff41' as text));",
);
plugin.close();
await emptyTarget();
@@ -114,11 +115,14 @@ afterAll(async () => {
});
describe('the move changes nothing silently', () => {
it("PGMOVENUL-001: a plugin's NUL refuses the move before anything is written", async () => {
it("PGMOVENUL-001: a plugin's NUL or bytes that are not UTF-8 refuse the move before anything is written", async () => {
const failure = move(true);
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
await expect(failure).rejects.toThrow(
/1 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
/2 value\(s\) in plugin data cannot move to Postgres unchanged; nothing was written:\n {2}- plugin nul-notes: notes row k=team\\u0000a, column k: the text "team\\u0000a" holds a NUL character/,
);
await expect(failure).rejects.toThrow(
'plugin nul-notes: notes row k=bytes, column v: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
);
expect(await tablesInTarget()).toBe(0);
fs.rmSync(path.join(dir, 'plugins-data'), { recursive: true, force: true });
@@ -149,11 +153,33 @@ describe('the move changes nothing silently', () => {
it('PGMOVENUL-004: with --drop-nul and no collision, each changed value is listed and stored without the NUL', async () => {
handle.exec(`update users set username = 'mover' || char(0) || 'b' where id = ${movedId}`);
// JSON loses the escaped NUL and nothing else: the digits of a large number and the spelling 1.0 stay.
handle.exec(
`update addons set config = '{"note":"a\\u0000b","big":12345678901234567890,"ratio":1.0}' where id = 'collab'`,
);
const summary = await move(true);
expect(summary.nulDropped).toEqual([`users row id=${movedId}, column username`]);
expect([...summary.nulDropped].sort()).toEqual([
'addons row id=collab, column config',
`users row id=${movedId}, column username`,
]);
const stored = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
run<{ username: string }>('select username from users where id = ?', [movedId]),
);
expect(stored).toEqual([{ username: 'moverb' }]);
const config = await withAdmin(process.env.TREK_DB_URL as string, (run) =>
run<{ config: string }>("select config::text as config from addons where id = 'collab'"),
);
expect(config).toEqual([{ config: '{"big": 12345678901234567890, "note": "ab", "ratio": 1.0}' }]);
});
it("PGMOVENUL-005: TREK's own text whose bytes are not UTF-8 is refused, with --drop-nul too", async () => {
await emptyTarget();
handle.exec("insert into app_settings (key, value) values ('move-bytes', cast(x'ff41' as text))");
const failure = move(true);
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
await expect(failure).rejects.toThrow(
'app_settings row key=move-bytes, column value: the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41)',
);
expect(await tablesInTarget()).toBe(0);
});
});
+5 -3
View File
@@ -127,9 +127,10 @@ function seedPluginData(root: string): void {
'create unique index notes_body on notes (body);' +
"insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" +
"delete from notes where body = 'gone';" +
// Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them.
// Text in a column declared BLOB, and numbers in a column without a type: both come back as the plugin stored them,
// negative zero (which a column without a type keeps) with its sign.
'create table docs (id integer primary key, payload blob, n);' +
`insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5);`,
`insert into docs values (1, '{"a":1}', 5), (2, 'plain', 2.5), (3, 'signed', -0.0);`,
);
plugin.close();
}
@@ -212,7 +213,7 @@ describe('moving a SQLite install to Postgres', () => {
schema: 'trek_plugin_move_notes',
tables: [
expect.objectContaining({ label: '_plugin_migrations', rows: 1 }),
expect.objectContaining({ label: 'docs', rows: 2 }),
expect.objectContaining({ label: 'docs', rows: 3 }),
expect.objectContaining({ label: 'notes', rows: 2 }),
],
}),
@@ -289,6 +290,7 @@ describe('moving a SQLite install to Postgres', () => {
expect(await store.query('select id, payload, n from docs order by id')).toEqual([
{ id: 1, payload: '{"a":1}', n: 5 },
{ id: 2, payload: 'plain', n: 2.5 },
{ id: 3, payload: 'signed', n: -0 },
]);
// The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out.
expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false });