Files
TREK/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts
T
Maurice f2cfbc4041 fix(db): close the move's remaining silent changes
An attack on the move to Postgres with real SQLite files found five values
that changed without a refusal or a listing, or a valid file that failed:

- Negative zero in a plugin column became 0: pg sends the number -0 as "0",
  and the checksum printed both as "0". It is sent as text and kept apart in
  the checksum on both sides.
- Text whose bytes are not valid UTF-8 was read with each bad sequence
  replaced by U+FFFD, and the checksum was built from the replaced text.
  Text is now read as its bytes and decoded strictly in the file's encoding
  (UTF-8, or UTF-16 for a file created so); such a value is refused in
  TREK's and in plugin data, with --drop-nul too, and listed with its bytes.
- --drop-nul serialized a JSON value again, so a large number lost digits
  and 1.0 became 1. Only the \u0000 escapes are removed now.
- A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB
  column) failed the move, and DEFAULT 1.5 on an integer column would have
  been rounded. Such a default is left off and listed as not carried over.
- A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM
  became one that compares exactly. It is still created, and the summary
  lists the collation it lost.

Each case has a regression test that fails without its fix, on SQLite and
on the Postgres lane.
2026-11-17 23:30:35 +00:00

298 lines
14 KiB
TypeScript

/**
* MOVEVAL-001..017: how a SQLite value becomes the value a Postgres column of
* the move's target takes, under TREK's rules and under a plugin's, the text
* both sides are compared in, and the order-free table checksum
* (src/db/engine/postgres/sqlite-move/values.ts).
*/
import {
beyondSafeInteger,
canonicalFromPostgres,
canonicalFromSource,
columnKind,
convertValue,
PLUGIN_RULES,
stableJson,
TableChecksum,
type ColumnKind,
} from '../../../../../../src/db/engine/postgres/sqlite-move/values';
import { InvalidUtf8Text } from '../../../../../../src/db/engine/sqlite/sqlite-source';
import fs from 'node:fs';
import path from 'node:path';
import { describe, expect, it } from 'vitest';
const ok = (value: unknown) => ({ ok: true, value });
const DROP_NUL = { exact: false, dropNul: true };
describe('columnKind', () => {
it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => {
const sql = fs.readFileSync(path.join(__dirname, '../../../../../../assets/postgres/baseline.sql'), 'utf8');
const types = new Set(
[
...sql.matchAll(
/" (bigserial|bigint|integer|smallint|text|varchar\(\d+\)|double precision|real|jsonb|json|bytea|boolean|numeric|timestamptz|date|uuid)(?=[ ,)])/g,
),
].map((match) => match[1]),
);
const asCatalog: Record<string, string> = { bigserial: 'bigint' };
for (const type of types) {
const dataType = asCatalog[type] ?? (type.startsWith('varchar') ? 'character varying' : type);
expect(columnKind(dataType), type).not.toBeNull();
}
});
it('MOVEVAL-002: has no rule for a type it does not know, so the move refuses instead of guessing', () => {
expect(columnKind('timestamp with time zone')).toBeNull();
expect(columnKind('boolean')).toBeNull();
expect(columnKind('smallint')).toBe('integer');
expect(columnKind('real')).toBe('float');
expect(columnKind('character')).toBe('text');
expect(columnKind('json')).toBe('json');
});
});
describe('convertValue', () => {
it('MOVEVAL-003: integers keep all 64 bits and accept whole numbers in any spelling SQLite stored them in', () => {
expect(convertValue(9007199254740993n, 'integer')).toEqual(ok('9007199254740993'));
expect(convertValue(42, 'integer')).toEqual(ok('42'));
expect(convertValue(' -7 ', 'integer')).toEqual(ok('-7'));
expect(convertValue(null, 'integer')).toEqual(ok(null));
expect(convertValue(1.5, 'integer')).toMatchObject({ ok: false, reason: 'the value 1.5 is not a whole number' });
expect(convertValue('abc', 'integer')).toMatchObject({ ok: false, reason: 'the text "abc" is not a whole number' });
expect(convertValue('99999999999999999999', 'integer')).toMatchObject({
ok: false,
reason: expect.stringMatching(/does not fit in 64 bits/),
});
expect(convertValue(new Uint8Array([1]), 'integer')).toMatchObject({
ok: false,
reason: 'a 1-byte BLOB is not a whole number',
});
});
it('MOVEVAL-004: floats take numbers and numeric text, never a value a double cannot hold', () => {
expect(convertValue(1.25, 'float')).toEqual(ok(1.25));
expect(convertValue(12n, 'float')).toEqual(ok(12));
expect(convertValue('3.5e2', 'float')).toEqual(ok(350));
expect(convertValue(9007199254740993n, 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/more digits than a double holds/),
});
expect(convertValue('north', 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not a number/),
});
expect(convertValue('', 'float')).toMatchObject({ ok: false });
});
it('MOVEVAL-005: text takes strings and numbers, refuses a NUL unless told to drop it, and refuses a BLOB', () => {
expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa'));
expect(convertValue(7n, 'text')).toEqual(ok('7'));
expect(convertValue(2.5, 'text')).toEqual(ok('2.5'));
expect(convertValue('a\u0000b\u0000', 'text')).toEqual({
ok: false,
reason: 'the text "a\\u0000b\\u0000" holds a NUL character (U+0000), which Postgres text cannot hold',
});
expect(convertValue('a\u0000b\u0000', 'text', DROP_NUL)).toEqual({ ok: true, value: 'ab', droppedNul: true });
// The six characters of an escape are text like any other.
expect(convertValue('a\\u0000b', 'text')).toEqual(ok('a\\u0000b'));
expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not text/),
});
const long = 'x'.repeat(60);
expect(convertValue(long, 'integer')).toMatchObject({
reason: `the text "${'x'.repeat(40)}..." is not a whole number`,
});
});
it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes only when told to', () => {
expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}'));
expect(convertValue(5n, 'json')).toEqual(ok('5'));
expect(convertValue('{broken', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not valid JSON/),
});
expect(convertValue('"\\u0000"', 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/holds a NUL character \(U\+0000\), .* \(escaped as \\u0000\)$/),
});
expect(convertValue('"\\u0000"', 'json', DROP_NUL)).toEqual({ ok: true, value: '""', droppedNul: true });
expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json', DROP_NUL)).toEqual({
ok: true,
value: '{"a":["xy",1]}',
droppedNul: true,
});
// An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is.
expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"'));
// A raw NUL is not valid JSON in the first place.
expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) });
expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({
ok: false,
reason: expect.stringMatching(/is not JSON text/),
});
});
it('MOVEVAL-007: bytes take a BLOB, or the UTF-8 of text, and refuse a number', () => {
expect(convertValue(new Uint8Array([0, 255]), 'bytes')).toEqual(ok(Buffer.from([0, 255])));
expect(convertValue('hé', 'bytes')).toEqual(ok(Buffer.from('hé', 'utf8')));
expect(convertValue(3n, 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not a BLOB/) });
});
it('MOVEVAL-008: flags an integer past 2^53, which the Postgres driver refuses to read', () => {
expect(beyondSafeInteger('9007199254740993')).toBe(true);
expect(beyondSafeInteger('-9007199254740993')).toBe(true);
expect(beyondSafeInteger('9007199254740991')).toBe(false);
expect(beyondSafeInteger(12)).toBe(false);
expect(beyondSafeInteger('text')).toBe(false);
});
});
describe('the compared text', () => {
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSource>[0], string]> = [
['integer', ' -12 ', '-12'],
['integer', 9007199254740993n, '9007199254740993'],
['integer', 42, '42'],
['float', 0.30000000000000004, '0.30000000000000004'],
['float', 1e21, '1e+21'],
['float', 12n, '12'],
['float', '3.5e2', '350'],
['text', 'São Paulo', 'São Paulo'],
['text', 7n, '7'],
['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'],
['json', 5n, '5'],
['bytes', new Uint8Array([0, 171]), '\\x00ab'],
];
it.each(cases)(
'MOVEVAL-009: %s values compare equal as read from SQLite and as Postgres prints them',
(kind, read, printed) => {
expect(canonicalFromSource(read, kind)).toBe(canonicalFromPostgres(printed, kind));
expect(canonicalFromSource(null, kind)).toBeNull();
expect(canonicalFromPostgres(null, kind)).toBeNull();
},
);
it('MOVEVAL-009b: text read for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
expect(canonicalFromSource('ab', 'bytes')).toBe('6162');
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSource(1e20, 'float'));
expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain');
expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}');
});
it('MOVEVAL-010: the checksum ignores row order and catches a changed, missing or doubled row', () => {
const sum = (rows: Array<Array<string | null>>) => {
const checksum = new TableChecksum();
for (const row of rows) checksum.add(row);
return checksum.hex();
};
const rows = [
['1', 'a'],
['2', null],
];
expect(sum(rows)).toBe(sum([...rows].reverse()));
expect(sum(rows)).toHaveLength(64);
expect(sum(rows)).not.toBe(
sum([
['1', 'a'],
['2', 'b'],
]),
);
expect(sum(rows)).not.toBe(sum([['1', 'a']]));
expect(sum(rows)).not.toBe(sum([...rows, ['2', null]]));
// A NULL and the text "null" are different values.
expect(sum([['null']])).not.toBe(sum([[null]]));
});
});
describe("a plugin's values (PLUGIN_RULES)", () => {
const exact = (value: Parameters<typeof convertValue>[0], kind: ColumnKind) =>
convertValue(value, kind, PLUGIN_RULES);
const changed =
/is stored as .*, but the column holds .* on Postgres, so the plugin would read it back as another type/;
it('MOVEVAL-011: a value keeps its storage class and its bytes, as the Postgres plugin store hands it back', () => {
expect(exact(42n, 'integer')).toEqual(ok('42'));
expect(exact(1.5, 'float')).toEqual(ok(1.5));
expect(exact(3n, 'float')).toEqual(ok(3));
expect(exact('{"a":1}', 'text')).toEqual(ok('{"a":1}'));
expect(exact(new Uint8Array([0, 1]), 'bytes')).toEqual(ok(Buffer.from([0, 1])));
expect(exact(null, 'text')).toEqual(ok(null));
});
it('MOVEVAL-012: a value that would come back as another type is refused, not converted', () => {
expect(exact(1.5, 'text')).toEqual({
ok: false,
reason:
'the value 1.5 is stored as a REAL, but the column holds text on Postgres, ' +
'so the plugin would read it back as another type',
});
expect(exact(5n, 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact('{"a":1}', 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact(' 7', 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact(2.5, 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact('2.5', 'float')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact(new Uint8Array([1]), 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
expect(exact(9007199254740993n, 'integer')).toMatchObject({
ok: false,
reason: expect.stringMatching(/beyond 2\^53, which the Postgres plugin store refuses to read/),
});
expect(exact(-9007199254740993n, 'float')).toMatchObject({
ok: false,
reason: expect.stringMatching(/more digits than a double holds/),
});
});
it('MOVEVAL-013: a NUL in a plugin value is refused, never dropped, whatever the operator asked for', () => {
expect(exact('team\u0000a', 'text')).toEqual({
ok: false,
reason: 'the text "team\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold',
});
expect(convertValue('team\u0000a', 'text', { exact: true, dropNul: true })).toMatchObject({ ok: false });
});
it('MOVEVAL-014: the compared text comes from the SQLite value, so a value that changed on the way does not match', () => {
// What Postgres would print had the NUL been dropped, or the type changed, on the way.
expect(canonicalFromSource('team\u0000a', 'text', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('teama', 'text'));
expect(canonicalFromSource('team\u0000a', 'text', { exact: true, dropNul: true })).not.toBe('teama');
expect(canonicalFromSource(new Uint8Array([0x61]), 'text', PLUGIN_RULES)).not.toBe(
canonicalFromPostgres('a', 'text'),
);
expect(canonicalFromSource(5n, 'bytes')).not.toBe(canonicalFromPostgres('\\x35', 'bytes'));
// The one change allowed is the one the operator asked for on TREK's own data.
expect(canonicalFromSource('team\u0000a', 'text', DROP_NUL)).toBe(canonicalFromPostgres('teama', 'text'));
expect(canonicalFromSource('{"k\\u0000":1}', 'json', DROP_NUL)).toBe(canonicalFromPostgres('{"k": 1}', 'json'));
});
});
describe('values a conversion could change unseen', () => {
it('MOVEVAL-015: negative zero keeps its sign on the way and in the compared text', () => {
// `pg` spells the number -0 as "0", so it is sent as text.
expect(convertValue(-0, 'float')).toEqual(ok('-0'));
expect(convertValue('-0', 'float')).toEqual(ok('-0'));
expect(convertValue(-0, 'float', PLUGIN_RULES)).toEqual(ok('-0'));
expect(convertValue(0, 'float', PLUGIN_RULES)).toEqual(ok(0));
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).toBe(canonicalFromPostgres('-0', 'float'));
expect(canonicalFromSource('-0', 'float')).toBe(canonicalFromPostgres('-0', 'float'));
// A zero that lost its sign on the way fails the verification.
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('0', 'float'));
});
it('MOVEVAL-016: dropping a NUL from JSON changes nothing else in it', () => {
const written = '{"a\\u0000":"x\\u0000y","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
const dropped = '{"a":"xy","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true });
expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json'));
});
it('MOVEVAL-017: text whose bytes are not UTF-8 is refused under either rules, for every column kind', () => {
const invalid = new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA');
const reason = 'the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41), which Postgres text cannot hold';
for (const kind of ['integer', 'float', 'text', 'json', 'bytes'] as const) {
expect(convertValue(invalid, kind)).toEqual({ ok: false, reason });
expect(convertValue(invalid, kind, DROP_NUL)).toEqual({ ok: false, reason });
expect(convertValue(invalid, kind, PLUGIN_RULES)).toEqual({ ok: false, reason });
}
expect(canonicalFromSource(invalid, 'text')).not.toBe(canonicalFromPostgres('\uFFFDA', 'text'));
});
});