mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
e66330e226
A raw U+0000 in TREK's JSON text, which an old or hand-edited travel.db can hold, was refused as "not valid JSON" without naming the NUL or the remedy. It is now refused with the NUL reason, so the refusal names --drop-nul. With --drop-nul, raw and escaped NULs are removed and nothing else, the way TREK's own writes drop them, and each change is listed. Text that is not valid JSON even without its NULs is refused as that, and the checksum's source side follows the same rule. A \u escape of half a surrogate pair parses in JavaScript but jsonb refuses it. Before, that failed the insert without naming a row. It is now refused by table, row and column.
322 lines
16 KiB
TypeScript
322 lines
16 KiB
TypeScript
/**
|
|
* MOVEVAL-001..019: how a SQLite value becomes the value a Postgres column of
|
|
* the move's target takes, under TREK's rules and under a plugin's, the text
|
|
* both sides are compared in, and the order-free table checksum
|
|
* (src/db/engine/postgres/sqlite-move/values.ts).
|
|
*/
|
|
import {
|
|
beyondSafeInteger,
|
|
canonicalFromPostgres,
|
|
canonicalFromSource,
|
|
columnKind,
|
|
convertValue,
|
|
PLUGIN_RULES,
|
|
stableJson,
|
|
TableChecksum,
|
|
type ColumnKind,
|
|
} from '../../../../../../src/db/engine/postgres/sqlite-move/values';
|
|
import { InvalidUtf8Text } from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { describe, expect, it } from 'vitest';
|
|
|
|
const ok = (value: unknown) => ({ ok: true, value });
|
|
const DROP_NUL = { exact: false, dropNul: true };
|
|
|
|
describe('columnKind', () => {
|
|
it('MOVEVAL-001: knows every column type the Postgres baseline uses', () => {
|
|
const sql = fs.readFileSync(path.join(__dirname, '../../../../../../assets/postgres/baseline.sql'), 'utf8');
|
|
const types = new Set(
|
|
[
|
|
...sql.matchAll(
|
|
/" (bigserial|bigint|integer|smallint|text|varchar\(\d+\)|double precision|real|jsonb|json|bytea|boolean|numeric|timestamptz|date|uuid)(?=[ ,)])/g,
|
|
),
|
|
].map((match) => match[1]),
|
|
);
|
|
const asCatalog: Record<string, string> = { bigserial: 'bigint' };
|
|
for (const type of types) {
|
|
const dataType = asCatalog[type] ?? (type.startsWith('varchar') ? 'character varying' : type);
|
|
expect(columnKind(dataType), type).not.toBeNull();
|
|
}
|
|
});
|
|
|
|
it('MOVEVAL-002: has no rule for a type it does not know, so the move refuses instead of guessing', () => {
|
|
expect(columnKind('timestamp with time zone')).toBeNull();
|
|
expect(columnKind('boolean')).toBeNull();
|
|
expect(columnKind('smallint')).toBe('integer');
|
|
expect(columnKind('real')).toBe('float');
|
|
expect(columnKind('character')).toBe('text');
|
|
expect(columnKind('json')).toBe('json');
|
|
});
|
|
});
|
|
|
|
describe('convertValue', () => {
|
|
it('MOVEVAL-003: integers keep all 64 bits and accept whole numbers in any spelling SQLite stored them in', () => {
|
|
expect(convertValue(9007199254740993n, 'integer')).toEqual(ok('9007199254740993'));
|
|
expect(convertValue(42, 'integer')).toEqual(ok('42'));
|
|
expect(convertValue(' -7 ', 'integer')).toEqual(ok('-7'));
|
|
expect(convertValue(null, 'integer')).toEqual(ok(null));
|
|
expect(convertValue(1.5, 'integer')).toMatchObject({ ok: false, reason: 'the value 1.5 is not a whole number' });
|
|
expect(convertValue('abc', 'integer')).toMatchObject({ ok: false, reason: 'the text "abc" is not a whole number' });
|
|
expect(convertValue('99999999999999999999', 'integer')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/does not fit in 64 bits/),
|
|
});
|
|
expect(convertValue(new Uint8Array([1]), 'integer')).toMatchObject({
|
|
ok: false,
|
|
reason: 'a 1-byte BLOB is not a whole number',
|
|
});
|
|
});
|
|
|
|
it('MOVEVAL-004: floats take numbers and numeric text, never a value a double cannot hold', () => {
|
|
expect(convertValue(1.25, 'float')).toEqual(ok(1.25));
|
|
expect(convertValue(12n, 'float')).toEqual(ok(12));
|
|
expect(convertValue('3.5e2', 'float')).toEqual(ok(350));
|
|
expect(convertValue(9007199254740993n, 'float')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/more digits than a double holds/),
|
|
});
|
|
expect(convertValue('north', 'float')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/is not a number/),
|
|
});
|
|
expect(convertValue('', 'float')).toMatchObject({ ok: false });
|
|
});
|
|
|
|
it('MOVEVAL-005: text takes strings and numbers, refuses a NUL unless told to drop it, and refuses a BLOB', () => {
|
|
expect(convertValue('Lisboa', 'text')).toEqual(ok('Lisboa'));
|
|
expect(convertValue(7n, 'text')).toEqual(ok('7'));
|
|
expect(convertValue(2.5, 'text')).toEqual(ok('2.5'));
|
|
expect(convertValue('a\u0000b\u0000', 'text')).toEqual({
|
|
ok: false,
|
|
reason: 'the text "a\\u0000b\\u0000" holds a NUL character (U+0000), which Postgres text cannot hold',
|
|
});
|
|
expect(convertValue('a\u0000b\u0000', 'text', DROP_NUL)).toEqual({ ok: true, value: 'ab', droppedNul: true });
|
|
// The six characters of an escape are text like any other.
|
|
expect(convertValue('a\\u0000b', 'text')).toEqual(ok('a\\u0000b'));
|
|
expect(convertValue(new Uint8Array([0]), 'text')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/is not text/),
|
|
});
|
|
const long = 'x'.repeat(60);
|
|
expect(convertValue(long, 'integer')).toMatchObject({
|
|
reason: `the text "${'x'.repeat(40)}..." is not a whole number`,
|
|
});
|
|
});
|
|
|
|
it('MOVEVAL-006: JSON must parse, and loses the NUL characters it escapes only when told to', () => {
|
|
expect(convertValue('{"a":1}', 'json')).toEqual(ok('{"a":1}'));
|
|
expect(convertValue(5n, 'json')).toEqual(ok('5'));
|
|
expect(convertValue('{broken', 'json')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/is not valid JSON/),
|
|
});
|
|
expect(convertValue('"\\u0000"', 'json')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/holds a NUL character \(U\+0000\), .* \(escaped as \\u0000\)$/),
|
|
});
|
|
expect(convertValue('"\\u0000"', 'json', DROP_NUL)).toEqual({ ok: true, value: '""', droppedNul: true });
|
|
expect(convertValue('{"a\\u0000":["x\\u0000y",1]}', 'json', DROP_NUL)).toEqual({
|
|
ok: true,
|
|
value: '{"a":["xy",1]}',
|
|
droppedNul: true,
|
|
});
|
|
// An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is.
|
|
expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"'));
|
|
expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/is not JSON text/),
|
|
});
|
|
});
|
|
|
|
it('MOVEVAL-007: bytes take a BLOB, or the UTF-8 of text, and refuse a number', () => {
|
|
expect(convertValue(new Uint8Array([0, 255]), 'bytes')).toEqual(ok(Buffer.from([0, 255])));
|
|
expect(convertValue('hé', 'bytes')).toEqual(ok(Buffer.from('hé', 'utf8')));
|
|
expect(convertValue(3n, 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not a BLOB/) });
|
|
});
|
|
|
|
it('MOVEVAL-008: flags an integer past 2^53, which the Postgres driver refuses to read', () => {
|
|
expect(beyondSafeInteger('9007199254740993')).toBe(true);
|
|
expect(beyondSafeInteger('-9007199254740993')).toBe(true);
|
|
expect(beyondSafeInteger('9007199254740991')).toBe(false);
|
|
expect(beyondSafeInteger(12)).toBe(false);
|
|
expect(beyondSafeInteger('text')).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('the compared text', () => {
|
|
const cases: Array<[ColumnKind, Parameters<typeof canonicalFromSource>[0], string]> = [
|
|
['integer', ' -12 ', '-12'],
|
|
['integer', 9007199254740993n, '9007199254740993'],
|
|
['integer', 42, '42'],
|
|
['float', 0.30000000000000004, '0.30000000000000004'],
|
|
['float', 1e21, '1e+21'],
|
|
['float', 12n, '12'],
|
|
['float', '3.5e2', '350'],
|
|
['text', 'São Paulo', 'São Paulo'],
|
|
['text', 7n, '7'],
|
|
['json', '{"b":1,"a":[2,{"d":1,"c":0}]}', '{"a": [2, {"c": 0, "d": 1}], "b": 1}'],
|
|
['json', 5n, '5'],
|
|
['bytes', new Uint8Array([0, 171]), '\\x00ab'],
|
|
];
|
|
|
|
it.each(cases)(
|
|
'MOVEVAL-009: %s values compare equal as read from SQLite and as Postgres prints them',
|
|
(kind, read, printed) => {
|
|
expect(canonicalFromSource(read, kind)).toBe(canonicalFromPostgres(printed, kind));
|
|
expect(canonicalFromSource(null, kind)).toBeNull();
|
|
expect(canonicalFromPostgres(null, kind)).toBeNull();
|
|
},
|
|
);
|
|
|
|
it('MOVEVAL-009b: text read for a BLOB compares as its bytes, and a float as Postgres spells it', () => {
|
|
expect(canonicalFromSource('ab', 'bytes')).toBe('6162');
|
|
expect(canonicalFromPostgres('1e+20', 'float')).toBe(canonicalFromSource(1e20, 'float'));
|
|
expect(canonicalFromPostgres('plain', 'bytes')).toBe('plain');
|
|
expect(stableJson('{"z":null,"y":true}')).toBe('{"y":true,"z":null}');
|
|
});
|
|
|
|
it('MOVEVAL-010: the checksum ignores row order and catches a changed, missing or doubled row', () => {
|
|
const sum = (rows: Array<Array<string | null>>) => {
|
|
const checksum = new TableChecksum();
|
|
for (const row of rows) checksum.add(row);
|
|
return checksum.hex();
|
|
};
|
|
const rows = [
|
|
['1', 'a'],
|
|
['2', null],
|
|
];
|
|
expect(sum(rows)).toBe(sum([...rows].reverse()));
|
|
expect(sum(rows)).toHaveLength(64);
|
|
expect(sum(rows)).not.toBe(
|
|
sum([
|
|
['1', 'a'],
|
|
['2', 'b'],
|
|
]),
|
|
);
|
|
expect(sum(rows)).not.toBe(sum([['1', 'a']]));
|
|
expect(sum(rows)).not.toBe(sum([...rows, ['2', null]]));
|
|
// A NULL and the text "null" are different values.
|
|
expect(sum([['null']])).not.toBe(sum([[null]]));
|
|
});
|
|
});
|
|
|
|
describe("a plugin's values (PLUGIN_RULES)", () => {
|
|
const exact = (value: Parameters<typeof convertValue>[0], kind: ColumnKind) =>
|
|
convertValue(value, kind, PLUGIN_RULES);
|
|
const changed =
|
|
/is stored as .*, but the column holds .* on Postgres, so the plugin would read it back as another type/;
|
|
|
|
it('MOVEVAL-011: a value keeps its storage class and its bytes, as the Postgres plugin store hands it back', () => {
|
|
expect(exact(42n, 'integer')).toEqual(ok('42'));
|
|
expect(exact(1.5, 'float')).toEqual(ok(1.5));
|
|
expect(exact(3n, 'float')).toEqual(ok(3));
|
|
expect(exact('{"a":1}', 'text')).toEqual(ok('{"a":1}'));
|
|
expect(exact(new Uint8Array([0, 1]), 'bytes')).toEqual(ok(Buffer.from([0, 1])));
|
|
expect(exact(null, 'text')).toEqual(ok(null));
|
|
});
|
|
|
|
it('MOVEVAL-012: a value that would come back as another type is refused, not converted', () => {
|
|
expect(exact(1.5, 'text')).toEqual({
|
|
ok: false,
|
|
reason:
|
|
'the value 1.5 is stored as a REAL, but the column holds text on Postgres, ' +
|
|
'so the plugin would read it back as another type',
|
|
});
|
|
expect(exact(5n, 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact('{"a":1}', 'bytes')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact(' 7', 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact(2.5, 'integer')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact('2.5', 'float')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact(new Uint8Array([1]), 'text')).toMatchObject({ ok: false, reason: expect.stringMatching(changed) });
|
|
expect(exact(9007199254740993n, 'integer')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/beyond 2\^53, which the Postgres plugin store refuses to read/),
|
|
});
|
|
expect(exact(-9007199254740993n, 'float')).toMatchObject({
|
|
ok: false,
|
|
reason: expect.stringMatching(/more digits than a double holds/),
|
|
});
|
|
});
|
|
|
|
it('MOVEVAL-013: a NUL in a plugin value is refused, never dropped, whatever the operator asked for', () => {
|
|
expect(exact('team\u0000a', 'text')).toEqual({
|
|
ok: false,
|
|
reason: 'the text "team\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold',
|
|
});
|
|
expect(convertValue('team\u0000a', 'text', { exact: true, dropNul: true })).toMatchObject({ ok: false });
|
|
});
|
|
|
|
it('MOVEVAL-014: the compared text comes from the SQLite value, so a value that changed on the way does not match', () => {
|
|
// What Postgres would print had the NUL been dropped, or the type changed, on the way.
|
|
expect(canonicalFromSource('team\u0000a', 'text', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('teama', 'text'));
|
|
expect(canonicalFromSource('team\u0000a', 'text', { exact: true, dropNul: true })).not.toBe('teama');
|
|
expect(canonicalFromSource(new Uint8Array([0x61]), 'text', PLUGIN_RULES)).not.toBe(
|
|
canonicalFromPostgres('a', 'text'),
|
|
);
|
|
expect(canonicalFromSource(5n, 'bytes')).not.toBe(canonicalFromPostgres('\\x35', 'bytes'));
|
|
// The one change allowed is the one the operator asked for on TREK's own data.
|
|
expect(canonicalFromSource('team\u0000a', 'text', DROP_NUL)).toBe(canonicalFromPostgres('teama', 'text'));
|
|
expect(canonicalFromSource('{"k\\u0000":1}', 'json', DROP_NUL)).toBe(canonicalFromPostgres('{"k": 1}', 'json'));
|
|
});
|
|
});
|
|
|
|
describe('values a conversion could change unseen', () => {
|
|
it('MOVEVAL-015: negative zero keeps its sign on the way and in the compared text', () => {
|
|
// `pg` spells the number -0 as "0", so it is sent as text.
|
|
expect(convertValue(-0, 'float')).toEqual(ok('-0'));
|
|
expect(convertValue('-0', 'float')).toEqual(ok('-0'));
|
|
expect(convertValue(-0, 'float', PLUGIN_RULES)).toEqual(ok('-0'));
|
|
expect(convertValue(0, 'float', PLUGIN_RULES)).toEqual(ok(0));
|
|
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).toBe(canonicalFromPostgres('-0', 'float'));
|
|
expect(canonicalFromSource('-0', 'float')).toBe(canonicalFromPostgres('-0', 'float'));
|
|
// A zero that lost its sign on the way fails the verification.
|
|
expect(canonicalFromSource(-0, 'float', PLUGIN_RULES)).not.toBe(canonicalFromPostgres('0', 'float'));
|
|
});
|
|
|
|
it('MOVEVAL-016: dropping a NUL from JSON changes nothing else in it', () => {
|
|
const written = '{"a\\u0000":"x\\u0000y","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
|
|
const dropped = '{"a":"xy","big":12345678901234567890,"ratio":1.0,"kept":"\\\\u0000"}';
|
|
expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true });
|
|
expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json'));
|
|
});
|
|
|
|
it('MOVEVAL-018: a raw NUL in JSON text is named as a NUL, and --drop-nul drops it like an escaped one', () => {
|
|
// Inside a string and between tokens: JSON allows neither, so the text alone would not even parse.
|
|
const written = '{"note":"a\u0000b",\u0000"esc":"c\\u0000d","ratio":1.0}';
|
|
const reason = `the text ${JSON.stringify(`${written.slice(0, 40)}...`)} holds a NUL character (U+0000), which Postgres text cannot hold`;
|
|
expect(convertValue(written, 'json')).toEqual({ ok: false, reason });
|
|
// A plugin's JSON is never changed: refused for the NUL, whatever the operator asked for.
|
|
expect(convertValue(written, 'json', PLUGIN_RULES)).toEqual({ ok: false, reason });
|
|
const dropped = '{"note":"ab","esc":"cd","ratio":1.0}';
|
|
expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true });
|
|
expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json'));
|
|
// Text that is not JSON even without its NUL is refused as that: --drop-nul would not help.
|
|
expect(convertValue('{broken\u0000', 'json', DROP_NUL)).toEqual({
|
|
ok: false,
|
|
reason: 'the text "{broken\\u0000" is not valid JSON, even without its NUL characters (U+0000)',
|
|
});
|
|
});
|
|
|
|
it('MOVEVAL-019: an escape of half a surrogate pair, which jsonb refuses, is named instead of failing the insert', () => {
|
|
const reason = 'holds a \\u escape of half a UTF-16 surrogate pair, which Postgres JSON refuses';
|
|
for (const text of ['"\\ud800"', '"\\udc00x"', '{"\\ud83d":1}', '["a\\ud83d"]']) {
|
|
expect(convertValue(text, 'json')).toEqual({ ok: false, reason: `the text ${JSON.stringify(text)} ${reason}` });
|
|
}
|
|
// A whole pair is one character, which jsonb takes.
|
|
expect(convertValue('"\\ud83d\\ude00"', 'json')).toEqual(ok('"\\ud83d\\ude00"'));
|
|
});
|
|
|
|
it('MOVEVAL-017: text whose bytes are not UTF-8 is refused under either rules, for every column kind', () => {
|
|
const invalid = new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA');
|
|
const reason = 'the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41), which Postgres text cannot hold';
|
|
for (const kind of ['integer', 'float', 'text', 'json', 'bytes'] as const) {
|
|
expect(convertValue(invalid, kind)).toEqual({ ok: false, reason });
|
|
expect(convertValue(invalid, kind, DROP_NUL)).toEqual({ ok: false, reason });
|
|
expect(convertValue(invalid, kind, PLUGIN_RULES)).toEqual({ ok: false, reason });
|
|
}
|
|
expect(canonicalFromSource(invalid, 'text')).not.toBe(canonicalFromPostgres('\uFFFDA', 'text'));
|
|
});
|
|
});
|