From e66330e2269b74f998b353ca94209b7c036da6f0 Mon Sep 17 00:00:00 2001 From: Maurice Date: Wed, 18 Nov 2026 13:20:07 +0100 Subject: [PATCH] fix(db): treat a raw NUL in JSON like any NUL in the move A raw U+0000 in TREK's JSON text, which an old or hand-edited travel.db can hold, was refused as "not valid JSON" without naming the NUL or the remedy. It is now refused with the NUL reason, so the refusal names --drop-nul. With --drop-nul, raw and escaped NULs are removed and nothing else, the way TREK's own writes drop them, and each change is listed. Text that is not valid JSON even without its NULs is refused as that, and the checksum's source side follows the same rule. A \u escape of half a surrogate pair parses in JavaScript but jsonb refuses it. Before, that failed the insert without naming a row. It is now refused by table, row and column. --- .../db/engine/postgres/sqlite-move/values.ts | 60 +++++++++++++++---- .../postgres/sqlite-move/values.test.ts | 30 +++++++++- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/server/src/db/engine/postgres/sqlite-move/values.ts b/server/src/db/engine/postgres/sqlite-move/values.ts index f558d8cb0..8af67e8ef 100644 --- a/server/src/db/engine/postgres/sqlite-move/values.ts +++ b/server/src/db/engine/postgres/sqlite-move/values.ts @@ -181,19 +181,59 @@ function dropJsonNul(text: string): string { return text.replace(/\\(?:u0000|[\s\S])/g, (escape) => (escape === '\\u0000' ? '' : escape)); } +/** JSON text without its NUL characters, raw or escaped, as `--drop-nul` stores it; see {@link toJson}. */ +function jsonWithoutNul(text: string): string { + return dropJsonNul(stripNul(text)); +} + +/** Half a UTF-16 surrogate pair: a high one without a low one after it, or a low one without a high one before. */ +const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? LONE_SURROGATE.test(key) || jsonHoldsLoneSurrogate(item)); + } + return false; +} + +/** The text parsed as JSON, or undefined when it is not valid JSON. */ +function parseJson(text: string): { parsed: unknown } | undefined { + try { + return { parsed: JSON.parse(text) as unknown }; + } catch { + return undefined; + } +} + +/** + * JSON text for a `jsonb` column. A NUL can sit in it two ways: escaped + * (`\u0000`), which is valid JSON that jsonb refuses, or raw, which is not + * valid JSON at all (JSON escapes every control character inside a string and + * allows none outside one) and which TREK's own writes drop like any NUL in + * text (`db/nul-free.ts`). Both are a NUL: refused with the NUL reason, and + * under `--drop-nul` removed, the raw characters and the escapes and nothing + * else. Text that is not valid JSON even without its NUL characters is + * refused as that. An escape of half a surrogate pair (`\ud800` alone) parses + * in JavaScript but not in jsonb, so it is named here instead of failing the + * insert without a row. + */ function toJson(value: Exclude, dropNul: boolean): Converted { if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`); - let parsed: unknown; - try { - parsed = JSON.parse(value); - } catch { - return refuse(`${describe(value)} is not valid JSON`); + const rawNul = value.includes(NUL); + const json = parseJson(rawNul ? stripNul(value) : value); + if (!json) { + return refuse(`${describe(value)} is not valid JSON${rawNul ? ', even without its NUL characters (U+0000)' : ''}`); } - // Valid JSON holds a NUL only escaped, which jsonb refuses as well. - if (!jsonHoldsNul(parsed)) return { ok: true, value }; - if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`); - return { ok: true, value: dropJsonNul(value), droppedNul: true }; + if (jsonHoldsLoneSurrogate(json.parsed)) { + return refuse(`${describe(value)} holds a \\u escape of half a UTF-16 surrogate pair, which Postgres JSON refuses`); + } + if (!rawNul && !jsonHoldsNul(json.parsed)) return { ok: true, value }; + if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON}${rawNul ? '' : ' (escaped as \\u0000)'}`); + return { ok: true, value: jsonWithoutNul(value), droppedNul: true }; } function toBytes(value: Exclude): Converted { @@ -335,7 +375,7 @@ export function canonicalFromSource( return typeof value === 'string' && dropNul ? stripNul(value) : String(value); case 'json': if (typeof value !== 'string') return stableJson(String(value)); - return stableJson(dropNul ? dropJsonNul(value) : value); + return stableJson(dropNul ? jsonWithoutNul(value) : value); case 'bytes': return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`; } diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts index eac237989..84fd1d840 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts @@ -1,5 +1,5 @@ /** - * MOVEVAL-001..017: how a SQLite value becomes the value a Postgres column of + * MOVEVAL-001..019: how a SQLite value becomes the value a Postgres column of * the move's target takes, under TREK's rules and under a plugin's, the text * both sides are compared in, and the order-free table checksum * (src/db/engine/postgres/sqlite-move/values.ts). @@ -124,8 +124,6 @@ describe('convertValue', () => { }); // An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is. expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"')); - // A raw NUL is not valid JSON in the first place. - expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) }); expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not JSON text/), @@ -284,6 +282,32 @@ describe('values a conversion could change unseen', () => { expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json')); }); + it('MOVEVAL-018: a raw NUL in JSON text is named as a NUL, and --drop-nul drops it like an escaped one', () => { + // Inside a string and between tokens: JSON allows neither, so the text alone would not even parse. + const written = '{"note":"a\u0000b",\u0000"esc":"c\\u0000d","ratio":1.0}'; + const reason = `the text ${JSON.stringify(`${written.slice(0, 40)}...`)} holds a NUL character (U+0000), which Postgres text cannot hold`; + expect(convertValue(written, 'json')).toEqual({ ok: false, reason }); + // A plugin's JSON is never changed: refused for the NUL, whatever the operator asked for. + expect(convertValue(written, 'json', PLUGIN_RULES)).toEqual({ ok: false, reason }); + const dropped = '{"note":"ab","esc":"cd","ratio":1.0}'; + expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true }); + expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json')); + // Text that is not JSON even without its NUL is refused as that: --drop-nul would not help. + expect(convertValue('{broken\u0000', 'json', DROP_NUL)).toEqual({ + ok: false, + reason: 'the text "{broken\\u0000" is not valid JSON, even without its NUL characters (U+0000)', + }); + }); + + it('MOVEVAL-019: an escape of half a surrogate pair, which jsonb refuses, is named instead of failing the insert', () => { + const reason = 'holds a \\u escape of half a UTF-16 surrogate pair, which Postgres JSON refuses'; + for (const text of ['"\\ud800"', '"\\udc00x"', '{"\\ud83d":1}', '["a\\ud83d"]']) { + expect(convertValue(text, 'json')).toEqual({ ok: false, reason: `the text ${JSON.stringify(text)} ${reason}` }); + } + // A whole pair is one character, which jsonb takes. + expect(convertValue('"\\ud83d\\ude00"', 'json')).toEqual(ok('"\\ud83d\\ude00"')); + }); + it('MOVEVAL-017: text whose bytes are not UTF-8 is refused under either rules, for every column kind', () => { const invalid = new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA'); const reason = 'the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41), which Postgres text cannot hold';