diff --git a/server/src/db/engine/postgres/sqlite-move/values.ts b/server/src/db/engine/postgres/sqlite-move/values.ts index f558d8cb0..8af67e8ef 100644 --- a/server/src/db/engine/postgres/sqlite-move/values.ts +++ b/server/src/db/engine/postgres/sqlite-move/values.ts @@ -181,19 +181,59 @@ function dropJsonNul(text: string): string { return text.replace(/\\(?:u0000|[\s\S])/g, (escape) => (escape === '\\u0000' ? '' : escape)); } +/** JSON text without its NUL characters, raw or escaped, as `--drop-nul` stores it; see {@link toJson}. */ +function jsonWithoutNul(text: string): string { + return dropJsonNul(stripNul(text)); +} + +/** Half a UTF-16 surrogate pair: a high one without a low one after it, or a low one without a high one before. */ +const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? LONE_SURROGATE.test(key) || jsonHoldsLoneSurrogate(item)); + } + return false; +} + +/** The text parsed as JSON, or undefined when it is not valid JSON. */ +function parseJson(text: string): { parsed: unknown } | undefined { + try { + return { parsed: JSON.parse(text) as unknown }; + } catch { + return undefined; + } +} + +/** + * JSON text for a `jsonb` column. A NUL can sit in it two ways: escaped + * (`\u0000`), which is valid JSON that jsonb refuses, or raw, which is not + * valid JSON at all (JSON escapes every control character inside a string and + * allows none outside one) and which TREK's own writes drop like any NUL in + * text (`db/nul-free.ts`). Both are a NUL: refused with the NUL reason, and + * under `--drop-nul` removed, the raw characters and the escapes and nothing + * else. Text that is not valid JSON even without its NUL characters is + * refused as that. An escape of half a surrogate pair (`\ud800` alone) parses + * in JavaScript but not in jsonb, so it is named here instead of failing the + * insert without a row. + */ function toJson(value: Exclude, dropNul: boolean): Converted { if (typeof value === 'bigint' || typeof value === 'number') return { ok: true, value: String(value) }; if (typeof value !== 'string') return refuse(`${describe(value)} is not JSON text`); - let parsed: unknown; - try { - parsed = JSON.parse(value); - } catch { - return refuse(`${describe(value)} is not valid JSON`); + const rawNul = value.includes(NUL); + const json = parseJson(rawNul ? stripNul(value) : value); + if (!json) { + return refuse(`${describe(value)} is not valid JSON${rawNul ? ', even without its NUL characters (U+0000)' : ''}`); } - // Valid JSON holds a NUL only escaped, which jsonb refuses as well. - if (!jsonHoldsNul(parsed)) return { ok: true, value }; - if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON} (escaped as \\u0000)`); - return { ok: true, value: dropJsonNul(value), droppedNul: true }; + if (jsonHoldsLoneSurrogate(json.parsed)) { + return refuse(`${describe(value)} holds a \\u escape of half a UTF-16 surrogate pair, which Postgres JSON refuses`); + } + if (!rawNul && !jsonHoldsNul(json.parsed)) return { ok: true, value }; + if (!dropNul) return refuse(`${describe(value)} ${NUL_REASON}${rawNul ? '' : ' (escaped as \\u0000)'}`); + return { ok: true, value: jsonWithoutNul(value), droppedNul: true }; } function toBytes(value: Exclude): Converted { @@ -335,7 +375,7 @@ export function canonicalFromSource( return typeof value === 'string' && dropNul ? stripNul(value) : String(value); case 'json': if (typeof value !== 'string') return stableJson(String(value)); - return stableJson(dropNul ? dropJsonNul(value) : value); + return stableJson(dropNul ? jsonWithoutNul(value) : value); case 'bytes': return typeof value === 'string' ? Buffer.from(value, 'utf8').toString('hex') : `number:${String(value)}`; } diff --git a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts index eac237989..84fd1d840 100644 --- a/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts +++ b/server/tests/unit/db/engine/postgres/sqlite-move/values.test.ts @@ -1,5 +1,5 @@ /** - * MOVEVAL-001..017: how a SQLite value becomes the value a Postgres column of + * MOVEVAL-001..019: how a SQLite value becomes the value a Postgres column of * the move's target takes, under TREK's rules and under a plugin's, the text * both sides are compared in, and the order-free table checksum * (src/db/engine/postgres/sqlite-move/values.ts). @@ -124,8 +124,6 @@ describe('convertValue', () => { }); // An escaped backslash followed by u0000 holds no NUL, and stays exactly as it is. expect(convertValue('"\\\\u0000"', 'json', DROP_NUL)).toEqual(ok('"\\\\u0000"')); - // A raw NUL is not valid JSON in the first place. - expect(convertValue('"\u0000"', 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/valid JSON/) }); expect(convertValue(new Uint8Array([1]), 'json')).toMatchObject({ ok: false, reason: expect.stringMatching(/is not JSON text/), @@ -284,6 +282,32 @@ describe('values a conversion could change unseen', () => { expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json')); }); + it('MOVEVAL-018: a raw NUL in JSON text is named as a NUL, and --drop-nul drops it like an escaped one', () => { + // Inside a string and between tokens: JSON allows neither, so the text alone would not even parse. + const written = '{"note":"a\u0000b",\u0000"esc":"c\\u0000d","ratio":1.0}'; + const reason = `the text ${JSON.stringify(`${written.slice(0, 40)}...`)} holds a NUL character (U+0000), which Postgres text cannot hold`; + expect(convertValue(written, 'json')).toEqual({ ok: false, reason }); + // A plugin's JSON is never changed: refused for the NUL, whatever the operator asked for. + expect(convertValue(written, 'json', PLUGIN_RULES)).toEqual({ ok: false, reason }); + const dropped = '{"note":"ab","esc":"cd","ratio":1.0}'; + expect(convertValue(written, 'json', DROP_NUL)).toEqual({ ok: true, value: dropped, droppedNul: true }); + expect(canonicalFromSource(written, 'json', DROP_NUL)).toBe(canonicalFromPostgres(dropped, 'json')); + // Text that is not JSON even without its NUL is refused as that: --drop-nul would not help. + expect(convertValue('{broken\u0000', 'json', DROP_NUL)).toEqual({ + ok: false, + reason: 'the text "{broken\\u0000" is not valid JSON, even without its NUL characters (U+0000)', + }); + }); + + it('MOVEVAL-019: an escape of half a surrogate pair, which jsonb refuses, is named instead of failing the insert', () => { + const reason = 'holds a \\u escape of half a UTF-16 surrogate pair, which Postgres JSON refuses'; + for (const text of ['"\\ud800"', '"\\udc00x"', '{"\\ud83d":1}', '["a\\ud83d"]']) { + expect(convertValue(text, 'json')).toEqual({ ok: false, reason: `the text ${JSON.stringify(text)} ${reason}` }); + } + // A whole pair is one character, which jsonb takes. + expect(convertValue('"\\ud83d\\ude00"', 'json')).toEqual(ok('"\\ud83d\\ude00"')); + }); + it('MOVEVAL-017: text whose bytes are not UTF-8 is refused under either rules, for every column kind', () => { const invalid = new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA'); const reason = 'the text "\uFFFDA" holds bytes that are not valid UTF-8 (ff41), which Postgres text cannot hold';