mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
fix(db): let the move to Postgres change nothing silently
The move stripped NUL from a plugin's own text, merging keys that were distinct, and turned a plugin's values into another type (text in a BLOB column into bytea, numbers in an untyped column into text). Its checksum was built from the converted values, so both passed as verified. A plugin's data now moves exactly as the Postgres plugin store would hand it back, or not at all: a NUL, a value of another storage class than its column, or an integer past 2^53 refuses the move before anything is written, listing plugin, table, row and column. A column left open by SQLite's affinity takes the kind its values need, so text in a BLOB column stays text. TREK's own text holding a NUL from an older install is refused by default too. --drop-nul drops it, lists every changed value in the summary, and names a unique key that dropping a NUL broke. Every table's checksum is now built from the values read from SQLite, so a conversion that changes a value fails the verification.
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVE-001..016: the move of a SQLite install into Postgres
|
||||
* MOVE-001..025: the move of a SQLite install into Postgres
|
||||
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
|
||||
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
|
||||
* the refusals before anything is written, the copy in foreign key order with
|
||||
@@ -15,7 +15,12 @@ import {
|
||||
USAGE,
|
||||
type MoveCommandIo,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/command';
|
||||
import { batchRows, describePgError, insertSql } from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import {
|
||||
batchRows,
|
||||
describePgError,
|
||||
insertSql,
|
||||
refusedRowsMessage,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/copy';
|
||||
import {
|
||||
migrationLevelProblem,
|
||||
MoveRefusal,
|
||||
@@ -82,6 +87,7 @@ const SCHEMA: FakeTableDef[] = [
|
||||
],
|
||||
primaryKey: ['id'],
|
||||
serial: ['id'],
|
||||
unique: [{ name: 'users_name_unique', columns: ['name'] }],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -308,6 +314,134 @@ describe('moveSqliteToPostgres', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('nothing is changed silently', () => {
|
||||
/** A plugin's `plugin.db` under the plugins' data directory. */
|
||||
function pluginDb(pluginId: string, sql: string): void {
|
||||
fs.mkdirSync(path.join(dir, 'plugins-data', pluginId), { recursive: true });
|
||||
sqlite(path.join(dir, 'plugins-data', pluginId, 'plugin.db'), sql);
|
||||
}
|
||||
|
||||
it("MOVE-021: a plugin's NUL or a value that would change type refuses the move before anything is written", async () => {
|
||||
pluginDb(
|
||||
'nul-notes',
|
||||
`create table notes (id integer primary key, tag text, body text);
|
||||
insert into notes values (1, 'team' || char(0) || 'a', 'secret' || char(0) || 'payload'), (2, 'teama', 'other');
|
||||
create table kv (k text primary key, v);
|
||||
insert into kv values ('a', 5), ('b', 1.5), ('c', 'x');`,
|
||||
);
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
const message = await failure.catch((err: Error) => err.message);
|
||||
expect(message).toContain(
|
||||
'4 value(s) in plugin data cannot move to Postgres unchanged; nothing was written:\n' +
|
||||
' - plugin nul-notes: kv row k=a, column v: the value 5 is stored as an INTEGER, but the column holds text ' +
|
||||
'on Postgres, so the plugin would read it back as another type\n' +
|
||||
' - plugin nul-notes: kv row k=b, column v: the value 1.5 is stored as a REAL',
|
||||
);
|
||||
expect(message).toContain(
|
||||
' - plugin nul-notes: notes row id=1, column tag: the text "team\\u0000a" holds a NUL character (U+0000), ' +
|
||||
'which Postgres text cannot hold\n' +
|
||||
' - plugin nul-notes: notes row id=1, column body: the text "secret\\u0000payload" holds a NUL character',
|
||||
);
|
||||
expect(message).toMatch(/A plugin's own data moves exactly as the plugin stored it, or not at all/);
|
||||
// Refused before the target was even looked at: nothing created, nothing inserted.
|
||||
expect(pg.statements).toEqual([]);
|
||||
});
|
||||
|
||||
it('MOVE-022: a BLOB column holding text stays text, and a column holding only numbers stays numbers', async () => {
|
||||
pluginDb(
|
||||
'typed-notes',
|
||||
`create table docs (id integer primary key, payload blob, raw blob, data);
|
||||
insert into docs values (1, '{"a":1}', x'00ff', 5), (2, 'plain', null, 1.5);`,
|
||||
);
|
||||
const summary = await moveSqliteToPostgres(options());
|
||||
expect(pg.statements).toContain(
|
||||
'create table "trek_plugin_typed_notes"."docs" ("id" bigint generated by default as identity, ' +
|
||||
'"payload" text, "raw" bytea, "data" double precision, primary key ("id"))',
|
||||
);
|
||||
expect(pg.rowsOf('"trek_plugin_typed_notes"."docs"')).toEqual([
|
||||
['1', '{"a":1}', Buffer.from([0, 255]), 5],
|
||||
['2', 'plain', null, 1.5],
|
||||
]);
|
||||
expect(summary.plugins[0].tables).toEqual([{ label: 'docs', rows: 2, checksum: expect.any(String) }]);
|
||||
});
|
||||
|
||||
it("MOVE-023: a NUL in TREK's own text is refused by default, and with dropNul dropped and listed", async () => {
|
||||
sqlite(
|
||||
path.join(dir, 'travel.db'),
|
||||
`update users set name = 'Ad' || char(0) || 'a' where id = 1;
|
||||
update trips set meta = '{"k\\u0000":1}' where id = 1;`,
|
||||
);
|
||||
const failure = moveSqliteToPostgres(options());
|
||||
await expect(failure).rejects.toBeInstanceOf(MoveRefusal);
|
||||
const message = await failure.catch((err: Error) => err.message);
|
||||
expect(message).toContain(
|
||||
'2 value(s) in the SQLite file cannot be stored in Postgres exactly; nothing was written:\n' +
|
||||
' - users row id=1, column name: the text "Ad\\u0000a" holds a NUL character (U+0000), which Postgres text cannot hold\n' +
|
||||
' - trips row id=1, column meta: the text "{\\"k\\\\u0000\\":1}" holds a NUL character (U+0000), which ' +
|
||||
'Postgres text cannot hold (escaped as \\u0000)\n' +
|
||||
'2 of them hold a NUL character (U+0000). TREK no longer stores this character and drops it on its next write. ' +
|
||||
'Run the move again with --drop-nul',
|
||||
);
|
||||
expect(pg.statements.at(-1)).toBe(CLEAR_SCHEMA_SQL);
|
||||
|
||||
const summary = await moveSqliteToPostgres(options({ dropNul: true }));
|
||||
expect(pg.rowsOf('"users"')).toEqual([
|
||||
['1', 'Ada'],
|
||||
['2', 'Grace'],
|
||||
]);
|
||||
expect(pg.rowsOf('"trips"')[0][3]).toBe('{"k":1}');
|
||||
expect(summary.nulDropped).toEqual(['users row id=1, column name', 'trips row id=1, column meta']);
|
||||
expect(renderSummary(summary, 'postgres://db/trek')).toContain(
|
||||
' --drop-nul removed NUL characters (U+0000) from 2 value(s):\n' +
|
||||
' users row id=1, column name\n' +
|
||||
' trips row id=1, column meta',
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-024: a unique key that dropping a NUL broke is named as that, never as a bare duplicate', async () => {
|
||||
sqlite(path.join(dir, 'travel.db'), "update users set name = 'Ad' || char(0) || 'a' where id = 2;");
|
||||
pg.failOn = /^insert into "users"/;
|
||||
pg.failWith = {
|
||||
code: '23505',
|
||||
constraint: 'users_name_unique',
|
||||
detail: 'Key (name)=(Ada) already exists.',
|
||||
};
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
'Postgres refused rows of users: refused by the fake; constraint users_name_unique; Key (name)=(Ada) already ' +
|
||||
'exists.\nThe key is broken because --drop-nul removed NUL characters (U+0000) from 1 value(s) of its ' +
|
||||
'column(s) name, and a value that differed from another only by a NUL equals it once the NUL is gone:\n' +
|
||||
' - users row id=2, column name\n' +
|
||||
'Change or remove one of the two rows in the SQLite file (with TREK stopped), then run the move again.',
|
||||
);
|
||||
// A broken key whose columns lost no NUL is reported as Postgres reported it.
|
||||
pg.failWith = { code: '23505', constraint: 'users_pkey', detail: 'Key (id)=(2) already exists.' };
|
||||
await expect(moveSqliteToPostgres(options({ dropNul: true }))).rejects.toThrow(
|
||||
/^Postgres refused rows of users: refused by the fake; constraint users_pkey; Key \(id\)=\(2\) already exists\.$/,
|
||||
);
|
||||
});
|
||||
|
||||
it('MOVE-025: a refused key is explained by every dropped NUL of the table when the key is not known', () => {
|
||||
const spec = {
|
||||
source: 't',
|
||||
target: '"t"',
|
||||
label: 't',
|
||||
columns: [],
|
||||
deferred: [],
|
||||
primaryKey: [],
|
||||
exact: false,
|
||||
uniqueKeys: [],
|
||||
};
|
||||
const dropped = [{ where: 't row 1, column a', column: 'a' }];
|
||||
const err = Object.assign(new Error('duplicate key'), { code: '23505', constraint: 'gone' });
|
||||
expect(refusedRowsMessage(spec, err, dropped)).toMatch(
|
||||
/from 1 value\(s\) of t, and a value .*\n {2}- t row 1, column a\n/,
|
||||
);
|
||||
expect(refusedRowsMessage(spec, new Error('other'), dropped)).toBe('Postgres refused rows of t: other');
|
||||
expect(refusedRowsMessage(spec, err, [])).toBe('Postgres refused rows of t: duplicate key; constraint gone');
|
||||
});
|
||||
});
|
||||
|
||||
describe('migrationLevelProblem', () => {
|
||||
it('MOVE-014: a file at this level passes; older, newer or unversioned ones say what to run', () => {
|
||||
const known = new Set([M1, M2]);
|
||||
@@ -327,12 +461,19 @@ describe('the pieces', () => {
|
||||
expect(pluginColumnKind('INTEGER', classes('text'))).toBe('integer');
|
||||
expect(pluginColumnKind('VARCHAR(20)', classes())).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('blob'))).toBe('bytes');
|
||||
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('integer'))).toBe('integer');
|
||||
// Text in a column declared BLOB stays text: the plugin reads a string back, not a Buffer.
|
||||
expect(pluginColumnKind('BLOB', classes('text'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes('blob', 'text'))).toBe('text');
|
||||
expect(pluginColumnKind('BLOB', classes())).toBe('bytes');
|
||||
expect(pluginColumnKind('DOUBLE', classes())).toBe('float');
|
||||
expect(pluginColumnKind('REAL', classes('text'))).toBe('text');
|
||||
// A REAL column keeps its type; text in it is refused by the copy, not turned into text with the numbers.
|
||||
expect(pluginColumnKind('REAL', classes('text'))).toBe('float');
|
||||
expect(pluginColumnKind('BOOLEAN', classes('integer'))).toBe('integer');
|
||||
expect(pluginColumnKind('', classes('integer', 'real'))).toBe('float');
|
||||
expect(pluginColumnKind('', classes('blob'))).toBe('bytes');
|
||||
expect(pluginColumnKind('', classes())).toBe('text');
|
||||
expect(pluginColumnKind('', classes('integer', 'text'))).toBe('text');
|
||||
expect(pluginColumnKind('DATETIME', classes('text'))).toBe('text');
|
||||
|
||||
expect(pluginDefault(null, 'text')).toBeNull();
|
||||
@@ -394,6 +535,7 @@ describe('runMoveCommand', () => {
|
||||
tables: [{ label: 'users', rows: 2, checksum: 'x' }],
|
||||
plugins: [],
|
||||
sequences: 1,
|
||||
nulDropped: [],
|
||||
warnings: ['careful'],
|
||||
seconds: 0.4,
|
||||
};
|
||||
@@ -421,6 +563,7 @@ describe('runMoveCommand', () => {
|
||||
const help = io();
|
||||
expect(await runMoveCommand(['--help'], help.io)).toBe(0);
|
||||
expect(help.out).toEqual([USAGE]);
|
||||
expect(USAGE).toMatch(/\[--drop-nul\][\s\S]*--drop-nul +drop NUL characters \(U\+0000\) from TREK's own text/);
|
||||
const unknown = io();
|
||||
expect(await runMoveCommand(['--to=x'], unknown.io)).toBe(1);
|
||||
expect(unknown.err[0]).toMatch(/The move was refused: Unknown argument --to=x/);
|
||||
@@ -436,8 +579,12 @@ describe('runMoveCommand', () => {
|
||||
expect.objectContaining({
|
||||
sqliteFile: path.join(dir, 'other.db'),
|
||||
pluginsDataDir: expect.stringMatching(/plugins-data$/),
|
||||
dropNul: false,
|
||||
}),
|
||||
);
|
||||
const dropping = io();
|
||||
expect(await runMoveCommand(['--drop-nul'], dropping.io)).toBe(0);
|
||||
expect(dropping.io.move).toHaveBeenCalledWith(expect.objectContaining({ dropNul: true }));
|
||||
const printed = run.out.join('\n');
|
||||
expect(printed).toContain('postgres://trek:***@db:5432/trek');
|
||||
expect(printed).not.toContain('secret');
|
||||
|
||||
Reference in New Issue
Block a user