mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
fix(db): close the move's remaining silent changes
An attack on the move to Postgres with real SQLite files found five values that changed without a refusal or a listing, or a valid file that failed: - Negative zero in a plugin column became 0: pg sends the number -0 as "0", and the checksum printed both as "0". It is sent as text and kept apart in the checksum on both sides. - Text whose bytes are not valid UTF-8 was read with each bad sequence replaced by U+FFFD, and the checksum was built from the replaced text. Text is now read as its bytes and decoded strictly in the file's encoding (UTF-8, or UTF-16 for a file created so); such a value is refused in TREK's and in plugin data, with --drop-nul too, and listed with its bytes. - --drop-nul serialized a JSON value again, so a large number lost digits and 1.0 became 1. Only the \u0000 escapes are removed now. - A plugin default Postgres refuses for the column (DEFAULT 0 on a BLOB column) failed the move, and DEFAULT 1.5 on an integer column would have been rounded. Such a default is left off and listed as not carried over. - A plugin index or primary key SQLite compared with COLLATE NOCASE or RTRIM became one that compares exactly. It is still created, and the summary lists the collation it lost. Each case has a regression test that fails without its fix, on SQLite and on the Postgres lane.
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* MOVE-001..025: the move of a SQLite install into Postgres
|
||||
* MOVE-001..026: the move of a SQLite install into Postgres
|
||||
* (src/db/engine/postgres/sqlite-move/), over real SQLite files and a
|
||||
* stand-in for Postgres that keeps what it is sent (tests/helpers/fake-postgres-move.ts):
|
||||
* the refusals before anything is written, the copy in foreign key order with
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
import type { AppEnv } from '../../../../../../src/app-config';
|
||||
import { CLEAR_SCHEMA_SQL } from '../../../../../../src/db/engine/postgres/clear-schema';
|
||||
import { PG_NOW_TEXT_DEFAULT } from '../../../../../../src/db/engine/postgres/schema-adjust';
|
||||
import {
|
||||
runMoveCommand,
|
||||
USAGE,
|
||||
@@ -34,7 +35,11 @@ import {
|
||||
pluginColumnKind,
|
||||
pluginDefault,
|
||||
} from '../../../../../../src/db/engine/postgres/sqlite-move/plugin-data';
|
||||
import { sqliteInstallHoldsAccounts, SqliteSource } from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
||||
import {
|
||||
InvalidUtf8Text,
|
||||
sqliteInstallHoldsAccounts,
|
||||
SqliteSource,
|
||||
} from '../../../../../../src/db/engine/sqlite/sqlite-source';
|
||||
import type { AnyOrm } from '../../../../../../src/db/orm';
|
||||
import { FakePostgres, fakeMoveOrm, type FakeTableDef } from '../../../../../helpers/fake-postgres-move';
|
||||
import { PostgreSqlPlatform } from '@mikro-orm/postgresql';
|
||||
@@ -487,6 +492,15 @@ describe('the pieces', () => {
|
||||
expect(pluginDefault("strftime('%s','now')", 'integer')).toBe(
|
||||
'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)',
|
||||
);
|
||||
// A default the column's Postgres type would refuse, or store as another value, is not carried over.
|
||||
expect(pluginDefault('0', 'bytes')).toBeNull();
|
||||
expect(pluginDefault("'a'", 'bytes')).toBeNull();
|
||||
expect(pluginDefault('1.5', 'integer')).toBeNull();
|
||||
expect(pluginDefault('1.5', 'float')).toBe('1.5');
|
||||
expect(pluginDefault("'abc'", 'integer')).toBeNull();
|
||||
expect(pluginDefault('CURRENT_TIMESTAMP', 'integer')).toBeNull();
|
||||
expect(pluginDefault('CURRENT_TIMESTAMP', 'text')).toBe(PG_NOW_TEXT_DEFAULT);
|
||||
expect(pluginDefault("strftime('%s','now')", 'bytes')).toBeNull();
|
||||
|
||||
expect(batchRows(2)).toBe(500);
|
||||
expect(batchRows(200)).toBe(150);
|
||||
@@ -524,6 +538,57 @@ describe('the pieces', () => {
|
||||
source.close();
|
||||
expect(fs.readFileSync(path.join(dir, 'travel.db')).equals(before)).toBe(true);
|
||||
});
|
||||
|
||||
it('MOVE-026: text is read by its bytes, so bytes that are not UTF-8 are never replaced unseen', () => {
|
||||
const file = sqlite(
|
||||
path.join(dir, 'bytes.db'),
|
||||
'create table t (v);' +
|
||||
"insert into t values (cast(x'ff41' as text)), (cast(x'f09080' as text)), (cast(x'efbbbf41' as text)), " +
|
||||
"(cast(x'efbfbd' as text)), ('a' || char(0) || 'b'), (''), (x''), (7), (2.5), (null);",
|
||||
);
|
||||
const source = SqliteSource.open(file);
|
||||
const values = [...source.rows('t', ['v'])].map(([value]) => value);
|
||||
source.close();
|
||||
expect(values[0]).toEqual(new InvalidUtf8Text(Buffer.from([0xff, 0x41]), '\uFFFDA'));
|
||||
expect(values[1]).toBeInstanceOf(InvalidUtf8Text);
|
||||
expect(String(values[1])).toBe('\uFFFD');
|
||||
// A byte order mark and a real U+FFFD are text like any other, kept as they are.
|
||||
expect(values.slice(2)).toEqual(['\uFEFFA', '\uFFFD', 'a\u0000b', '', Buffer.alloc(0), 7n, 2.5, null]);
|
||||
});
|
||||
|
||||
it('MOVE-028: text of a file created as UTF-16 is read in that encoding, its bad sequences refused like bad UTF-8', () => {
|
||||
const file = sqlite(
|
||||
path.join(dir, 'utf16.db'),
|
||||
"pragma encoding = 'UTF-16le'; create table t (v);" +
|
||||
"insert into t values ('h\u00e9llo \u{1F600}'), (cast(x'00d841' || x'00' as text)), ('');",
|
||||
);
|
||||
const source = SqliteSource.open(file);
|
||||
const values = [...source.rows('t', ['v'])].map(([value]) => value);
|
||||
source.close();
|
||||
expect(values[0]).toBe('h\u00e9llo \u{1F600}');
|
||||
expect(values[1]).toEqual(new InvalidUtf8Text(Buffer.from([0x00, 0xd8, 0x41, 0x00]), '\uFFFDA'));
|
||||
expect(values[2]).toBe('');
|
||||
});
|
||||
|
||||
it('MOVE-027: a plugin key SQLite compared with NOCASE or RTRIM is listed, since Postgres compares it exactly', async () => {
|
||||
const pluginDir = path.join(dir, 'plugins-data', 'move-tags');
|
||||
fs.mkdirSync(pluginDir, { recursive: true });
|
||||
sqlite(
|
||||
path.join(pluginDir, 'plugin.db'),
|
||||
`create table tags (name text primary key collate nocase, label text, slug text collate rtrim, note text);
|
||||
create unique index tags_label on tags (label collate nocase, slug);
|
||||
create index tags_note on tags (note);
|
||||
insert into tags values ('Alpha', 'a', 's', 'n');`,
|
||||
);
|
||||
const summary = await moveSqliteToPostgres(options());
|
||||
expect(summary.plugins[0].notCarried).toEqual([
|
||||
'the collation NOCASE, RTRIM of the index tags_label on tags (Postgres compares its values exactly)',
|
||||
'the collation NOCASE of the primary key on tags (Postgres compares its values exactly)',
|
||||
]);
|
||||
const statements = pg.statements.join('\n');
|
||||
expect(statements).toContain('create unique index on "trek_plugin_move_tags"."tags" ("label", "slug")');
|
||||
expect(statements).toContain('create index on "trek_plugin_move_tags"."tags" ("note")');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runMoveCommand', () => {
|
||||
|
||||
Reference in New Issue
Block a user