mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
a884539a2b
Plugin statements on Postgres went over the extended query protocol through a marker parameter and a hook that replaced query() on every client of TREK's pool. That kept a plugin to one statement per round trip, but it changed TREK's own hot path for a concern only plugin data has. The plugin data store now checks a pg client out of the ORM's pool for each transaction, through MikroORM's getNativeClient, and runs the transaction on it directly. Every plugin write goes as a query with queryMode 'extended' and every read through a pg-cursor on that client, so Postgres parses exactly one statement either way. The settings the transaction makes are local, and before the client goes back to the pool the store drops temporary tables, cursors, listeners, session advisory locks and sequence state, or closes a client it cannot bring back to that state. The row cap, quota, time budget, untimed migrate, per-plugin queue, pool share and script commits are as before, and plugin args are still sent without NUL. The marker, the hook and its install in the Postgres driver are gone.
291 lines
14 KiB
TypeScript
291 lines
14 KiB
TypeScript
/**
|
|
* PGMOVE-001..006: moving a SQLite install into Postgres
|
|
* (`src/db/engine/postgres/sqlite-move/`), end to end on the lane's server.
|
|
*
|
|
* A SQLite file is migrated by the SQLite engine's own bootstrap and seeded
|
|
* through the test factories with rows across the main tables, including a
|
|
* reply to a chat message (a table that points at itself) and a plugin's own
|
|
* database. The move copies it into this file's database, emptied first, and
|
|
* the app then boots on the result: the rows read back through the entities
|
|
* and the repositories exactly as they read from the SQLite file, the plugin
|
|
* finds its rows in its schema, new ids continue after the copied ones, and
|
|
* a second move into the now filled database is refused.
|
|
*/
|
|
import { buildApp } from '../../src/bootstrap';
|
|
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
|
import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline';
|
|
import { PluginDataSlots, poolClients, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store';
|
|
import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move';
|
|
import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine';
|
|
import { AppSettings } from '../../src/db/entities/AppSettings.entity';
|
|
import { BudgetItems } from '../../src/db/entities/BudgetItems.entity';
|
|
import { CollabMessages } from '../../src/db/entities/CollabMessages.entity';
|
|
import { DayAssignments } from '../../src/db/entities/DayAssignments.entity';
|
|
import { Days } from '../../src/db/entities/Days.entity';
|
|
import { Places } from '../../src/db/entities/Places.entity';
|
|
import { ReservationEndpoints } from '../../src/db/entities/ReservationEndpoints.entity';
|
|
import { Reservations } from '../../src/db/entities/Reservations.entity';
|
|
import { Settings } from '../../src/db/entities/Settings.entity';
|
|
import { TripFiles } from '../../src/db/entities/TripFiles.entity';
|
|
import { Trips } from '../../src/db/entities/Trips.entity';
|
|
import { Users } from '../../src/db/entities/Users.entity';
|
|
import { knownMigrationNames } from '../../src/db/known-migrations';
|
|
import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm';
|
|
import { createBoundSqliteDriver } from '../../src/db/orm-driver';
|
|
import { makeBudgetItem } from '../helpers/factories/budget';
|
|
import { makeCollabMessage } from '../helpers/factories/collab';
|
|
import { inContext, type FactoryOrm } from '../helpers/factories/context';
|
|
import { makeTripFile } from '../helpers/factories/files';
|
|
import { makeDayAssignment } from '../helpers/factories/itinerary';
|
|
import { makePlace } from '../helpers/factories/places';
|
|
import { makeReservation, makeReservationEndpoint } from '../helpers/factories/reservations';
|
|
import { findRows } from '../helpers/factories/rows';
|
|
import { setAppSetting, setUserSetting } from '../helpers/factories/settings';
|
|
import { makeTrip, readTripDays } from '../helpers/factories/trips';
|
|
import { makeUser } from '../helpers/factories/users';
|
|
import { databaseUrl, withAdmin } from './databases';
|
|
import { MikroORM, type EntityClass, type Options } from '@mikro-orm/core';
|
|
import type { INestApplication } from '@nestjs/common';
|
|
|
|
import Database from 'better-sqlite3';
|
|
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
const PLUGIN_ID = 'move-notes';
|
|
|
|
let dir: string;
|
|
let handle: Database.Database;
|
|
let sqliteOrm: AnyOrm;
|
|
let app: INestApplication;
|
|
let pgOrm: MikroORM;
|
|
let summary: MoveSummary;
|
|
let tripId: number;
|
|
|
|
/** An ORM on its own SQLite file, migrated and seeded the way the SQLite engine brings a file to head. */
|
|
async function migratedSqlite(file: string): Promise<AnyOrm> {
|
|
handle = new Database(file);
|
|
handle.pragma('foreign_keys = ON');
|
|
const config = buildOrmConfig(new SqliteEngine());
|
|
// Typed as the engine's plain Options: the bound driver alone would make
|
|
// init infer the SQLite driver's own entity manager, which config does not name.
|
|
const options: Options = {
|
|
...config,
|
|
driver: createBoundSqliteDriver(() => handle),
|
|
dbName: file,
|
|
migrations: { ...config.migrations, silent: true },
|
|
};
|
|
const orm = await MikroORM.init(options);
|
|
await runSchemaBootstrap(orm, { snapshot: false });
|
|
return orm;
|
|
}
|
|
|
|
/** Rows across the main tables, through the factories. */
|
|
async function seed(orm: FactoryOrm): Promise<void> {
|
|
const { user } = await makeUser(orm, { display_name: 'Ada Lovelace' });
|
|
const { user: friend } = await makeUser(orm);
|
|
const trip = await makeTrip(orm, user.id, {
|
|
title: 'Lisbon in spring',
|
|
start_date: '2027-04-01',
|
|
end_date: '2027-04-03',
|
|
currency: 'EUR',
|
|
});
|
|
tripId = trip.id;
|
|
const days = await readTripDays(orm, trip.id);
|
|
const tower = await makePlace(orm, trip.id, { name: 'Torre de Belém', lat: 38.6916, lng: -9.216, notes: 'Go early' });
|
|
const tram = await makePlace(orm, trip.id, { name: 'Tram 28', lat: 38.7139, lng: -9.1334, category: null });
|
|
await makeDayAssignment(orm, days[0].id, tower.id, { notes: 'Morning' });
|
|
await makeDayAssignment(orm, days[1].id, tram.id);
|
|
const flight = await makeReservation(orm, trip.id, { title: 'TP 1234', type: 'flight', day: days[0].id });
|
|
await makeReservationEndpoint(orm, flight.id, { role: 'from', sequence: 0 });
|
|
await makeReservationEndpoint(orm, flight.id, {
|
|
role: 'to',
|
|
sequence: 1,
|
|
name: 'Lisbon Humberto Delgado',
|
|
code: 'LIS',
|
|
lat: 38.7742,
|
|
lng: -9.1342,
|
|
});
|
|
await makeBudgetItem(orm, trip.id, { name: 'Flights', total_price: 412.37, category: 'Transport' });
|
|
await makeBudgetItem(orm, trip.id, { name: 'Pastéis', total_price: 7.5, category: 'Food' });
|
|
await makeTripFile(orm, trip.id, { original_name: 'Boarding pass.pdf', file_size: 52_431 });
|
|
await setUserSetting(orm, user.id, 'temperature_unit', 'celsius');
|
|
await setAppSetting(orm, 'move_test_marker', 'ünïcode ✓');
|
|
const question = await makeCollabMessage(orm, trip.id, user.id, { text: 'Who books the tram?' });
|
|
await makeCollabMessage(orm, trip.id, friend.id, { text: 'I do', replyToRef: question.id });
|
|
}
|
|
|
|
/** A plugin's own database, as the SQLite store keeps it. */
|
|
function seedPluginData(root: string): void {
|
|
fs.mkdirSync(path.join(root, PLUGIN_ID), { recursive: true });
|
|
const plugin = new Database(path.join(root, PLUGIN_ID, 'plugin.db'));
|
|
plugin.exec(
|
|
'create table _plugin_migrations (id text primary key, applied_at integer);' +
|
|
"insert into _plugin_migrations values ('001-notes', 1700000000000);" +
|
|
"create table notes (id integer primary key autoincrement, body text not null default '', score real, raw blob);" +
|
|
'create unique index notes_body on notes (body);' +
|
|
"insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" +
|
|
"delete from notes where body = 'gone';",
|
|
);
|
|
plugin.close();
|
|
}
|
|
|
|
/** The worker's database, emptied: the move fills an empty database only. */
|
|
async function emptyWorkerDatabase(): Promise<void> {
|
|
const adminUrl = process.env.TREK_TEST_DB_URL as string;
|
|
const name = new URL(process.env.TREK_DB_URL as string).pathname.slice(1);
|
|
expect(process.env.TREK_DB_URL).toBe(databaseUrl(adminUrl, name));
|
|
await withAdmin(adminUrl, async (run) => {
|
|
await run(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
|
await run(`CREATE DATABASE "${name}"`);
|
|
});
|
|
}
|
|
|
|
/** Runs the move into the worker's database through an ORM of the Postgres engine's own config. */
|
|
async function move(): Promise<MoveSummary> {
|
|
const config = buildOrmConfig(resolveEngine());
|
|
const orm = await MikroORM.init({
|
|
...config,
|
|
entities: [],
|
|
entitiesTs: [],
|
|
discovery: { warnWhenNoEntities: false },
|
|
migrations: { ...config.migrations, silent: true },
|
|
});
|
|
try {
|
|
return await moveSqliteToPostgres({
|
|
sqliteFile: path.join(dir, 'travel.db'),
|
|
pluginsDataDir: path.join(dir, 'plugins-data'),
|
|
orm,
|
|
known: knownMigrationNames(),
|
|
baseline: readPostgresBaseline,
|
|
log: () => undefined,
|
|
});
|
|
} finally {
|
|
await orm.close(true);
|
|
}
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-pg-move-'));
|
|
sqliteOrm = await migratedSqlite(path.join(dir, 'travel.db'));
|
|
await seed(sqliteOrm);
|
|
seedPluginData(path.join(dir, 'plugins-data'));
|
|
await emptyWorkerDatabase();
|
|
summary = await move();
|
|
app = await buildApp();
|
|
pgOrm = app.get(MikroORM);
|
|
}, 180_000);
|
|
|
|
afterAll(async () => {
|
|
await app?.close();
|
|
await sqliteOrm?.close(true);
|
|
if (handle?.open) handle.close();
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
/** Rows in a stable order: Postgres and SQLite need not sort text alike. */
|
|
function sorted<T>(rows: T[]): T[] {
|
|
return [...rows].sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
|
|
}
|
|
|
|
/** Runs `read` on both databases in a request context of its own. */
|
|
async function bothSides<T>(read: (orm: FactoryOrm) => Promise<T>): Promise<{ sqlite: T; postgres: T }> {
|
|
return { sqlite: await read(sqliteOrm), postgres: await read(pgOrm) };
|
|
}
|
|
|
|
describe('moving a SQLite install to Postgres', () => {
|
|
it('PGMOVE-001: copies every table and reports what it moved', async () => {
|
|
const byLabel = new Map(summary.tables.map((table) => [table.label, table.rows]));
|
|
expect(byLabel.get('users')).toBe((await findRows(sqliteOrm, Users)).length);
|
|
expect(byLabel.get('trips')).toBe(1);
|
|
expect(byLabel.get('days')).toBe(3);
|
|
expect(byLabel.get('reservation_endpoints')).toBe(2);
|
|
expect(byLabel.has('mikro_orm_migrations')).toBe(false);
|
|
expect(summary.sequences).toBeGreaterThan(100);
|
|
expect(summary.plugins).toEqual([
|
|
expect.objectContaining({
|
|
pluginId: PLUGIN_ID,
|
|
schema: 'trek_plugin_move_notes',
|
|
tables: [
|
|
expect.objectContaining({ label: '_plugin_migrations', rows: 1 }),
|
|
expect.objectContaining({ label: 'notes', rows: 2 }),
|
|
],
|
|
}),
|
|
]);
|
|
});
|
|
|
|
const entities: Array<[string, EntityClass<object>]> = [
|
|
['users', Users],
|
|
['trips', Trips],
|
|
['days', Days],
|
|
['places', Places],
|
|
['day_assignments', DayAssignments],
|
|
['reservations', Reservations],
|
|
['reservation_endpoints', ReservationEndpoints],
|
|
['budget_items', BudgetItems],
|
|
['trip_files', TripFiles],
|
|
['settings', Settings],
|
|
['collab_messages', CollabMessages],
|
|
];
|
|
it.each(entities)('PGMOVE-002: %s read through the entities exactly as on SQLite', async (_name, entity) => {
|
|
const { sqlite, postgres } = await bothSides((orm) => findRows(orm, entity));
|
|
expect(sqlite.length).toBeGreaterThan(0);
|
|
expect(sorted(postgres)).toEqual(sorted(sqlite));
|
|
});
|
|
|
|
it('PGMOVE-002b: app_settings keeps every row of the file (the boot adds its own keys beside them)', async () => {
|
|
const { sqlite, postgres } = await bothSides((orm) => findRows(orm, AppSettings));
|
|
expect(sqlite.some((row) => row.key === 'move_test_marker')).toBe(true);
|
|
expect(postgres).toEqual(expect.arrayContaining(sqlite));
|
|
});
|
|
|
|
it('PGMOVE-003: the repositories answer the same on both engines', async () => {
|
|
const { sqlite, postgres } = await bothSides((orm) =>
|
|
inContext(orm, async (em) => ({
|
|
days: await em.getRepository(Days).listByTrip(tripId),
|
|
places: await em.getRepository(Places).listAllForTrip(tripId),
|
|
assignments: await em.getRepository(DayAssignments).listAllForTrip(tripId),
|
|
reservations: await em.getRepository(Reservations).listForTrip(tripId),
|
|
endpoints: await em.getRepository(Reservations).listEndpointsForShare(tripId),
|
|
budget: await em.getRepository(BudgetItems).listWithCategoryOrder(tripId),
|
|
})),
|
|
);
|
|
for (const key of Object.keys(sqlite) as Array<keyof typeof sqlite>) {
|
|
expect(sorted(postgres[key] as unknown[]), key).toEqual(sorted(sqlite[key] as unknown[]));
|
|
}
|
|
expect(postgres.endpoints).toHaveLength(2);
|
|
});
|
|
|
|
it('PGMOVE-004: a reply keeps the message it answers (a table pointing at itself)', async () => {
|
|
const messages = await findRows(pgOrm, CollabMessages);
|
|
const reply = messages.find((message) => message.text === 'I do');
|
|
const question = messages.find((message) => message.text === 'Who books the tram?');
|
|
expect(reply?.reply_to).toBe(question?.id);
|
|
});
|
|
|
|
it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows and its migration record', async () => {
|
|
const { user } = await makeUser(pgOrm);
|
|
const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id));
|
|
expect(user.id).toBeGreaterThan(highest);
|
|
|
|
const store = new PostgresPluginDataStore(PLUGIN_ID, poolClients(pgOrm.em.getConnection()), new PluginDataSlots(1));
|
|
expect(await store.query('select id, body, score from notes order by id')).toEqual([
|
|
{ id: 1, body: 'first', score: 1.5 },
|
|
{ id: 2, body: 'second', score: null },
|
|
]);
|
|
// The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out.
|
|
expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false });
|
|
await store.exec("insert into notes (body) values ('third')");
|
|
expect(await store.query("select id from notes where body = 'third'")).toEqual([{ id: 4 }]);
|
|
store.close();
|
|
});
|
|
|
|
it('PGMOVE-006: a second move into the filled database is refused and changes nothing', async () => {
|
|
const before = await findRows(pgOrm, Trips);
|
|
await expect(move()).rejects.toBeInstanceOf(MoveRefusal);
|
|
await expect(move()).rejects.toThrow(/The Postgres database is not empty/);
|
|
expect(await findRows(pgOrm, Trips)).toEqual(before);
|
|
});
|
|
});
|